usearch 2.21.4 → 2.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (602) hide show
  1. package/README.md +24 -24
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +7 -7
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +83 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +224 -48
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/dist/cjs/usearch.d.ts +8 -3
  16. package/javascript/dist/cjs/usearch.js +12 -4
  17. package/javascript/dist/esm/usearch.d.ts +8 -3
  18. package/javascript/dist/esm/usearch.js +12 -4
  19. package/javascript/lib.cpp +17 -0
  20. package/javascript/usearch.ts +14 -6
  21. package/numkong/.clang-tidy +30 -0
  22. package/{simsimd → numkong}/.cmake-format.py +18 -0
  23. package/numkong/.editorconfig +12 -0
  24. package/numkong/.git-blame-ignore-revs +123 -0
  25. package/numkong/.mailmap +15 -0
  26. package/numkong/.spi.yml +11 -0
  27. package/numkong/.swift-format +14 -0
  28. package/numkong/CITATION.cff +81 -0
  29. package/numkong/CONTRIBUTING.md +500 -0
  30. package/numkong/Cargo.toml +54 -0
  31. package/numkong/MANIFEST.in +8 -0
  32. package/numkong/Package.swift +84 -0
  33. package/numkong/README.md +569 -0
  34. package/numkong/bench/README.md +217 -0
  35. package/numkong/bench/bench-browser.html +323 -0
  36. package/numkong/bench/bench.cpp +344 -0
  37. package/numkong/bench/bench.hpp +816 -0
  38. package/numkong/bench/bench.js +148 -0
  39. package/numkong/bench/bench.mjs +616 -0
  40. package/numkong/bench/bench_attention.cpp +113 -0
  41. package/numkong/bench/bench_cast.cpp +264 -0
  42. package/numkong/bench/bench_cross_amx.cpp +62 -0
  43. package/numkong/bench/bench_cross_arm.cpp +245 -0
  44. package/numkong/bench/bench_cross_blas.cpp +309 -0
  45. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  46. package/numkong/bench/bench_cross_power.cpp +84 -0
  47. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  48. package/numkong/bench/bench_cross_serial.cpp +151 -0
  49. package/numkong/bench/bench_cross_sme.cpp +163 -0
  50. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  51. package/numkong/bench/bench_cross_x86.cpp +455 -0
  52. package/numkong/bench/bench_curved.cpp +210 -0
  53. package/numkong/bench/bench_dot.cpp +315 -0
  54. package/numkong/bench/bench_each.cpp +503 -0
  55. package/numkong/bench/bench_geospatial.cpp +112 -0
  56. package/numkong/bench/bench_maxsim.cpp +154 -0
  57. package/numkong/bench/bench_mesh.cpp +148 -0
  58. package/numkong/bench/bench_probability.cpp +60 -0
  59. package/numkong/bench/bench_reduce.cpp +371 -0
  60. package/numkong/bench/bench_set.cpp +73 -0
  61. package/numkong/bench/bench_sparse.cpp +198 -0
  62. package/numkong/bench/bench_spatial.cpp +347 -0
  63. package/numkong/bench/bench_trigonometry.cpp +137 -0
  64. package/numkong/bench/swift/Bench.swift +506 -0
  65. package/numkong/binding.gyp +169 -0
  66. package/numkong/build.rs +484 -0
  67. package/numkong/c/dispatch.h +512 -0
  68. package/numkong/c/dispatch_bf16.c +447 -0
  69. package/numkong/c/dispatch_bf16c.c +52 -0
  70. package/numkong/c/dispatch_e2m3.c +296 -0
  71. package/numkong/c/dispatch_e3m2.c +284 -0
  72. package/numkong/c/dispatch_e4m3.c +318 -0
  73. package/numkong/c/dispatch_e5m2.c +316 -0
  74. package/numkong/c/dispatch_f16.c +452 -0
  75. package/numkong/c/dispatch_f16c.c +58 -0
  76. package/numkong/c/dispatch_f32.c +434 -0
  77. package/numkong/c/dispatch_f32c.c +99 -0
  78. package/numkong/c/dispatch_f64.c +348 -0
  79. package/numkong/c/dispatch_f64c.c +98 -0
  80. package/numkong/c/dispatch_i16.c +96 -0
  81. package/numkong/c/dispatch_i32.c +89 -0
  82. package/numkong/c/dispatch_i4.c +153 -0
  83. package/numkong/c/dispatch_i64.c +86 -0
  84. package/numkong/c/dispatch_i8.c +348 -0
  85. package/numkong/c/dispatch_other.c +348 -0
  86. package/numkong/c/dispatch_u1.c +193 -0
  87. package/numkong/c/dispatch_u16.c +124 -0
  88. package/numkong/c/dispatch_u32.c +118 -0
  89. package/numkong/c/dispatch_u4.c +153 -0
  90. package/numkong/c/dispatch_u64.c +102 -0
  91. package/numkong/c/dispatch_u8.c +364 -0
  92. package/numkong/c/numkong.c +940 -0
  93. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  94. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  95. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  96. package/numkong/cmake/nk_math_lib.cmake +46 -0
  97. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  99. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  100. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  101. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  102. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  103. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  104. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  105. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  106. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  107. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  108. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  109. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  110. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  111. package/numkong/include/README.md +643 -0
  112. package/numkong/include/module.modulemap +129 -0
  113. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  114. package/numkong/include/numkong/attention/sme.h +2075 -0
  115. package/numkong/include/numkong/attention.h +49 -0
  116. package/numkong/include/numkong/capabilities.h +930 -0
  117. package/numkong/include/numkong/cast/README.md +262 -0
  118. package/numkong/include/numkong/cast/diamond.h +64 -0
  119. package/numkong/include/numkong/cast/haswell.h +823 -0
  120. package/numkong/include/numkong/cast/icelake.h +475 -0
  121. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  122. package/numkong/include/numkong/cast/neon.h +1174 -0
  123. package/numkong/include/numkong/cast/powervsx.h +449 -0
  124. package/numkong/include/numkong/cast/rvv.h +970 -0
  125. package/numkong/include/numkong/cast/sapphire.h +262 -0
  126. package/numkong/include/numkong/cast/serial.h +2336 -0
  127. package/numkong/include/numkong/cast/skylake.h +915 -0
  128. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  129. package/numkong/include/numkong/cast.h +254 -0
  130. package/numkong/include/numkong/cast.hpp +44 -0
  131. package/numkong/include/numkong/curved/README.md +223 -0
  132. package/numkong/include/numkong/curved/genoa.h +182 -0
  133. package/numkong/include/numkong/curved/haswell.h +276 -0
  134. package/numkong/include/numkong/curved/neon.h +329 -0
  135. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  136. package/numkong/include/numkong/curved/rvv.h +305 -0
  137. package/numkong/include/numkong/curved/serial.h +207 -0
  138. package/numkong/include/numkong/curved/skylake.h +457 -0
  139. package/numkong/include/numkong/curved/smef64.h +530 -0
  140. package/numkong/include/numkong/curved.h +513 -0
  141. package/numkong/include/numkong/curved.hpp +144 -0
  142. package/numkong/include/numkong/dot/README.md +442 -0
  143. package/numkong/include/numkong/dot/alder.h +563 -0
  144. package/numkong/include/numkong/dot/diamond.h +158 -0
  145. package/numkong/include/numkong/dot/genoa.h +289 -0
  146. package/numkong/include/numkong/dot/haswell.h +1723 -0
  147. package/numkong/include/numkong/dot/icelake.h +997 -0
  148. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  149. package/numkong/include/numkong/dot/neon.h +869 -0
  150. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  151. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  152. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  153. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  154. package/numkong/include/numkong/dot/powervsx.h +752 -0
  155. package/numkong/include/numkong/dot/rvv.h +722 -0
  156. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  157. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  158. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  159. package/numkong/include/numkong/dot/sapphire.h +141 -0
  160. package/numkong/include/numkong/dot/serial.h +874 -0
  161. package/numkong/include/numkong/dot/sierra.h +405 -0
  162. package/numkong/include/numkong/dot/skylake.h +1116 -0
  163. package/numkong/include/numkong/dot/sve.h +420 -0
  164. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  165. package/numkong/include/numkong/dot/svehalf.h +168 -0
  166. package/numkong/include/numkong/dot/svesdot.h +90 -0
  167. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  168. package/numkong/include/numkong/dot.h +1136 -0
  169. package/numkong/include/numkong/dot.hpp +94 -0
  170. package/numkong/include/numkong/dots/README.md +496 -0
  171. package/numkong/include/numkong/dots/alder.h +117 -0
  172. package/numkong/include/numkong/dots/diamond.h +86 -0
  173. package/numkong/include/numkong/dots/genoa.h +100 -0
  174. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  175. package/numkong/include/numkong/dots/haswell.h +310 -0
  176. package/numkong/include/numkong/dots/icelake.h +180 -0
  177. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  178. package/numkong/include/numkong/dots/neon.h +123 -0
  179. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  180. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  181. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  182. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  183. package/numkong/include/numkong/dots/powervsx.h +194 -0
  184. package/numkong/include/numkong/dots/rvv.h +2593 -0
  185. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  186. package/numkong/include/numkong/dots/serial.h +2867 -0
  187. package/numkong/include/numkong/dots/sierra.h +100 -0
  188. package/numkong/include/numkong/dots/skylake.h +207 -0
  189. package/numkong/include/numkong/dots/sme.h +5041 -0
  190. package/numkong/include/numkong/dots/smebi32.h +477 -0
  191. package/numkong/include/numkong/dots/smef64.h +1328 -0
  192. package/numkong/include/numkong/dots/smehalf.h +47 -0
  193. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  194. package/numkong/include/numkong/dots.h +3110 -0
  195. package/numkong/include/numkong/dots.hpp +641 -0
  196. package/numkong/include/numkong/each/README.md +465 -0
  197. package/numkong/include/numkong/each/haswell.h +1658 -0
  198. package/numkong/include/numkong/each/icelake.h +272 -0
  199. package/numkong/include/numkong/each/neon.h +1138 -0
  200. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  201. package/numkong/include/numkong/each/neonhalf.h +318 -0
  202. package/numkong/include/numkong/each/rvv.h +1121 -0
  203. package/numkong/include/numkong/each/sapphire.h +343 -0
  204. package/numkong/include/numkong/each/serial.h +282 -0
  205. package/numkong/include/numkong/each/skylake.h +1562 -0
  206. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  207. package/numkong/include/numkong/each.h +2232 -0
  208. package/numkong/include/numkong/each.hpp +434 -0
  209. package/numkong/include/numkong/geospatial/README.md +147 -0
  210. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  211. package/numkong/include/numkong/geospatial/neon.h +615 -0
  212. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  213. package/numkong/include/numkong/geospatial/serial.h +309 -0
  214. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  215. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  216. package/numkong/include/numkong/geospatial.h +453 -0
  217. package/numkong/include/numkong/geospatial.hpp +235 -0
  218. package/numkong/include/numkong/matrix.hpp +336 -0
  219. package/numkong/include/numkong/maxsim/README.md +187 -0
  220. package/numkong/include/numkong/maxsim/alder.h +511 -0
  221. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  222. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  223. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  224. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  225. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  226. package/numkong/include/numkong/maxsim/serial.h +490 -0
  227. package/numkong/include/numkong/maxsim/sme.h +993 -0
  228. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  229. package/numkong/include/numkong/maxsim.h +571 -0
  230. package/numkong/include/numkong/maxsim.hpp +133 -0
  231. package/numkong/include/numkong/mesh/README.md +219 -0
  232. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  233. package/numkong/include/numkong/mesh/neon.h +1821 -0
  234. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  235. package/numkong/include/numkong/mesh/rvv.h +936 -0
  236. package/numkong/include/numkong/mesh/serial.h +721 -0
  237. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  238. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  239. package/numkong/include/numkong/mesh.h +666 -0
  240. package/numkong/include/numkong/mesh.hpp +701 -0
  241. package/numkong/include/numkong/numkong.h +78 -0
  242. package/numkong/include/numkong/numkong.hpp +58 -0
  243. package/numkong/include/numkong/probability/README.md +177 -0
  244. package/numkong/include/numkong/probability/haswell.h +267 -0
  245. package/numkong/include/numkong/probability/neon.h +255 -0
  246. package/numkong/include/numkong/probability/rvv.h +414 -0
  247. package/numkong/include/numkong/probability/serial.h +181 -0
  248. package/numkong/include/numkong/probability/skylake.h +324 -0
  249. package/numkong/include/numkong/probability.h +380 -0
  250. package/numkong/include/numkong/probability.hpp +120 -0
  251. package/numkong/include/numkong/random.h +50 -0
  252. package/numkong/include/numkong/random.hpp +285 -0
  253. package/numkong/include/numkong/reduce/README.md +552 -0
  254. package/numkong/include/numkong/reduce/alder.h +636 -0
  255. package/numkong/include/numkong/reduce/genoa.h +201 -0
  256. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  257. package/numkong/include/numkong/reduce/icelake.h +549 -0
  258. package/numkong/include/numkong/reduce/neon.h +3980 -0
  259. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  260. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  261. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  262. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  263. package/numkong/include/numkong/reduce/serial.h +753 -0
  264. package/numkong/include/numkong/reduce/sierra.h +338 -0
  265. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  266. package/numkong/include/numkong/reduce/sve.h +52 -0
  267. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  268. package/numkong/include/numkong/reduce.h +1582 -0
  269. package/numkong/include/numkong/reduce.hpp +778 -0
  270. package/numkong/include/numkong/scalar/README.md +89 -0
  271. package/numkong/include/numkong/scalar/haswell.h +122 -0
  272. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  273. package/numkong/include/numkong/scalar/neon.h +122 -0
  274. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  275. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  276. package/numkong/include/numkong/scalar/rvv.h +211 -0
  277. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  278. package/numkong/include/numkong/scalar/serial.h +332 -0
  279. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  280. package/numkong/include/numkong/scalar.h +696 -0
  281. package/numkong/include/numkong/set/README.md +179 -0
  282. package/numkong/include/numkong/set/haswell.h +334 -0
  283. package/numkong/include/numkong/set/icelake.h +485 -0
  284. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  285. package/numkong/include/numkong/set/neon.h +363 -0
  286. package/numkong/include/numkong/set/powervsx.h +326 -0
  287. package/numkong/include/numkong/set/rvv.h +226 -0
  288. package/numkong/include/numkong/set/rvvbb.h +117 -0
  289. package/numkong/include/numkong/set/serial.h +174 -0
  290. package/numkong/include/numkong/set/sve.h +187 -0
  291. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  292. package/numkong/include/numkong/set.h +471 -0
  293. package/numkong/include/numkong/set.hpp +114 -0
  294. package/numkong/include/numkong/sets/README.md +149 -0
  295. package/numkong/include/numkong/sets/haswell.h +63 -0
  296. package/numkong/include/numkong/sets/icelake.h +66 -0
  297. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  298. package/numkong/include/numkong/sets/neon.h +61 -0
  299. package/numkong/include/numkong/sets/powervsx.h +65 -0
  300. package/numkong/include/numkong/sets/serial.h +43 -0
  301. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  302. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  303. package/numkong/include/numkong/sets.h +382 -0
  304. package/numkong/include/numkong/sparse/README.md +156 -0
  305. package/numkong/include/numkong/sparse/icelake.h +463 -0
  306. package/numkong/include/numkong/sparse/neon.h +288 -0
  307. package/numkong/include/numkong/sparse/serial.h +117 -0
  308. package/numkong/include/numkong/sparse/sve2.h +504 -0
  309. package/numkong/include/numkong/sparse/turin.h +322 -0
  310. package/numkong/include/numkong/sparse.h +363 -0
  311. package/numkong/include/numkong/sparse.hpp +113 -0
  312. package/numkong/include/numkong/spatial/README.md +439 -0
  313. package/numkong/include/numkong/spatial/alder.h +604 -0
  314. package/numkong/include/numkong/spatial/diamond.h +240 -0
  315. package/numkong/include/numkong/spatial/genoa.h +222 -0
  316. package/numkong/include/numkong/spatial/haswell.h +979 -0
  317. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  318. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  319. package/numkong/include/numkong/spatial/neon.h +846 -0
  320. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  321. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  322. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  323. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  324. package/numkong/include/numkong/spatial/rvv.h +991 -0
  325. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  326. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  327. package/numkong/include/numkong/spatial/serial.h +347 -0
  328. package/numkong/include/numkong/spatial/sierra.h +516 -0
  329. package/numkong/include/numkong/spatial/skylake.h +605 -0
  330. package/numkong/include/numkong/spatial/sve.h +246 -0
  331. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  332. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  333. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  334. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  335. package/numkong/include/numkong/spatial.h +1681 -0
  336. package/numkong/include/numkong/spatial.hpp +183 -0
  337. package/numkong/include/numkong/spatials/README.md +580 -0
  338. package/numkong/include/numkong/spatials/alder.h +94 -0
  339. package/numkong/include/numkong/spatials/diamond.h +82 -0
  340. package/numkong/include/numkong/spatials/genoa.h +94 -0
  341. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  342. package/numkong/include/numkong/spatials/haswell.h +219 -0
  343. package/numkong/include/numkong/spatials/icelake.h +113 -0
  344. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  345. package/numkong/include/numkong/spatials/neon.h +109 -0
  346. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  347. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  348. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  349. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  350. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  351. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  352. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  353. package/numkong/include/numkong/spatials/serial.h +248 -0
  354. package/numkong/include/numkong/spatials/sierra.h +96 -0
  355. package/numkong/include/numkong/spatials/skylake.h +184 -0
  356. package/numkong/include/numkong/spatials/sme.h +1927 -0
  357. package/numkong/include/numkong/spatials/smef64.h +483 -0
  358. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  359. package/numkong/include/numkong/spatials.h +3374 -0
  360. package/numkong/include/numkong/spatials.hpp +509 -0
  361. package/numkong/include/numkong/tensor.hpp +1981 -0
  362. package/numkong/include/numkong/trigonometry/README.md +184 -0
  363. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  364. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  365. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  366. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  367. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  368. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  369. package/numkong/include/numkong/trigonometry.h +467 -0
  370. package/numkong/include/numkong/trigonometry.hpp +166 -0
  371. package/numkong/include/numkong/types.h +1695 -0
  372. package/numkong/include/numkong/types.hpp +6027 -0
  373. package/numkong/include/numkong/vector.hpp +766 -0
  374. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  375. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  376. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  377. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  378. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  380. package/numkong/javascript/README.md +254 -0
  381. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  382. package/numkong/javascript/numkong-browser.ts +40 -0
  383. package/numkong/javascript/numkong-wasm.ts +750 -0
  384. package/numkong/javascript/numkong.c +712 -0
  385. package/numkong/javascript/numkong.ts +613 -0
  386. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  387. package/numkong/javascript/tsconfig-cjs.json +13 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  389. package/numkong/javascript/types.ts +784 -0
  390. package/numkong/numkong.gypi +104 -0
  391. package/numkong/package.json +108 -0
  392. package/numkong/probes/arm_neon.c +8 -0
  393. package/numkong/probes/arm_neon_bfdot.c +9 -0
  394. package/numkong/probes/arm_neon_fhm.c +9 -0
  395. package/numkong/probes/arm_neon_half.c +8 -0
  396. package/numkong/probes/arm_neon_sdot.c +9 -0
  397. package/numkong/probes/arm_neonfp8.c +9 -0
  398. package/numkong/probes/arm_sme.c +16 -0
  399. package/numkong/probes/arm_sme2.c +16 -0
  400. package/numkong/probes/arm_sme2p1.c +16 -0
  401. package/numkong/probes/arm_sme_bf16.c +16 -0
  402. package/numkong/probes/arm_sme_bi32.c +16 -0
  403. package/numkong/probes/arm_sme_f64.c +16 -0
  404. package/numkong/probes/arm_sme_fa64.c +14 -0
  405. package/numkong/probes/arm_sme_half.c +16 -0
  406. package/numkong/probes/arm_sme_lut2.c +15 -0
  407. package/numkong/probes/arm_sve.c +18 -0
  408. package/numkong/probes/arm_sve2.c +20 -0
  409. package/numkong/probes/arm_sve2p1.c +18 -0
  410. package/numkong/probes/arm_sve_bfdot.c +20 -0
  411. package/numkong/probes/arm_sve_half.c +18 -0
  412. package/numkong/probes/arm_sve_sdot.c +21 -0
  413. package/numkong/probes/loongarch_lasx.c +12 -0
  414. package/numkong/probes/power_vsx.c +12 -0
  415. package/numkong/probes/probe.js +127 -0
  416. package/numkong/probes/riscv_rvv.c +14 -0
  417. package/numkong/probes/riscv_rvv_bb.c +15 -0
  418. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  419. package/numkong/probes/riscv_rvv_half.c +14 -0
  420. package/numkong/probes/wasm_v128relaxed.c +11 -0
  421. package/numkong/probes/x86_alder.c +17 -0
  422. package/numkong/probes/x86_diamond.c +17 -0
  423. package/numkong/probes/x86_genoa.c +17 -0
  424. package/numkong/probes/x86_graniteamx.c +19 -0
  425. package/numkong/probes/x86_haswell.c +11 -0
  426. package/numkong/probes/x86_icelake.c +17 -0
  427. package/numkong/probes/x86_sapphire.c +16 -0
  428. package/numkong/probes/x86_sapphireamx.c +18 -0
  429. package/numkong/probes/x86_sierra.c +17 -0
  430. package/numkong/probes/x86_skylake.c +15 -0
  431. package/numkong/probes/x86_turin.c +17 -0
  432. package/numkong/rust/README.md +698 -0
  433. package/numkong/rust/capabilities.rs +96 -0
  434. package/numkong/rust/cast.rs +314 -0
  435. package/numkong/rust/curved.rs +391 -0
  436. package/numkong/rust/each.rs +2721 -0
  437. package/numkong/rust/geospatial.rs +343 -0
  438. package/numkong/rust/matrix.rs +5424 -0
  439. package/numkong/rust/maxsim.rs +540 -0
  440. package/numkong/rust/mesh.rs +837 -0
  441. package/numkong/rust/numkong.rs +409 -0
  442. package/numkong/rust/probability.rs +272 -0
  443. package/numkong/rust/reduce.rs +1249 -0
  444. package/numkong/rust/set.rs +194 -0
  445. package/numkong/rust/sparse.rs +513 -0
  446. package/numkong/rust/spatial.rs +1815 -0
  447. package/numkong/rust/tensor.rs +7565 -0
  448. package/numkong/rust/types.rs +3666 -0
  449. package/numkong/rust/vector.rs +1899 -0
  450. package/numkong/rustfmt.toml +2 -0
  451. package/numkong/swift/Matrix.swift +1388 -0
  452. package/numkong/swift/MaxSim.swift +151 -0
  453. package/numkong/swift/NumKong.swift +283 -0
  454. package/numkong/swift/Numerics.swift +329 -0
  455. package/numkong/swift/README.md +433 -0
  456. package/numkong/swift/Scalar.swift +741 -0
  457. package/numkong/swift/Tensor.swift +209 -0
  458. package/numkong/test/README.md +315 -0
  459. package/numkong/test/playwright.config.ts +16 -0
  460. package/numkong/test/swift/Test.swift +534 -0
  461. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  462. package/numkong/test/test-browser-bundle.html +188 -0
  463. package/numkong/test/test-browser-runner.mjs +43 -0
  464. package/numkong/test/test-browser.html +305 -0
  465. package/numkong/test/test-dtypes.mjs +453 -0
  466. package/numkong/test/test-wasm.mjs +267 -0
  467. package/numkong/test/test.cpp +442 -0
  468. package/numkong/test/test.hpp +580 -0
  469. package/numkong/test/test.mjs +363 -0
  470. package/numkong/test/test_base.py +1041 -0
  471. package/numkong/test/test_cast.cpp +242 -0
  472. package/numkong/test/test_cross.hpp +551 -0
  473. package/numkong/test/test_cross.py +467 -0
  474. package/numkong/test/test_cross_amx.cpp +50 -0
  475. package/numkong/test/test_cross_arm.cpp +225 -0
  476. package/numkong/test/test_cross_blas.cpp +354 -0
  477. package/numkong/test/test_cross_loongarch.cpp +90 -0
  478. package/numkong/test/test_cross_power.cpp +73 -0
  479. package/numkong/test/test_cross_rvv.cpp +109 -0
  480. package/numkong/test/test_cross_serial.cpp +181 -0
  481. package/numkong/test/test_cross_sme.cpp +148 -0
  482. package/numkong/test/test_cross_wasm.cpp +141 -0
  483. package/numkong/test/test_cross_x86.cpp +442 -0
  484. package/numkong/test/test_curved.cpp +182 -0
  485. package/numkong/test/test_curved.py +207 -0
  486. package/numkong/test/test_dot.cpp +353 -0
  487. package/numkong/test/test_dot.py +296 -0
  488. package/numkong/test/test_each.cpp +437 -0
  489. package/numkong/test/test_each.py +756 -0
  490. package/numkong/test/test_geospatial.cpp +155 -0
  491. package/numkong/test/test_geospatial.py +241 -0
  492. package/numkong/test/test_maxsim.cpp +138 -0
  493. package/numkong/test/test_maxsim.py +209 -0
  494. package/numkong/test/test_mesh.cpp +203 -0
  495. package/numkong/test/test_mesh.py +394 -0
  496. package/numkong/test/test_probability.cpp +136 -0
  497. package/numkong/test/test_probability.py +166 -0
  498. package/numkong/test/test_reduce.cpp +563 -0
  499. package/numkong/test/test_reduce.py +781 -0
  500. package/numkong/test/test_set.cpp +151 -0
  501. package/numkong/test/test_set.py +120 -0
  502. package/numkong/test/test_similarities.py +579 -0
  503. package/numkong/test/test_sparse.cpp +121 -0
  504. package/numkong/test/test_sparse.py +118 -0
  505. package/numkong/test/test_spatial.cpp +518 -0
  506. package/numkong/test/test_spatial.py +269 -0
  507. package/numkong/test/test_tensor.cpp +884 -0
  508. package/numkong/test/test_tensor.py +1417 -0
  509. package/numkong/test/test_trigonometry.cpp +156 -0
  510. package/numkong/test/test_trigonometry.py +180 -0
  511. package/package.json +7 -8
  512. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  513. package/prebuilds/linux-arm64/usearch.node +0 -0
  514. package/prebuilds/linux-x64/usearch.node +0 -0
  515. package/fp16/CMakeLists.txt +0 -176
  516. package/fp16/LICENSE +0 -11
  517. package/fp16/README.md +0 -20
  518. package/fp16/bench/alt-element.cc +0 -89
  519. package/fp16/bench/from-alt-array.cc +0 -175
  520. package/fp16/bench/from-ieee-array.cc +0 -399
  521. package/fp16/bench/ieee-element.cc +0 -272
  522. package/fp16/bench/to-alt-array.cc +0 -91
  523. package/fp16/bench/to-ieee-array.cc +0 -303
  524. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  525. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  526. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  527. package/fp16/configure.py +0 -65
  528. package/fp16/confu.yaml +0 -8
  529. package/fp16/include/fp16/__init__.py +0 -0
  530. package/fp16/include/fp16/avx.py +0 -50
  531. package/fp16/include/fp16/avx2.py +0 -53
  532. package/fp16/include/fp16/bitcasts.h +0 -92
  533. package/fp16/include/fp16/fp16.h +0 -451
  534. package/fp16/include/fp16/psimd.h +0 -131
  535. package/fp16/include/fp16.h +0 -11
  536. package/fp16/jni/Android.mk +0 -6
  537. package/fp16/jni/Application.mk +0 -4
  538. package/fp16/test/alt-from-fp32-value.cc +0 -498
  539. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  540. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  541. package/fp16/test/alt-to-fp32-value.cc +0 -348
  542. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  543. package/fp16/test/bitcasts.cc +0 -56
  544. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  545. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  546. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  547. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  548. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  549. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  550. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  551. package/fp16/test/peachpy/stubs.py +0 -38
  552. package/fp16/test/tables.cc +0 -530
  553. package/fp16/test/tables.h +0 -13
  554. package/fp16/third-party/THHalf.h +0 -92
  555. package/fp16/third-party/eigen-half.h +0 -170
  556. package/fp16/third-party/float16-compressor.h +0 -80
  557. package/fp16/third-party/half.hpp +0 -2916
  558. package/fp16/third-party/npy-halffloat.h +0 -328
  559. package/simsimd/.git-blame-ignore-revs +0 -2
  560. package/simsimd/CONTRIBUTING.md +0 -321
  561. package/simsimd/Cargo.toml +0 -49
  562. package/simsimd/MANIFEST.in +0 -4
  563. package/simsimd/Package.swift +0 -28
  564. package/simsimd/README.md +0 -1493
  565. package/simsimd/binding.gyp +0 -35
  566. package/simsimd/build.rs +0 -70
  567. package/simsimd/c/lib.c +0 -374
  568. package/simsimd/include/module.modulemap +0 -4
  569. package/simsimd/include/simsimd/binary.h +0 -486
  570. package/simsimd/include/simsimd/curved.h +0 -1541
  571. package/simsimd/include/simsimd/dot.h +0 -1853
  572. package/simsimd/include/simsimd/elementwise.h +0 -2465
  573. package/simsimd/include/simsimd/geospatial.h +0 -43
  574. package/simsimd/include/simsimd/mesh.h +0 -69
  575. package/simsimd/include/simsimd/probability.h +0 -606
  576. package/simsimd/include/simsimd/simsimd.h +0 -2567
  577. package/simsimd/include/simsimd/sparse.h +0 -1384
  578. package/simsimd/include/simsimd/spatial.h +0 -2335
  579. package/simsimd/include/simsimd/types.h +0 -666
  580. package/simsimd/javascript/fallback.ts +0 -233
  581. package/simsimd/javascript/lib.c +0 -125
  582. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  583. package/simsimd/javascript/simsimd.ts +0 -179
  584. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  585. package/simsimd/package-lock.json +0 -1761
  586. package/simsimd/package.json +0 -61
  587. package/simsimd/rust/lib.rs +0 -1821
  588. package/simsimd/scripts/bench.cxx +0 -1170
  589. package/simsimd/scripts/bench.ipynb +0 -187
  590. package/simsimd/scripts/bench.js +0 -127
  591. package/simsimd/scripts/bench.rs +0 -876
  592. package/simsimd/scripts/bench_vectors.py +0 -734
  593. package/simsimd/scripts/bench_vectors_live.py +0 -213
  594. package/simsimd/scripts/test.c +0 -187
  595. package/simsimd/scripts/test.mjs +0 -220
  596. package/simsimd/scripts/test.py +0 -1505
  597. package/simsimd/swift/SimSIMD.swift +0 -149
  598. package/simsimd/swift/Test.swift +0 -128
  599. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  600. /package/{simsimd → numkong}/LICENSE +0 -0
  601. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  602. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -1,2465 +0,0 @@
1
- /**
2
- * @file elementwise.h
3
- * @brief SIMD-accelerated mixed-precision element-wise operations.
4
- * @author Ash Vardanian
5
- * @date October 16, 2024
6
- *
7
- * Contains following element-wise operations:
8
- * - Sum (Add): R[i] = A[i] + B[i]
9
- * - Scale (Multiply): R[i] = Alpha * A[i]
10
- * - WSum or Weighted-Sum: R[i] = Alpha * A[i] + Beta * B[i]
11
- * - FMA or Fused-Multiply-Add: R[i] = Alpha * A[i] * B[i] + Beta * C[i]
12
- *
13
- * This tiny set of operations if enough to implement a wide range of algorithms.
14
- * To scale a vector by a scalar, just call WSum with $Beta$ = 0.
15
- * To sum two vectors, just call WSum with $Alpha$ = $Beta$ = 1.
16
- * To average two vectors, just call WSum with $Alpha$ = $Beta$ = 0.5.
17
- * To multiply vectors element-wise, just call FMA with $Beta$ = 0.
18
- *
19
- * For datatypes:
20
- * - 64-bit IEEE floating point numbers
21
- * - 32-bit IEEE floating point numbers
22
- * - 16-bit IEEE floating point numbers
23
- * - 16-bit brain floating point numbers
24
- * - 8-bit unsigned integers
25
- * - 8-bit signed integers
26
- *
27
- * For hardware architectures:
28
- * - Arm: NEON
29
- * - x86: Haswell, Skylake, Sapphire
30
- *
31
- * We use `f16` for `i8` and `u8` arithmetic. This is because Arm received `f16` support earlier than `bf16`.
32
- * For example, Apple M1 has `f16` support and `bf16` was only added in M2. On the other hand, on paper,
33
- * AMD Genoa has `bf16` support, and `f16` is only available on Intel Sapphire Rapids and newer.
34
- * Sadly, the SIMD support for `bf16` is limited to mixed-precision dot-products, which makes it useless here.
35
- *
36
- * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
37
- * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
38
- */
39
- #ifndef SIMSIMD_ELEMENTWISE_H
40
- #define SIMSIMD_ELEMENTWISE_H
41
-
42
- #include "types.h"
43
-
44
- #ifdef __cplusplus
45
- extern "C" {
46
- #endif
47
-
48
- /* Serial backends for all numeric types.
49
- * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
50
- * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
51
- */
52
- SIMSIMD_PUBLIC void simsimd_wsum_f64_serial( //
53
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
54
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
55
- SIMSIMD_PUBLIC void simsimd_wsum_f32_serial( //
56
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
57
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
58
- SIMSIMD_PUBLIC void simsimd_wsum_f16_serial( //
59
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
60
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
61
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_serial( //
62
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
63
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
64
- SIMSIMD_PUBLIC void simsimd_wsum_i8_serial( //
65
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
66
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
67
- SIMSIMD_PUBLIC void simsimd_wsum_u8_serial( //
68
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
69
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
70
-
71
- SIMSIMD_PUBLIC void simsimd_fma_f64_serial( //
72
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
73
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
74
- SIMSIMD_PUBLIC void simsimd_fma_f32_serial( //
75
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
76
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
77
- SIMSIMD_PUBLIC void simsimd_fma_f16_serial( //
78
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
79
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
80
- SIMSIMD_PUBLIC void simsimd_fma_bf16_serial( //
81
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
82
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
83
- SIMSIMD_PUBLIC void simsimd_fma_i8_serial( //
84
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
85
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
86
- SIMSIMD_PUBLIC void simsimd_fma_u8_serial( //
87
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
88
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
89
-
90
- #define SIMSIMD_MAKE_WSUM(name, input_type, accumulator_type, load_and_convert, convert_and_store) \
91
- SIMSIMD_PUBLIC void simsimd_wsum_##input_type##_##name( \
92
- simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_size_t n, \
93
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_##input_type##_t *result) { \
94
- for (simsimd_size_t i = 0; i != n; ++i) { \
95
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
96
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
97
- simsimd_##accumulator_type##_t ai_scaled = (simsimd_##accumulator_type##_t)(ai * alpha); \
98
- simsimd_##accumulator_type##_t bi_scaled = (simsimd_##accumulator_type##_t)(bi * beta); \
99
- simsimd_##accumulator_type##_t sum = ai_scaled + bi_scaled; \
100
- convert_and_store(sum, result + i); \
101
- } \
102
- }
103
-
104
- #define SIMSIMD_MAKE_FMA(name, input_type, accumulator_type, load_and_convert, convert_and_store) \
105
- SIMSIMD_PUBLIC void simsimd_fma_##input_type##_##name( \
106
- simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
107
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_##input_type##_t *result) { \
108
- for (simsimd_size_t i = 0; i != n; ++i) { \
109
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
110
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
111
- simsimd_##accumulator_type##_t ci = load_and_convert(c + i); \
112
- simsimd_##accumulator_type##_t abi_scaled = (simsimd_##accumulator_type##_t)(ai * bi * alpha); \
113
- simsimd_##accumulator_type##_t ci_scaled = (simsimd_##accumulator_type##_t)(ci * beta); \
114
- simsimd_##accumulator_type##_t sum = abi_scaled + ci_scaled; \
115
- convert_and_store(sum, result + i); \
116
- } \
117
- }
118
-
119
- SIMSIMD_MAKE_WSUM(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f64_serial
120
- SIMSIMD_MAKE_WSUM(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f32_serial
121
- SIMSIMD_MAKE_WSUM(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_wsum_f16_serial
122
- SIMSIMD_MAKE_WSUM(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_wsum_bf16_serial
123
- SIMSIMD_MAKE_WSUM(serial, i8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_I8) // simsimd_wsum_i8_serial
124
- SIMSIMD_MAKE_WSUM(serial, u8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_U8) // simsimd_wsum_u8_serial
125
-
126
- SIMSIMD_MAKE_WSUM(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f32_accurate
127
- SIMSIMD_MAKE_WSUM(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_wsum_f16_accurate
128
- SIMSIMD_MAKE_WSUM(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_wsum_bf16_accurate
129
- SIMSIMD_MAKE_WSUM(accurate, i8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_I8) // simsimd_wsum_i8_accurate
130
- SIMSIMD_MAKE_WSUM(accurate, u8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_U8) // simsimd_wsum_u8_accurate
131
-
132
- SIMSIMD_MAKE_FMA(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f64_serial
133
- SIMSIMD_MAKE_FMA(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f32_serial
134
- SIMSIMD_MAKE_FMA(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_fma_f16_serial
135
- SIMSIMD_MAKE_FMA(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_fma_bf16_serial
136
- SIMSIMD_MAKE_FMA(serial, i8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_I8) // simsimd_fma_i8_serial
137
- SIMSIMD_MAKE_FMA(serial, u8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_U8) // simsimd_fma_u8_serial
138
-
139
- SIMSIMD_MAKE_FMA(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f32_accurate
140
- SIMSIMD_MAKE_FMA(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_fma_f16_accurate
141
- SIMSIMD_MAKE_FMA(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_fma_bf16_accurate
142
- SIMSIMD_MAKE_FMA(accurate, i8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_I8) // simsimd_fma_i8_accurate
143
- SIMSIMD_MAKE_FMA(accurate, u8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_U8) // simsimd_fma_u8_accurate
144
-
145
- /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
146
- * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
147
- * server CPUs produced before 2023.
148
- */
149
- SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
150
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
151
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
152
- SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
153
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
154
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
155
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
156
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
157
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
158
- SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
159
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
160
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
161
- SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
162
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
163
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
164
-
165
- SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
166
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
167
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
168
- SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
169
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
170
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
171
- SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( //
172
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
173
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
174
- SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
175
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, //
176
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
177
- SIMSIMD_PUBLIC void simsimd_fma_i8_neon( //
178
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, //
179
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
180
-
181
- /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
182
- * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
183
- * Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
184
- * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
185
- * properly vectorized by recent compilers.
186
- */
187
- SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
188
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
189
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
190
- SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
191
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
192
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
193
- SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
194
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
195
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
196
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
197
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
198
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
199
- SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
200
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
201
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
202
- SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
203
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
204
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
205
- SIMSIMD_PUBLIC void simsimd_fma_f64_haswell( //
206
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
207
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
208
- SIMSIMD_PUBLIC void simsimd_fma_f32_haswell( //
209
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
210
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
211
- SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
212
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
213
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
214
- SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( //
215
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
216
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
217
- SIMSIMD_PUBLIC void simsimd_fma_i8_haswell( //
218
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
219
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
220
- SIMSIMD_PUBLIC void simsimd_fma_u8_haswell( //
221
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
222
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
223
-
224
- /* SIMD-powered backends for various generations of AVX512 CPUs.
225
- * Unlike the distance metrics, the SIMD implementation of FMA and WSum benefits from aligned stores.
226
- * Assuming the size of ZMM register matches the width of the cache line, we skip the unaligned head
227
- * and tail of the output buffer, and only use aligned stores in the main loop.
228
- */
229
- SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
230
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
231
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
232
- SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
233
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
234
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
235
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
236
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
237
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
238
-
239
- SIMSIMD_PUBLIC void simsimd_fma_f64_skylake( //
240
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
241
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
242
- SIMSIMD_PUBLIC void simsimd_fma_f32_skylake( //
243
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
244
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
245
- SIMSIMD_PUBLIC void simsimd_fma_bf16_skylake( //
246
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
247
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
248
-
249
- SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
250
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
251
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
252
- SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
253
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
254
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
255
- SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
256
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
257
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
258
-
259
- SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire( //
260
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
261
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
262
- SIMSIMD_PUBLIC void simsimd_fma_i8_sapphire( //
263
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
264
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
265
- SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire( //
266
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
267
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
268
-
269
- #if _SIMSIMD_TARGET_X86
270
- #if SIMSIMD_TARGET_HASWELL
271
- #pragma GCC push_options
272
- #pragma GCC target("avx2", "f16c", "fma")
273
- #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
274
-
275
- SIMSIMD_PUBLIC void simsimd_sum_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
276
- simsimd_f32_t *result) {
277
- // The main loop:
278
- simsimd_size_t i = 0;
279
- for (; i + 8 <= n; i += 8) {
280
- __m256 a_vec = _mm256_loadu_ps(a + i);
281
- __m256 b_vec = _mm256_loadu_ps(b + i);
282
- __m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
283
- _mm256_storeu_ps(result + i, sum_vec);
284
- }
285
-
286
- // The tail:
287
- for (; i < n; ++i) result[i] = a[i] + b[i];
288
- }
289
-
290
- SIMSIMD_PUBLIC void simsimd_scale_f32_haswell(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
291
- simsimd_f32_t *result) {
292
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
293
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
294
-
295
- // The main loop:
296
- simsimd_size_t i = 0;
297
- for (; i + 8 <= n; i += 8) {
298
- __m256 a_vec = _mm256_loadu_ps(a + i);
299
- __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
300
- _mm256_storeu_ps(result + i, sum_vec);
301
- }
302
-
303
- // The tail:
304
- for (; i < n; ++i) result[i] = alpha_f32 * a[i];
305
- }
306
-
307
- SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
308
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
309
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
310
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
311
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
312
-
313
- // There are are several special cases we may want to implement:
314
- // 1. Simple addition, when both weights are equal to 1.0.
315
- if (alpha == 1 && beta == 1) {
316
- // In this case we can avoid expensive multiplications.
317
- simsimd_sum_f32_haswell(a, b, n, result);
318
- return;
319
- }
320
- // 2. Just scaling, when one of the weights is equal to zero.
321
- else if (alpha == 0 || beta == 0) {
322
- // In this case we can avoid half of the load instructions.
323
- if (beta == 0) { simsimd_scale_f32_haswell(a, n, alpha, result); }
324
- else { simsimd_scale_f32_haswell(b, n, beta, result); }
325
- return;
326
- }
327
-
328
- // The general case.
329
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
330
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
331
-
332
- // The main loop:
333
- simsimd_size_t i = 0;
334
- for (; i + 8 <= n; i += 8) {
335
- __m256 a_vec = _mm256_loadu_ps(a + i);
336
- __m256 b_vec = _mm256_loadu_ps(b + i);
337
- __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
338
- __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
339
- __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
340
- _mm256_storeu_ps(result + i, sum_vec);
341
- }
342
-
343
- // The tail:
344
- for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
345
- }
346
-
347
- SIMSIMD_PUBLIC void simsimd_sum_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
348
- simsimd_f64_t *result) {
349
- // The main loop:
350
- simsimd_size_t i = 0;
351
- for (; i + 4 <= n; i += 4) {
352
- __m256d a_vec = _mm256_loadu_pd(a + i);
353
- __m256d b_vec = _mm256_loadu_pd(b + i);
354
- __m256d sum_vec = _mm256_add_pd(a_vec, b_vec);
355
- _mm256_storeu_pd(result + i, sum_vec);
356
- }
357
-
358
- // The tail:
359
- for (; i < n; ++i) result[i] = a[i] + b[i];
360
- }
361
-
362
- SIMSIMD_PUBLIC void simsimd_scale_f64_haswell(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
363
- simsimd_f64_t *result) {
364
- __m256d alpha_vec = _mm256_set1_pd(alpha);
365
-
366
- // The main loop:
367
- simsimd_size_t i = 0;
368
- for (; i + 4 <= n; i += 4) {
369
- __m256d a_vec = _mm256_loadu_pd(a + i);
370
- __m256d sum_vec = _mm256_mul_pd(a_vec, alpha_vec);
371
- _mm256_storeu_pd(result + i, sum_vec);
372
- }
373
-
374
- // The tail:
375
- for (; i < n; ++i) result[i] = alpha * a[i];
376
- }
377
-
378
- SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
379
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
380
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
381
-
382
- // There are are several special cases we may want to implement:
383
- // 1. Simple addition, when both weights are equal to 1.0.
384
- if (alpha == 1 && beta == 1) {
385
- // In this case we can avoid expensive multiplications.
386
- simsimd_sum_f64_haswell(a, b, n, result);
387
- return;
388
- }
389
- // 2. Just scaling, when one of the weights is equal to zero.
390
- else if (alpha == 0 || beta == 0) {
391
- // In this case we can avoid half of the load instructions.
392
- if (beta == 0) { simsimd_scale_f64_haswell(a, n, alpha, result); }
393
- else { simsimd_scale_f64_haswell(b, n, beta, result); }
394
- return;
395
- }
396
-
397
- // The general case.
398
- __m256d alpha_vec = _mm256_set1_pd(alpha);
399
- __m256d beta_vec = _mm256_set1_pd(beta);
400
-
401
- // The main loop:
402
- simsimd_size_t i = 0;
403
- for (; i + 4 <= n; i += 4) {
404
- __m256d a_vec = _mm256_loadu_pd(a + i);
405
- __m256d b_vec = _mm256_loadu_pd(b + i);
406
- __m256d a_scaled_vec = _mm256_mul_pd(a_vec, alpha_vec);
407
- __m256d b_scaled_vec = _mm256_mul_pd(b_vec, beta_vec);
408
- __m256d sum_vec = _mm256_add_pd(a_scaled_vec, b_scaled_vec);
409
- _mm256_storeu_pd(result + i, sum_vec);
410
- }
411
-
412
- // The tail:
413
- for (; i < n; ++i) result[i] = alpha * a[i] + beta * b[i];
414
- }
415
-
416
- SIMSIMD_PUBLIC void simsimd_sum_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
417
- simsimd_f16_t *result) {
418
-
419
- // The main loop:
420
- simsimd_size_t i = 0;
421
- for (; i + 8 <= n; i += 8) {
422
- __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
423
- __m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
424
- __m256 a_vec = _mm256_cvtph_ps(a_f16);
425
- __m256 b_vec = _mm256_cvtph_ps(b_f16);
426
- __m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
427
- __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
428
- _mm_storeu_si128((__m128i *)(result + i), sum_f16);
429
- }
430
-
431
- // The tail:
432
- for (; i < n; ++i) {
433
- simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
434
- simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
435
- simsimd_f32_t sum = ai + bi;
436
- SIMSIMD_F32_TO_F16(sum, result + i);
437
- }
438
- }
439
-
440
- SIMSIMD_PUBLIC void simsimd_scale_f16_haswell(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
441
- simsimd_f16_t *result) {
442
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
443
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
444
-
445
- // The main loop:
446
- simsimd_size_t i = 0;
447
- for (; i + 8 <= n; i += 8) {
448
- __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
449
- __m256 a_vec = _mm256_cvtph_ps(a_f16);
450
- __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
451
- __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
452
- _mm_storeu_si128((__m128i *)(result + i), sum_f16);
453
- }
454
-
455
- // The tail:
456
- for (; i < n; ++i) {
457
- simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
458
- simsimd_f32_t sum = alpha_f32 * ai;
459
- SIMSIMD_F32_TO_F16(sum, result + i);
460
- }
461
- }
462
-
463
- SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
464
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
465
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
466
-
467
- // There are are several special cases we may want to implement:
468
- // 1. Simple addition, when both weights are equal to 1.0.
469
- if (alpha == 1 && beta == 1) {
470
- // In this case we can avoid expensive multiplications.
471
- simsimd_sum_f16_haswell(a, b, n, result);
472
- return;
473
- }
474
- // 2. Just scaling, when one of the weights is equal to zero.
475
- else if (alpha == 0 || beta == 0) {
476
- // In this case we can avoid half of the load instructions.
477
- if (beta == 0) { simsimd_scale_f16_haswell(a, n, alpha, result); }
478
- else { simsimd_scale_f16_haswell(b, n, beta, result); }
479
- return;
480
- }
481
-
482
- // The general case.
483
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
484
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
485
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
486
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
487
-
488
- // The main loop:
489
- simsimd_size_t i = 0;
490
- for (; i + 8 <= n; i += 8) {
491
- __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
492
- __m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
493
- __m256 a_vec = _mm256_cvtph_ps(a_f16);
494
- __m256 b_vec = _mm256_cvtph_ps(b_f16);
495
- __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
496
- __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
497
- __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
498
- __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
499
- _mm_storeu_si128((__m128i *)(result + i), sum_f16);
500
- }
501
-
502
- // The tail:
503
- for (; i < n; ++i) {
504
- simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
505
- simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
506
- simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
507
- SIMSIMD_F32_TO_F16(sum, result + i);
508
- }
509
- }
510
-
511
- SIMSIMD_PUBLIC void simsimd_sum_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
512
- simsimd_bf16_t *result) {
513
- // The main loop:
514
- simsimd_size_t i = 0;
515
- for (; i + 8 <= n; i += 8) {
516
- __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
517
- __m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
518
- __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
519
- __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
520
- __m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
521
- __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
522
- _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
523
- }
524
-
525
- // The tail:
526
- for (; i < n; ++i) {
527
- simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
528
- simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
529
- simsimd_f32_t sum = ai + bi;
530
- SIMSIMD_F32_TO_BF16(sum, result + i);
531
- }
532
- }
533
-
534
- SIMSIMD_PUBLIC void simsimd_scale_bf16_haswell(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
535
- simsimd_bf16_t *result) {
536
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
537
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
538
-
539
- // The main loop:
540
- simsimd_size_t i = 0;
541
- for (; i + 8 <= n; i += 8) {
542
- __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
543
- __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
544
- __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
545
- __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
546
- _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
547
- }
548
-
549
- // The tail:
550
- for (; i < n; ++i) {
551
- simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
552
- simsimd_f32_t sum = alpha_f32 * ai;
553
- SIMSIMD_F32_TO_BF16(sum, result + i);
554
- }
555
- }
556
-
557
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
558
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
559
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
560
-
561
- // There are are several special cases we may want to implement:
562
- // 1. Simple addition, when both weights are equal to 1.0.
563
- if (alpha == 1 && beta == 1) {
564
- // In this case we can avoid expensive multiplications.
565
- simsimd_sum_bf16_haswell(a, b, n, result);
566
- return;
567
- }
568
- // 2. Just scaling, when one of the weights is equal to zero.
569
- else if (alpha == 0 || beta == 0) {
570
- // In this case we can avoid half of the load instructions.
571
- if (beta == 0) { simsimd_scale_bf16_haswell(a, n, alpha, result); }
572
- else { simsimd_scale_bf16_haswell(b, n, beta, result); }
573
- return;
574
- }
575
-
576
- // The general case.
577
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
578
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
579
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
580
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
581
-
582
- // The main loop:
583
- simsimd_size_t i = 0;
584
- for (; i + 8 <= n; i += 8) {
585
- __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
586
- __m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
587
- __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
588
- __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
589
- __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
590
- __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
591
- __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
592
- __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
593
- _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
594
- }
595
-
596
- // The tail:
597
- for (; i < n; ++i) {
598
- simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
599
- simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
600
- simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
601
- SIMSIMD_F32_TO_BF16(sum, result + i);
602
- }
603
- }
604
-
605
- SIMSIMD_PUBLIC void simsimd_fma_f32_haswell( //
606
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
607
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
608
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
609
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
610
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
611
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
612
-
613
- // The main loop:
614
- simsimd_size_t i = 0;
615
- for (; i + 8 <= n; i += 8) {
616
- __m256 a_vec = _mm256_loadu_ps(a + i);
617
- __m256 b_vec = _mm256_loadu_ps(b + i);
618
- __m256 c_vec = _mm256_loadu_ps(c + i);
619
- __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
620
- __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
621
- __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
622
- __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
623
- _mm256_storeu_ps(result + i, sum_vec);
624
- }
625
-
626
- // The tail:
627
- for (; i < n; ++i) result[i] = alpha_f32 * a[i] * b[i] + beta_f32 * c[i];
628
- }
629
-
630
- SIMSIMD_PUBLIC void simsimd_fma_f64_haswell( //
631
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, //
632
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
633
- __m256d alpha_vec = _mm256_set1_pd(alpha);
634
- __m256d beta_vec = _mm256_set1_pd(beta);
635
-
636
- // The main loop:
637
- simsimd_size_t i = 0;
638
- for (; i + 4 <= n; i += 4) {
639
- __m256d a_vec = _mm256_loadu_pd(a + i);
640
- __m256d b_vec = _mm256_loadu_pd(b + i);
641
- __m256d c_vec = _mm256_loadu_pd(c + i);
642
- __m256d ab_vec = _mm256_mul_pd(a_vec, b_vec);
643
- __m256d ab_scaled_vec = _mm256_mul_pd(ab_vec, alpha_vec);
644
- __m256d c_scaled_vec = _mm256_mul_pd(c_vec, beta_vec);
645
- __m256d sum_vec = _mm256_add_pd(ab_scaled_vec, c_scaled_vec);
646
- _mm256_storeu_pd(result + i, sum_vec);
647
- }
648
-
649
- // The tail:
650
- for (; i < n; ++i) result[i] = alpha * a[i] * b[i] + beta * c[i];
651
- }
652
-
653
- SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
654
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
655
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
656
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
657
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
658
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
659
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
660
-
661
- // The main loop:
662
- simsimd_size_t i = 0;
663
- for (; i + 8 <= n; i += 8) {
664
- __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
665
- __m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
666
- __m128i c_f16 = _mm_lddqu_si128((__m128i const *)(c + i));
667
- __m256 a_vec = _mm256_cvtph_ps(a_f16);
668
- __m256 b_vec = _mm256_cvtph_ps(b_f16);
669
- __m256 c_vec = _mm256_cvtph_ps(c_f16);
670
- __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
671
- __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
672
- __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
673
- __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
674
- __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
675
- _mm_storeu_si128((__m128i *)(result + i), sum_f16);
676
- }
677
-
678
- // The tail:
679
- for (; i < n; ++i) {
680
- simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
681
- simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
682
- simsimd_f32_t ci = SIMSIMD_F16_TO_F32(c + i);
683
- simsimd_f32_t sum = alpha * ai * bi + beta * ci;
684
- SIMSIMD_F32_TO_F16(sum, result + i);
685
- }
686
- }
687
-
688
- SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( //
689
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
690
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
691
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
692
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
693
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
694
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
695
-
696
- // The main loop:
697
- simsimd_size_t i = 0;
698
- for (; i + 8 <= n; i += 8) {
699
- __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
700
- __m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
701
- __m128i c_bf16 = _mm_lddqu_si128((__m128i const *)(c + i));
702
- __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
703
- __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
704
- __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(c_bf16);
705
- __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
706
- __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
707
- __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
708
- __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
709
- __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
710
- _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
711
- }
712
-
713
- // The tail:
714
- for (; i < n; ++i) {
715
- simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
716
- simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
717
- simsimd_f32_t ci = SIMSIMD_BF16_TO_F32(c + i);
718
- simsimd_f32_t sum = alpha * ai * bi + beta * ci;
719
- SIMSIMD_F32_TO_BF16(sum, result + i);
720
- }
721
- }
722
-
723
- SIMSIMD_PUBLIC void simsimd_sum_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
724
- simsimd_i8_t *result) {
725
- // The main loop:
726
- simsimd_size_t i = 0;
727
- for (; i + 32 <= n; i += 32) {
728
- __m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
729
- __m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
730
- __m256i sum_vec = _mm256_adds_epi8(a_vec, b_vec);
731
- _mm256_storeu_si256((__m256i *)(result + i), sum_vec);
732
- }
733
-
734
- // The tail:
735
- for (; i < n; ++i) {
736
- simsimd_f32_t ai = a[i], bi = b[i];
737
- simsimd_f32_t sum = ai + bi;
738
- SIMSIMD_F32_TO_U8(sum, result + i);
739
- }
740
- }
741
-
742
- SIMSIMD_PUBLIC void simsimd_scale_i8_haswell(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
743
- simsimd_i8_t *result) {
744
-
745
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
746
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
747
- int sum_i32s[8], a_i32s[8];
748
-
749
- // The main loop:
750
- simsimd_size_t i = 0;
751
- for (; i + 8 <= n; i += 8) {
752
- //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
753
- //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
754
- a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
755
- a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
756
- //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
757
- //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
758
- __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
759
- // The normal part.
760
- __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
761
- // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
762
- __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
763
- sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
764
- sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
765
- // Export into a serial buffer.
766
- _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
767
- result[i + 0] = (simsimd_i8_t)sum_i32s[0];
768
- result[i + 1] = (simsimd_i8_t)sum_i32s[1];
769
- result[i + 2] = (simsimd_i8_t)sum_i32s[2];
770
- result[i + 3] = (simsimd_i8_t)sum_i32s[3];
771
- result[i + 4] = (simsimd_i8_t)sum_i32s[4];
772
- result[i + 5] = (simsimd_i8_t)sum_i32s[5];
773
- result[i + 6] = (simsimd_i8_t)sum_i32s[6];
774
- result[i + 7] = (simsimd_i8_t)sum_i32s[7];
775
- }
776
-
777
- // The tail:
778
- for (; i < n; ++i) {
779
- simsimd_f32_t ai = a[i];
780
- simsimd_f32_t sum = alpha_f32 * ai;
781
- SIMSIMD_F32_TO_I8(sum, result + i);
782
- }
783
- }
784
-
785
- SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
786
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
787
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
788
-
789
- // There are are several special cases we may want to implement:
790
- // 1. Simple addition, when both weights are equal to 1.0.
791
- if (alpha == 1 && beta == 1) {
792
- // In this case we can avoid expensive multiplications.
793
- simsimd_sum_i8_haswell(a, b, n, result);
794
- return;
795
- }
796
- // 2. Just scaling, when one of the weights is equal to zero.
797
- else if (alpha == 0 || beta == 0) {
798
- // In this case we can avoid half of the load instructions.
799
- if (beta == 0) { simsimd_scale_i8_haswell(a, n, alpha, result); }
800
- else { simsimd_scale_i8_haswell(b, n, beta, result); }
801
- return;
802
- }
803
-
804
- // The general case.
805
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
806
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
807
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
808
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
809
- int sum_i32s[8], a_i32s[8], b_i32s[8];
810
-
811
- // The main loop:
812
- simsimd_size_t i = 0;
813
- for (; i + 8 <= n; i += 8) {
814
- //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
815
- //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
816
- a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
817
- a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
818
- b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
819
- b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
820
- //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
821
- //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
822
- __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
823
- __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
824
- // The normal part.
825
- __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
826
- __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
827
- __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
828
- // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
829
- __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
830
- sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
831
- sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
832
- // Export into a serial buffer.
833
- _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
834
- result[i + 0] = (simsimd_i8_t)sum_i32s[0];
835
- result[i + 1] = (simsimd_i8_t)sum_i32s[1];
836
- result[i + 2] = (simsimd_i8_t)sum_i32s[2];
837
- result[i + 3] = (simsimd_i8_t)sum_i32s[3];
838
- result[i + 4] = (simsimd_i8_t)sum_i32s[4];
839
- result[i + 5] = (simsimd_i8_t)sum_i32s[5];
840
- result[i + 6] = (simsimd_i8_t)sum_i32s[6];
841
- result[i + 7] = (simsimd_i8_t)sum_i32s[7];
842
- }
843
-
844
- // The tail:
845
- for (; i < n; ++i) {
846
- simsimd_f32_t ai = a[i], bi = b[i];
847
- simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
848
- SIMSIMD_F32_TO_I8(sum, result + i);
849
- }
850
- }
851
-
852
- SIMSIMD_PUBLIC void simsimd_sum_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
853
- simsimd_u8_t *result) {
854
- // The main loop:
855
- simsimd_size_t i = 0;
856
- for (; i + 32 <= n; i += 32) {
857
- __m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
858
- __m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
859
- __m256i sum_vec = _mm256_adds_epu8(a_vec, b_vec);
860
- _mm256_storeu_si256((__m256i *)(result + i), sum_vec);
861
- }
862
-
863
- // The tail:
864
- for (; i < n; ++i) {
865
- simsimd_f32_t ai = a[i], bi = b[i];
866
- simsimd_f32_t sum = ai + bi;
867
- SIMSIMD_F32_TO_U8(sum, result + i);
868
- }
869
- }
870
-
871
- SIMSIMD_PUBLIC void simsimd_scale_u8_haswell(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
872
- simsimd_u8_t *result) {
873
-
874
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
875
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
876
- int sum_i32s[8], a_i32s[8];
877
-
878
- // The main loop:
879
- simsimd_size_t i = 0;
880
- for (; i + 8 <= n; i += 8) {
881
- //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
882
- //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
883
- a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
884
- a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
885
- //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
886
- //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
887
- __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
888
- // The normal part.
889
- __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
890
- // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
891
- __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
892
- sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
893
- sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
894
- // Export into a serial buffer.
895
- _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
896
- result[i + 0] = (simsimd_u8_t)sum_i32s[0];
897
- result[i + 1] = (simsimd_u8_t)sum_i32s[1];
898
- result[i + 2] = (simsimd_u8_t)sum_i32s[2];
899
- result[i + 3] = (simsimd_u8_t)sum_i32s[3];
900
- result[i + 4] = (simsimd_u8_t)sum_i32s[4];
901
- result[i + 5] = (simsimd_u8_t)sum_i32s[5];
902
- result[i + 6] = (simsimd_u8_t)sum_i32s[6];
903
- result[i + 7] = (simsimd_u8_t)sum_i32s[7];
904
- }
905
-
906
- // The tail:
907
- for (; i < n; ++i) {
908
- simsimd_f32_t ai = a[i];
909
- simsimd_f32_t sum = alpha_f32 * ai;
910
- SIMSIMD_F32_TO_U8(sum, result + i);
911
- }
912
- }
913
-
914
- SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
915
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
916
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
917
-
918
- // There are are several special cases we may want to implement:
919
- // 1. Simple addition, when both weights are equal to 1.0.
920
- if (alpha == 1 && beta == 1) {
921
- // In this case we can avoid expensive multiplications.
922
- simsimd_sum_u8_haswell(a, b, n, result);
923
- return;
924
- }
925
- // 2. Just scaling, when one of the weights is equal to zero.
926
- else if (alpha == 0 || beta == 0) {
927
- // In this case we can avoid half of the load instructions.
928
- if (beta == 0) { simsimd_scale_u8_haswell(a, n, alpha, result); }
929
- else { simsimd_scale_u8_haswell(b, n, beta, result); }
930
- return;
931
- }
932
-
933
- // The general case.
934
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
935
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
936
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
937
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
938
- int sum_i32s[8], a_i32s[8], b_i32s[8];
939
-
940
- // The main loop:
941
- simsimd_size_t i = 0;
942
- for (; i + 8 <= n; i += 8) {
943
- //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
944
- //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
945
- a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
946
- a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
947
- b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
948
- b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
949
- //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
950
- //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
951
- __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
952
- __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
953
- // The normal part.
954
- __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
955
- __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
956
- __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
957
- // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
958
- __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
959
- sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
960
- sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
961
- // Export into a serial buffer.
962
- _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
963
- result[i + 0] = (simsimd_u8_t)sum_i32s[0];
964
- result[i + 1] = (simsimd_u8_t)sum_i32s[1];
965
- result[i + 2] = (simsimd_u8_t)sum_i32s[2];
966
- result[i + 3] = (simsimd_u8_t)sum_i32s[3];
967
- result[i + 4] = (simsimd_u8_t)sum_i32s[4];
968
- result[i + 5] = (simsimd_u8_t)sum_i32s[5];
969
- result[i + 6] = (simsimd_u8_t)sum_i32s[6];
970
- result[i + 7] = (simsimd_u8_t)sum_i32s[7];
971
- }
972
-
973
- // The tail:
974
- for (; i < n; ++i) {
975
- simsimd_f32_t ai = a[i], bi = b[i];
976
- simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
977
- SIMSIMD_F32_TO_U8(sum, result + i);
978
- }
979
- }
980
-
981
- SIMSIMD_PUBLIC void simsimd_fma_i8_haswell( //
982
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
983
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
984
-
985
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
986
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
987
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
988
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
989
- int sum_i32s[8], a_i32s[8], b_i32s[8], c_i32s[8];
990
-
991
- // The main loop:
992
- simsimd_size_t i = 0;
993
- for (; i + 8 <= n; i += 8) {
994
- //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
995
- //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
996
- a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
997
- a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
998
- b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
999
- b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
1000
- c_i32s[0] = c[i + 0], c_i32s[1] = c[i + 1], c_i32s[2] = c[i + 2], c_i32s[3] = c[i + 3], //
1001
- c_i32s[4] = c[i + 4], c_i32s[5] = c[i + 5], c_i32s[6] = c[i + 6], c_i32s[7] = c[i + 7];
1002
- //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
1003
- //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
1004
- __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
1005
- __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
1006
- __m256 c_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)c_i32s));
1007
- // The normal part.
1008
- __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
1009
- __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
1010
- __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
1011
- __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
1012
- // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
1013
- __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
1014
- sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
1015
- sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
1016
- // Export into a serial buffer.
1017
- _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
1018
- result[i + 0] = (simsimd_i8_t)sum_i32s[0];
1019
- result[i + 1] = (simsimd_i8_t)sum_i32s[1];
1020
- result[i + 2] = (simsimd_i8_t)sum_i32s[2];
1021
- result[i + 3] = (simsimd_i8_t)sum_i32s[3];
1022
- result[i + 4] = (simsimd_i8_t)sum_i32s[4];
1023
- result[i + 5] = (simsimd_i8_t)sum_i32s[5];
1024
- result[i + 6] = (simsimd_i8_t)sum_i32s[6];
1025
- result[i + 7] = (simsimd_i8_t)sum_i32s[7];
1026
- }
1027
-
1028
- // The tail:
1029
- for (; i < n; ++i) {
1030
- simsimd_f32_t ai = a[i], bi = b[i], ci = c[i];
1031
- simsimd_f32_t sum = alpha_f32 * ai * bi + beta_f32 * ci;
1032
- SIMSIMD_F32_TO_I8(sum, result + i);
1033
- }
1034
- }
1035
-
1036
- SIMSIMD_PUBLIC void simsimd_fma_u8_haswell( //
1037
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
1038
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
1039
-
1040
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1041
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1042
- __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
1043
- __m256 beta_vec = _mm256_set1_ps(beta_f32);
1044
- int sum_i32s[8], a_i32s[8], b_i32s[8], c_i32s[8];
1045
-
1046
- // The main loop:
1047
- simsimd_size_t i = 0;
1048
- for (; i + 8 <= n; i += 8) {
1049
- //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
1050
- //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
1051
- a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
1052
- a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
1053
- b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
1054
- b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
1055
- c_i32s[0] = c[i + 0], c_i32s[1] = c[i + 1], c_i32s[2] = c[i + 2], c_i32s[3] = c[i + 3], //
1056
- c_i32s[4] = c[i + 4], c_i32s[5] = c[i + 5], c_i32s[6] = c[i + 6], c_i32s[7] = c[i + 7];
1057
- //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
1058
- //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
1059
- __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
1060
- __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
1061
- __m256 c_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)c_i32s));
1062
- // The normal part.
1063
- __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
1064
- __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
1065
- __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
1066
- __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
1067
- // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
1068
- __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
1069
- sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
1070
- sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
1071
- // Export into a serial buffer.
1072
- _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
1073
- result[i + 0] = (simsimd_u8_t)sum_i32s[0];
1074
- result[i + 1] = (simsimd_u8_t)sum_i32s[1];
1075
- result[i + 2] = (simsimd_u8_t)sum_i32s[2];
1076
- result[i + 3] = (simsimd_u8_t)sum_i32s[3];
1077
- result[i + 4] = (simsimd_u8_t)sum_i32s[4];
1078
- result[i + 5] = (simsimd_u8_t)sum_i32s[5];
1079
- result[i + 6] = (simsimd_u8_t)sum_i32s[6];
1080
- result[i + 7] = (simsimd_u8_t)sum_i32s[7];
1081
- }
1082
-
1083
- // The tail:
1084
- for (; i < n; ++i) {
1085
- simsimd_f32_t ai = a[i], bi = b[i], ci = c[i];
1086
- simsimd_f32_t sum = alpha_f32 * ai * bi + beta_f32 * ci;
1087
- SIMSIMD_F32_TO_U8(sum, result + i);
1088
- }
1089
- }
1090
-
1091
- #pragma clang attribute pop
1092
- #pragma GCC pop_options
1093
- #endif // SIMSIMD_TARGET_HASWELL
1094
-
1095
- #if SIMSIMD_TARGET_SKYLAKE
1096
- #pragma GCC push_options
1097
- #pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
1098
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
1099
-
1100
- SIMSIMD_PUBLIC void simsimd_sum_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1101
- simsimd_f64_t *result) {
1102
- __m512d a_vec, b_vec, sum_vec;
1103
- __mmask8 mask = 0xFF;
1104
- simsimd_sum_f64_skylake_cycle:
1105
- if (n < 8) {
1106
- mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1107
- a_vec = _mm512_maskz_loadu_pd(mask, a);
1108
- b_vec = _mm512_maskz_loadu_pd(mask, b);
1109
- n = 0;
1110
- }
1111
- else {
1112
- a_vec = _mm512_loadu_pd(a);
1113
- b_vec = _mm512_loadu_pd(b);
1114
- a += 8, b += 8, n -= 8;
1115
- }
1116
- sum_vec = _mm512_add_pd(a_vec, b_vec);
1117
- _mm512_mask_storeu_pd(result, mask, sum_vec);
1118
- result += 8;
1119
- if (n) goto simsimd_sum_f64_skylake_cycle;
1120
- }
1121
-
1122
- SIMSIMD_PUBLIC void simsimd_scale_f64_skylake(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1123
- simsimd_f64_t *result) {
1124
- __m512d alpha_vec = _mm512_set1_pd(alpha);
1125
- __m512d a_vec, b_vec, a_scaled_vec, sum_vec;
1126
- __mmask8 mask = 0xFF;
1127
- simsimd_scale_f64_skylake_cycle:
1128
- if (n < 8) {
1129
- mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1130
- a_vec = _mm512_maskz_loadu_pd(mask, a);
1131
- n = 0;
1132
- }
1133
- else {
1134
- a_vec = _mm512_loadu_pd(a);
1135
- a += 8, n -= 8;
1136
- }
1137
- sum_vec = _mm512_mul_pd(a_vec, alpha_vec);
1138
- _mm512_mask_storeu_pd(result, mask, sum_vec);
1139
- result += 8;
1140
- if (n) goto simsimd_scale_f64_skylake_cycle;
1141
- }
1142
-
1143
- SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
1144
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
1145
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
1146
-
1147
- // There are are several special cases we may want to implement:
1148
- // 1. Simple addition, when both weights are equal to 1.0.
1149
- if (alpha == 1 && beta == 1) {
1150
- // In this case we can avoid expensive multiplications.
1151
- simsimd_sum_f64_skylake(a, b, n, result);
1152
- return;
1153
- }
1154
- // 2. Just scaling, when one of the weights is equal to zero.
1155
- else if (alpha == 0 || beta == 0) {
1156
- // In this case we can avoid half of the load instructions.
1157
- if (beta == 0) { simsimd_scale_f64_skylake(a, n, alpha, result); }
1158
- else { simsimd_scale_f64_skylake(b, n, beta, result); }
1159
- return;
1160
- }
1161
-
1162
- // The general case.
1163
- __m512d alpha_vec = _mm512_set1_pd(alpha);
1164
- __m512d beta_vec = _mm512_set1_pd(beta);
1165
- __m512d a_vec, b_vec, a_scaled_vec, sum_vec;
1166
- __mmask8 mask = 0xFF;
1167
- simsimd_wsum_f64_skylake_cycle:
1168
- if (n < 8) {
1169
- mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1170
- a_vec = _mm512_maskz_loadu_pd(mask, a);
1171
- b_vec = _mm512_maskz_loadu_pd(mask, b);
1172
- n = 0;
1173
- }
1174
- else {
1175
- a_vec = _mm512_loadu_pd(a);
1176
- b_vec = _mm512_loadu_pd(b);
1177
- a += 8, b += 8, n -= 8;
1178
- }
1179
- a_scaled_vec = _mm512_mul_pd(a_vec, alpha_vec);
1180
- sum_vec = _mm512_fmadd_pd(b_vec, beta_vec, a_scaled_vec);
1181
- _mm512_mask_storeu_pd(result, mask, sum_vec);
1182
- result += 8;
1183
- if (n) goto simsimd_wsum_f64_skylake_cycle;
1184
- }
1185
-
1186
- SIMSIMD_PUBLIC void simsimd_sum_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1187
- simsimd_f32_t *result) {
1188
- __m512 a_vec, b_vec, sum_vec;
1189
- __mmask16 mask = 0xFFFF;
1190
-
1191
- simsimd_sum_f32_skylake_cycle:
1192
- if (n < 16) {
1193
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1194
- a_vec = _mm512_maskz_loadu_ps(mask, a);
1195
- b_vec = _mm512_maskz_loadu_ps(mask, b);
1196
- n = 0;
1197
- }
1198
- else {
1199
- a_vec = _mm512_loadu_ps(a);
1200
- b_vec = _mm512_loadu_ps(b);
1201
- a += 16, b += 16, n -= 16;
1202
- }
1203
- sum_vec = _mm512_add_ps(a_vec, b_vec);
1204
- _mm512_mask_storeu_ps(result, mask, sum_vec);
1205
- result += 16;
1206
- if (n) goto simsimd_sum_f32_skylake_cycle;
1207
- }
1208
-
1209
- SIMSIMD_PUBLIC void simsimd_scale_f32_skylake(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1210
- simsimd_f32_t *result) {
1211
- __m512 alpha_vec = _mm512_set1_ps(alpha);
1212
- __m512 a_vec, sum_vec;
1213
- __mmask16 mask = 0xFFFF;
1214
-
1215
- simsimd_scale_f32_skylake_cycle:
1216
- if (n < 16) {
1217
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1218
- a_vec = _mm512_maskz_loadu_ps(mask, a);
1219
- n = 0;
1220
- }
1221
- else {
1222
- a_vec = _mm512_loadu_ps(a);
1223
- a += 16, n -= 16;
1224
- }
1225
- sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
1226
- _mm512_mask_storeu_ps(result, mask, sum_vec);
1227
- result += 16;
1228
- if (n) goto simsimd_scale_f32_skylake_cycle;
1229
- }
1230
-
1231
- SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
1232
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
1233
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1234
-
1235
- // There are are several special cases we may want to implement:
1236
- // 1. Simple addition, when both weights are equal to 1.0.
1237
- if (alpha == 1 && beta == 1) {
1238
- // In this case we can avoid expensive multiplications.
1239
- simsimd_sum_f32_skylake(a, b, n, result);
1240
- return;
1241
- }
1242
- // 2. Just scaling, when one of the weights is equal to zero.
1243
- else if (alpha == 0 || beta == 0) {
1244
- // In this case we can avoid half of the load instructions.
1245
- if (beta == 0) { simsimd_scale_f32_skylake(a, n, alpha, result); }
1246
- else { simsimd_scale_f32_skylake(b, n, beta, result); }
1247
- return;
1248
- }
1249
-
1250
- // The general case.
1251
- __m512 alpha_vec = _mm512_set1_ps(alpha);
1252
- __m512 beta_vec = _mm512_set1_ps(beta);
1253
- __m512 a_vec, b_vec, a_scaled_vec, sum_vec;
1254
- __mmask16 mask = 0xFFFF;
1255
- simsimd_wsum_f32_skylake_cycle:
1256
- if (n < 16) {
1257
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1258
- a_vec = _mm512_maskz_loadu_ps(mask, a);
1259
- b_vec = _mm512_maskz_loadu_ps(mask, b);
1260
- n = 0;
1261
- }
1262
- else {
1263
- a_vec = _mm512_loadu_ps(a);
1264
- b_vec = _mm512_loadu_ps(b);
1265
- a += 16, b += 16, n -= 16;
1266
- }
1267
- a_scaled_vec = _mm512_mul_ps(a_vec, alpha_vec);
1268
- sum_vec = _mm512_fmadd_ps(b_vec, beta_vec, a_scaled_vec);
1269
- _mm512_mask_storeu_ps(result, mask, sum_vec);
1270
- result += 16;
1271
- if (n) goto simsimd_wsum_f32_skylake_cycle;
1272
- }
1273
-
1274
- SIMSIMD_PUBLIC void simsimd_sum_bf16_skylake(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1275
- simsimd_bf16_t *result) {
1276
- __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
1277
- __m512 a_vec, b_vec, sum_vec;
1278
- __mmask16 mask = 0xFFFF;
1279
- simsimd_sum_bf16_skylake_cycle:
1280
- if (n < 16) {
1281
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1282
- a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
1283
- b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
1284
- n = 0;
1285
- }
1286
- else {
1287
- a_bf16_vec = _mm256_loadu_epi16(a);
1288
- b_bf16_vec = _mm256_loadu_epi16(b);
1289
- a += 16, b += 16, n -= 16;
1290
- }
1291
- a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
1292
- b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
1293
- sum_vec = _mm512_add_ps(a_vec, b_vec);
1294
- sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
1295
- _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
1296
- result += 16;
1297
- if (n) goto simsimd_sum_bf16_skylake_cycle;
1298
- }
1299
-
1300
- SIMSIMD_PUBLIC void simsimd_scale_bf16_skylake(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1301
- simsimd_bf16_t *result) {
1302
- __m512 alpha_vec = _mm512_set1_ps(alpha);
1303
- __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
1304
- __m512 a_vec, b_vec, sum_vec;
1305
- __mmask16 mask = 0xFFFF;
1306
- simsimd_wsum_bf16_skylake_cycle:
1307
- if (n < 16) {
1308
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1309
- a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
1310
- n = 0;
1311
- }
1312
- else {
1313
- a_bf16_vec = _mm256_loadu_epi16(a);
1314
- a += 16, n -= 16;
1315
- }
1316
- a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
1317
- sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
1318
- sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
1319
- _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
1320
- result += 16;
1321
- if (n) goto simsimd_wsum_bf16_skylake_cycle;
1322
- }
1323
-
1324
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
1325
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
1326
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
1327
-
1328
- // There are are several special cases we may want to implement:
1329
- // 1. Simple addition, when both weights are equal to 1.0.
1330
- if (alpha == 1 && beta == 1) {
1331
- // In this case we can avoid expensive multiplications.
1332
- simsimd_sum_bf16_skylake(a, b, n, result);
1333
- return;
1334
- }
1335
- // 2. Just scaling, when one of the weights is equal to zero.
1336
- else if (alpha == 0 || beta == 0) {
1337
- // In this case we can avoid half of the load instructions.
1338
- if (beta == 0) { simsimd_scale_bf16_skylake(a, n, alpha, result); }
1339
- else { simsimd_scale_bf16_skylake(b, n, beta, result); }
1340
- return;
1341
- }
1342
-
1343
- // The general case.
1344
- __m512 alpha_vec = _mm512_set1_ps(alpha);
1345
- __m512 beta_vec = _mm512_set1_ps(beta);
1346
- __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
1347
- __m512 a_vec, b_vec, a_scaled_vec, sum_vec;
1348
- __mmask16 mask = 0xFFFF;
1349
- simsimd_wsum_bf16_skylake_cycle:
1350
- if (n < 16) {
1351
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1352
- a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
1353
- b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
1354
- n = 0;
1355
- }
1356
- else {
1357
- a_bf16_vec = _mm256_loadu_epi16(a);
1358
- b_bf16_vec = _mm256_loadu_epi16(b);
1359
- a += 16, b += 16, n -= 16;
1360
- }
1361
- a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
1362
- b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
1363
- a_scaled_vec = _mm512_mul_ps(a_vec, alpha_vec);
1364
- sum_vec = _mm512_fmadd_ps(b_vec, beta_vec, a_scaled_vec);
1365
- sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
1366
- _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
1367
- result += 16;
1368
- if (n) goto simsimd_wsum_bf16_skylake_cycle;
1369
- }
1370
-
1371
- SIMSIMD_PUBLIC void simsimd_fma_f64_skylake( //
1372
- simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
1373
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
1374
- __m512d alpha_vec = _mm512_set1_pd(alpha);
1375
- __m512d beta_vec = _mm512_set1_pd(beta);
1376
- __m512d a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
1377
- __mmask8 mask = 0xFF;
1378
-
1379
- simsimd_fma_f64_skylake_cycle:
1380
- if (n < 8) {
1381
- mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1382
- a_vec = _mm512_maskz_loadu_pd(mask, a);
1383
- b_vec = _mm512_maskz_loadu_pd(mask, b);
1384
- c_vec = _mm512_maskz_loadu_pd(mask, c);
1385
- n = 0;
1386
- }
1387
- else {
1388
- a_vec = _mm512_loadu_pd(a);
1389
- b_vec = _mm512_loadu_pd(b);
1390
- c_vec = _mm512_loadu_pd(c);
1391
- a += 8, b += 8, c += 8, n -= 8;
1392
- }
1393
- ab_vec = _mm512_mul_pd(a_vec, b_vec);
1394
- ab_scaled_vec = _mm512_mul_pd(ab_vec, alpha_vec);
1395
- sum_vec = _mm512_fmadd_pd(c_vec, beta_vec, ab_scaled_vec);
1396
- _mm512_mask_storeu_pd(result, mask, sum_vec);
1397
- result += 8;
1398
- if (n) goto simsimd_fma_f64_skylake_cycle;
1399
- }
1400
-
1401
- SIMSIMD_PUBLIC void simsimd_fma_f32_skylake( //
1402
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
1403
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1404
- __m512 alpha_vec = _mm512_set1_ps(alpha);
1405
- __m512 beta_vec = _mm512_set1_ps(beta);
1406
- __m512 a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
1407
- __mmask16 mask = 0xFFFF;
1408
-
1409
- simsimd_fma_f32_skylake_cycle:
1410
- if (n < 16) {
1411
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1412
- a_vec = _mm512_maskz_loadu_ps(mask, a);
1413
- b_vec = _mm512_maskz_loadu_ps(mask, b);
1414
- c_vec = _mm512_maskz_loadu_ps(mask, c);
1415
- n = 0;
1416
- }
1417
- else {
1418
- a_vec = _mm512_loadu_ps(a);
1419
- b_vec = _mm512_loadu_ps(b);
1420
- c_vec = _mm512_loadu_ps(c);
1421
- a += 16, b += 16, c += 16, n -= 16;
1422
- }
1423
- ab_vec = _mm512_mul_ps(a_vec, b_vec);
1424
- ab_scaled_vec = _mm512_mul_ps(ab_vec, alpha_vec);
1425
- sum_vec = _mm512_fmadd_ps(c_vec, beta_vec, ab_scaled_vec);
1426
- _mm512_mask_storeu_ps(result, mask, sum_vec);
1427
- result += 16;
1428
- if (n) goto simsimd_fma_f32_skylake_cycle;
1429
- }
1430
-
1431
- SIMSIMD_PUBLIC void simsimd_fma_bf16_skylake( //
1432
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
1433
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
1434
- __m512 alpha_vec = _mm512_set1_ps(alpha);
1435
- __m512 beta_vec = _mm512_set1_ps(beta);
1436
- __m256i a_bf16_vec, b_bf16_vec, c_bf16_vec, sum_bf16_vec;
1437
- __m512 a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
1438
- __mmask16 mask = 0xFFFF;
1439
-
1440
- simsimd_fma_bf16_skylake_cycle:
1441
- if (n < 16) {
1442
- mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1443
- a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
1444
- b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
1445
- c_bf16_vec = _mm256_maskz_loadu_epi16(mask, c);
1446
- n = 0;
1447
- }
1448
- else {
1449
- a_bf16_vec = _mm256_loadu_epi16(a);
1450
- b_bf16_vec = _mm256_loadu_epi16(b);
1451
- c_bf16_vec = _mm256_loadu_epi16(c);
1452
- a += 16, b += 16, c += 16, n -= 16;
1453
- }
1454
- a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
1455
- b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
1456
- c_vec = _simsimd_bf16x16_to_f32x16_skylake(c_bf16_vec);
1457
- ab_vec = _mm512_mul_ps(a_vec, b_vec);
1458
- ab_scaled_vec = _mm512_mul_ps(ab_vec, alpha_vec);
1459
- sum_vec = _mm512_fmadd_ps(c_vec, beta_vec, ab_scaled_vec);
1460
- sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
1461
- _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
1462
- result += 16;
1463
- if (n) goto simsimd_fma_bf16_skylake_cycle;
1464
- }
1465
-
1466
- #pragma clang attribute pop
1467
- #pragma GCC pop_options
1468
- #endif // SIMSIMD_TARGET_SKYLAKE
1469
-
1470
- #if SIMSIMD_TARGET_SAPPHIRE
1471
- #pragma GCC push_options
1472
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16", "f16c")
1473
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16,f16c"))), \
1474
- apply_to = function)
1475
-
1476
- /**
1477
- * Using `_mm512_set1_ph((_Float16)1.f)` results in compilation warnings if we are pedantic.
1478
- * https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-8/details-about-intrinsics-for-half-floats.html
1479
- */
1480
- SIMSIMD_INTERNAL __m512h _mm512_set1_ph_from_ps(float a) {
1481
- unsigned short h = _cvtss_sh(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
1482
- return (__m512h)_mm512_set1_epi16(h);
1483
- }
1484
-
1485
- SIMSIMD_PUBLIC void simsimd_sum_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1486
- simsimd_f16_t *result) {
1487
- __mmask32 mask = 0xFFFFFFFF;
1488
- __m512h a_f16_vec, b_f16_vec;
1489
- __m512h sum_f16_vec;
1490
- simsimd_sum_f16_sapphire_cycle:
1491
- if (n < 32) {
1492
- mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1493
- a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
1494
- b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
1495
- n = 0;
1496
- }
1497
- else {
1498
- a_f16_vec = _mm512_loadu_ph(a);
1499
- b_f16_vec = _mm512_loadu_ph(b);
1500
- a += 32, b += 32, n -= 32;
1501
- }
1502
- sum_f16_vec = _mm512_add_ph(a_f16_vec, b_f16_vec);
1503
- _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
1504
- result += 32;
1505
- if (n) goto simsimd_sum_f16_sapphire_cycle;
1506
- }
1507
-
1508
- SIMSIMD_PUBLIC void simsimd_scale_f16_sapphire(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1509
- simsimd_f16_t *result) {
1510
-
1511
- __mmask32 mask = 0xFFFFFFFF;
1512
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1513
- __m512h a_f16_vec, b_f16_vec;
1514
- __m512h sum_f16_vec;
1515
- simsimd_scale_f16_sapphire_cycle:
1516
- if (n < 32) {
1517
- mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1518
- a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
1519
- n = 0;
1520
- }
1521
- else {
1522
- a_f16_vec = _mm512_loadu_ph(a);
1523
- a += 32, n -= 32;
1524
- }
1525
- sum_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
1526
- _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
1527
- result += 32;
1528
- if (n) goto simsimd_scale_f16_sapphire_cycle;
1529
- }
1530
-
1531
- SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
1532
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
1533
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
1534
-
1535
- // There are are several special cases we may want to implement:
1536
- // 1. Simple addition, when both weights are equal to 1.0.
1537
- if (alpha == 1 && beta == 1) {
1538
- // In this case we can avoid expensive multiplications.
1539
- simsimd_sum_f16_sapphire(a, b, n, result);
1540
- return;
1541
- }
1542
- // 2. Just scaling, when one of the weights is equal to zero.
1543
- else if (alpha == 0 || beta == 0) {
1544
- // In this case we can avoid half of the load instructions.
1545
- if (beta == 0) { simsimd_scale_f16_sapphire(a, n, alpha, result); }
1546
- else { simsimd_scale_f16_sapphire(b, n, beta, result); }
1547
- return;
1548
- }
1549
-
1550
- // The general case.
1551
- __mmask32 mask = 0xFFFFFFFF;
1552
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1553
- __m512h beta_vec = _mm512_set1_ph_from_ps(beta);
1554
- __m512h a_f16_vec, b_f16_vec;
1555
- __m512h a_scaled_f16_vec, sum_f16_vec;
1556
- simsimd_wsum_f16_sapphire_cycle:
1557
- if (n < 32) {
1558
- mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1559
- a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
1560
- b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
1561
- n = 0;
1562
- }
1563
- else {
1564
- a_f16_vec = _mm512_loadu_ph(a);
1565
- b_f16_vec = _mm512_loadu_ph(b);
1566
- a += 32, b += 32, n -= 32;
1567
- }
1568
- a_scaled_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
1569
- sum_f16_vec = _mm512_fmadd_ph(b_f16_vec, beta_vec, a_scaled_f16_vec);
1570
- _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
1571
- result += 32;
1572
- if (n) goto simsimd_wsum_f16_sapphire_cycle;
1573
- }
1574
-
1575
- SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire( //
1576
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
1577
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
1578
-
1579
- __mmask32 mask = 0xFFFFFFFF;
1580
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1581
- __m512h beta_vec = _mm512_set1_ph_from_ps(beta);
1582
- __m512h a_f16_vec, b_f16_vec, c_f16_vec;
1583
- __m512h ab_f16_vec, ab_scaled_f16_vec, sum_f16_vec;
1584
- simsimd_fma_f16_sapphire_cycle:
1585
- if (n < 32) {
1586
- mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1587
- a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
1588
- b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
1589
- c_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c));
1590
- n = 0;
1591
- }
1592
- else {
1593
- a_f16_vec = _mm512_loadu_ph(a);
1594
- b_f16_vec = _mm512_loadu_ph(b);
1595
- c_f16_vec = _mm512_loadu_ph(c);
1596
- a += 32, b += 32, c += 32, n -= 32;
1597
- }
1598
- ab_f16_vec = _mm512_mul_ph(a_f16_vec, b_f16_vec);
1599
- ab_scaled_f16_vec = _mm512_mul_ph(ab_f16_vec, alpha_vec);
1600
- sum_f16_vec = _mm512_fmadd_ph(c_f16_vec, beta_vec, ab_scaled_f16_vec);
1601
- _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
1602
- result += 32;
1603
- if (n) goto simsimd_fma_f16_sapphire_cycle;
1604
- }
1605
-
1606
- SIMSIMD_PUBLIC void simsimd_sum_u8_sapphire(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1607
- simsimd_u8_t *result) {
1608
- __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1609
- __m512i a_u8_vec, b_u8_vec, sum_u8_vec;
1610
- simsimd_sum_u8_sapphire_cycle:
1611
- if (n < 64) {
1612
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1613
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1614
- b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1615
- n = 0;
1616
- }
1617
- else {
1618
- a_u8_vec = _mm512_loadu_epi8(a);
1619
- b_u8_vec = _mm512_loadu_epi8(b);
1620
- a += 64, b += 64, n -= 64;
1621
- }
1622
- sum_u8_vec = _mm512_adds_epu8(a_u8_vec, b_u8_vec);
1623
- _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1624
- result += 64;
1625
- if (n) goto simsimd_sum_u8_sapphire_cycle;
1626
- }
1627
-
1628
- SIMSIMD_PUBLIC void simsimd_scale_u8_sapphire(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1629
- simsimd_u8_t *result) {
1630
- __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1631
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1632
- __m512i a_u8_vec, b_u8_vec, sum_u8_vec;
1633
- __m512h a_f16_low_vec, a_f16_high_vec;
1634
- __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1635
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1636
- simsimd_scale_u8_sapphire_cycle:
1637
- if (n < 64) {
1638
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1639
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1640
- n = 0;
1641
- }
1642
- else {
1643
- a_u8_vec = _mm512_loadu_epi8(a);
1644
- a += 64, n -= 64;
1645
- }
1646
- // Upcast:
1647
- a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
1648
- a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
1649
- // Scale:
1650
- sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1651
- sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1652
- // Downcast:
1653
- sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1654
- sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1655
- sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
1656
- _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1657
- result += 64;
1658
- if (n) goto simsimd_scale_u8_sapphire_cycle;
1659
- }
1660
-
1661
- SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
1662
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
1663
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
1664
-
1665
- // There are are several special cases we may want to implement:
1666
- // 1. Simple addition, when both weights are equal to 1.0.
1667
- if (alpha == 1 && beta == 1) {
1668
- // In this case we can avoid expensive multiplications.
1669
- simsimd_sum_u8_sapphire(a, b, n, result);
1670
- return;
1671
- }
1672
- // 2. Just scaling, when one of the weights is equal to zero.
1673
- else if (alpha == 0 || beta == 0) {
1674
- // In this case we can avoid half of the load instructions.
1675
- if (beta == 0) { simsimd_scale_u8_sapphire(a, n, alpha, result); }
1676
- else { simsimd_scale_u8_sapphire(b, n, beta, result); }
1677
- return;
1678
- }
1679
-
1680
- // The general case.
1681
- __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1682
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1683
- __m512h beta_vec = _mm512_set1_ph_from_ps(beta);
1684
- __m512i a_u8_vec, b_u8_vec, sum_u8_vec;
1685
- __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1686
- __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1687
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1688
- simsimd_wsum_u8_sapphire_cycle:
1689
- if (n < 64) {
1690
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1691
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1692
- b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1693
- n = 0;
1694
- }
1695
- else {
1696
- a_u8_vec = _mm512_loadu_epi8(a);
1697
- b_u8_vec = _mm512_loadu_epi8(b);
1698
- a += 64, b += 64, n -= 64;
1699
- }
1700
- // Upcast:
1701
- a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
1702
- a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
1703
- b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8_vec, _mm512_setzero_si512()));
1704
- b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8_vec, _mm512_setzero_si512()));
1705
- // Scale:
1706
- a_scaled_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1707
- a_scaled_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1708
- // Add:
1709
- sum_f16_low_vec = _mm512_fmadd_ph(b_f16_low_vec, beta_vec, a_scaled_f16_low_vec);
1710
- sum_f16_high_vec = _mm512_fmadd_ph(b_f16_high_vec, beta_vec, a_scaled_f16_high_vec);
1711
- // Downcast:
1712
- sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1713
- sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1714
- sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
1715
- _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1716
- result += 64;
1717
- if (n) goto simsimd_wsum_u8_sapphire_cycle;
1718
- }
1719
-
1720
- SIMSIMD_PUBLIC void simsimd_sum_i8_sapphire(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1721
- simsimd_i8_t *result) {
1722
-
1723
- __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1724
- __m512i a_i8_vec, b_i8_vec, sum_i8_vec;
1725
- __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1726
- __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1727
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1728
-
1729
- simsimd_sum_i8_sapphire_cycle:
1730
- if (n < 64) {
1731
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1732
- a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1733
- b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
1734
- n = 0;
1735
- }
1736
- else {
1737
- a_i8_vec = _mm512_loadu_epi8(a);
1738
- b_i8_vec = _mm512_loadu_epi8(b);
1739
- a += 64, b += 64, n -= 64;
1740
- }
1741
- sum_i8_vec = _mm512_adds_epi8(a_i8_vec, b_i8_vec);
1742
- _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1743
- result += 64;
1744
- if (n) goto simsimd_sum_i8_sapphire_cycle;
1745
- }
1746
-
1747
- SIMSIMD_PUBLIC void simsimd_scale_i8_sapphire(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1748
- simsimd_i8_t *result) {
1749
-
1750
- __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1751
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1752
- __m512i a_i8_vec, sum_i8_vec;
1753
- __m512h a_f16_low_vec, a_f16_high_vec;
1754
- __m512h sum_f16_low_vec, sum_f16_high_vec;
1755
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1756
- simsimd_wsum_i8_sapphire_cycle:
1757
- if (n < 64) {
1758
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1759
- a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1760
- n = 0;
1761
- }
1762
- else {
1763
- a_i8_vec = _mm512_loadu_epi8(a);
1764
- a += 64, n -= 64;
1765
- }
1766
- // Upcast:
1767
- a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
1768
- a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
1769
- // Scale:
1770
- sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1771
- sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1772
- // Downcast:
1773
- sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1774
- sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1775
- sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
1776
- _mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
1777
- _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1778
- result += 64;
1779
- if (n) goto simsimd_wsum_i8_sapphire_cycle;
1780
- }
1781
-
1782
- SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
1783
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
1784
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1785
-
1786
- // There are are several special cases we may want to implement:
1787
- // 1. Simple addition, when both weights are equal to 1.0.
1788
- if (alpha == 1 && beta == 1) {
1789
- // In this case we can avoid expensive multiplications.
1790
- simsimd_sum_i8_sapphire(a, b, n, result);
1791
- return;
1792
- }
1793
- // 2. Just scaling, when one of the weights is equal to zero.
1794
- else if (alpha == 0 || beta == 0) {
1795
- // In this case we can avoid half of the load instructions.
1796
- if (beta == 0) { simsimd_scale_i8_sapphire(a, n, alpha, result); }
1797
- else { simsimd_scale_i8_sapphire(b, n, beta, result); }
1798
- return;
1799
- }
1800
-
1801
- // The general case.
1802
- __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1803
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1804
- __m512h beta_vec = _mm512_set1_ph_from_ps(beta);
1805
- __m512i a_i8_vec, b_i8_vec, sum_i8_vec;
1806
- __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1807
- __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1808
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1809
-
1810
- simsimd_wsum_i8_sapphire_cycle:
1811
- if (n < 64) {
1812
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1813
- a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1814
- b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
1815
- n = 0;
1816
- }
1817
- else {
1818
- a_i8_vec = _mm512_loadu_epi8(a);
1819
- b_i8_vec = _mm512_loadu_epi8(b);
1820
- a += 64, b += 64, n -= 64;
1821
- }
1822
- // Upcast:
1823
- a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
1824
- a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
1825
- b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1826
- b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1827
- // Scale:
1828
- a_scaled_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1829
- a_scaled_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1830
- // Add:
1831
- sum_f16_low_vec = _mm512_fmadd_ph(b_f16_low_vec, beta_vec, a_scaled_f16_low_vec);
1832
- sum_f16_high_vec = _mm512_fmadd_ph(b_f16_high_vec, beta_vec, a_scaled_f16_high_vec);
1833
- // Downcast:
1834
- sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1835
- sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1836
- sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
1837
- _mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
1838
- _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1839
- result += 64;
1840
- if (n) goto simsimd_wsum_i8_sapphire_cycle;
1841
- }
1842
-
1843
- SIMSIMD_PUBLIC void simsimd_fma_i8_sapphire( //
1844
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
1845
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1846
-
1847
- __mmask64 mask = 0xFFFFFFFFFFFFFFFF;
1848
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1849
- __m512h beta_vec = _mm512_set1_ph_from_ps(beta);
1850
- __m512i a_i8_vec, b_i8_vec, c_i8_vec, sum_i8_vec;
1851
- __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1852
- __m512h c_f16_low_vec, c_f16_high_vec, ab_f16_low_vec, ab_f16_high_vec;
1853
- __m512h ab_scaled_f16_low_vec, ab_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1854
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1855
-
1856
- simsimd_fma_i8_sapphire_cycle:
1857
- if (n < 64) {
1858
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1859
- a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1860
- b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
1861
- c_i8_vec = _mm512_maskz_loadu_epi8(mask, c);
1862
- n = 0;
1863
- }
1864
- else {
1865
- a_i8_vec = _mm512_loadu_epi8(a);
1866
- b_i8_vec = _mm512_loadu_epi8(b);
1867
- c_i8_vec = _mm512_loadu_epi8(c);
1868
- a += 64, b += 64, c += 64, n -= 64;
1869
- }
1870
- // Upcast:
1871
- a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
1872
- a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
1873
- b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1874
- b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1875
- c_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(c_i8_vec)));
1876
- c_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(c_i8_vec, 1)));
1877
- // Multiply:
1878
- ab_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, b_f16_low_vec);
1879
- ab_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, b_f16_high_vec);
1880
- // Scale:
1881
- ab_scaled_f16_low_vec = _mm512_mul_ph(ab_f16_low_vec, alpha_vec);
1882
- ab_scaled_f16_high_vec = _mm512_mul_ph(ab_f16_high_vec, alpha_vec);
1883
- // Add:
1884
- sum_f16_low_vec = _mm512_fmadd_ph(c_f16_low_vec, beta_vec, ab_scaled_f16_low_vec);
1885
- sum_f16_high_vec = _mm512_fmadd_ph(c_f16_high_vec, beta_vec, ab_scaled_f16_high_vec);
1886
- // Downcast:
1887
- sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1888
- sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1889
- sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
1890
- _mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
1891
- _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1892
- result += 64;
1893
- if (n) goto simsimd_fma_i8_sapphire_cycle;
1894
- }
1895
-
1896
- SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire( //
1897
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
1898
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
1899
-
1900
- __mmask64 mask = 0xFFFFFFFFFFFFFFFF;
1901
- __m512h alpha_vec = _mm512_set1_ph_from_ps(alpha);
1902
- __m512h beta_vec = _mm512_set1_ph_from_ps(beta);
1903
- __m512i a_u8_vec, b_u8_vec, c_u8_vec, sum_u8_vec;
1904
- __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1905
- __m512h c_f16_low_vec, c_f16_high_vec, ab_f16_low_vec, ab_f16_high_vec;
1906
- __m512h ab_scaled_f16_low_vec, ab_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1907
- __m512i sum_i16_low_vec, sum_i16_high_vec;
1908
-
1909
- simsimd_fma_u8_sapphire_cycle:
1910
- if (n < 64) {
1911
- mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1912
- a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1913
- b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1914
- c_u8_vec = _mm512_maskz_loadu_epi8(mask, c);
1915
- n = 0;
1916
- }
1917
- else {
1918
- a_u8_vec = _mm512_loadu_epi8(a);
1919
- b_u8_vec = _mm512_loadu_epi8(b);
1920
- c_u8_vec = _mm512_loadu_epi8(c);
1921
- a += 64, b += 64, c += 64, n -= 64;
1922
- }
1923
- // Upcast:
1924
- a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
1925
- a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
1926
- b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8_vec, _mm512_setzero_si512()));
1927
- b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8_vec, _mm512_setzero_si512()));
1928
- c_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(c_u8_vec, _mm512_setzero_si512()));
1929
- c_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(c_u8_vec, _mm512_setzero_si512()));
1930
- // Multiply:
1931
- ab_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, b_f16_low_vec);
1932
- ab_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, b_f16_high_vec);
1933
- // Scale:
1934
- ab_scaled_f16_low_vec = _mm512_mul_ph(ab_f16_low_vec, alpha_vec);
1935
- ab_scaled_f16_high_vec = _mm512_mul_ph(ab_f16_high_vec, alpha_vec);
1936
- // Add:
1937
- sum_f16_low_vec = _mm512_fmadd_ph(c_f16_low_vec, beta_vec, ab_scaled_f16_low_vec);
1938
- sum_f16_high_vec = _mm512_fmadd_ph(c_f16_high_vec, beta_vec, ab_scaled_f16_high_vec);
1939
- // Downcast:
1940
- sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1941
- sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1942
- sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
1943
- _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1944
- result += 64;
1945
- if (n) goto simsimd_fma_u8_sapphire_cycle;
1946
- }
1947
-
1948
- #pragma clang attribute pop
1949
- #pragma GCC pop_options
1950
- #endif // SIMSIMD_TARGET_SAPPHIRE
1951
- #endif // _SIMSIMD_TARGET_X86
1952
-
1953
- #if _SIMSIMD_TARGET_ARM
1954
- #if SIMSIMD_TARGET_NEON
1955
- #pragma GCC push_options
1956
- #pragma GCC target("arch=armv8.2-a+simd")
1957
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
1958
-
1959
- SIMSIMD_PUBLIC void simsimd_sum_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1960
- simsimd_f32_t *result) {
1961
- // The main loop:
1962
- simsimd_size_t i = 0;
1963
- for (; i + 4 <= n; i += 4) {
1964
- float32x4_t a_vec = vld1q_f32(a + i);
1965
- float32x4_t b_vec = vld1q_f32(b + i);
1966
- float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
1967
- vst1q_f32(result + i, sum_vec);
1968
- }
1969
-
1970
- // The tail:
1971
- for (; i < n; ++i) result[i] = a[i] + b[i];
1972
- }
1973
-
1974
- SIMSIMD_PUBLIC void simsimd_scale_f32_neon(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1975
- simsimd_f32_t *result) {
1976
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1977
-
1978
- // The main loop:
1979
- simsimd_size_t i = 0;
1980
- for (; i + 4 <= n; i += 4) {
1981
- float32x4_t a_vec = vld1q_f32(a + i);
1982
- float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
1983
- vst1q_f32(result + i, sum_vec);
1984
- }
1985
-
1986
- // The tail:
1987
- for (; i < n; ++i) result[i] = alpha_f32 * a[i];
1988
- }
1989
-
1990
- SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
1991
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
1992
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1993
-
1994
- // There are are several special cases we may want to implement:
1995
- // 1. Simple addition, when both weights are equal to 1.0.
1996
- if (alpha == 1 && beta == 1) {
1997
- // In this case we can avoid expensive multiplications.
1998
- simsimd_sum_f32_neon(a, b, n, result);
1999
- return;
2000
- }
2001
- // 2. Just scaling, when one of the weights is equal to zero.
2002
- else if (alpha == 0 || beta == 0) {
2003
- // In this case we can avoid half of the load instructions.
2004
- if (beta == 0) { simsimd_scale_f32_neon(a, n, alpha, result); }
2005
- else { simsimd_scale_f32_neon(b, n, beta, result); }
2006
- return;
2007
- }
2008
-
2009
- // The general case.
2010
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
2011
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
2012
-
2013
- // The main loop:
2014
- simsimd_size_t i = 0;
2015
- for (; i + 4 <= n; i += 4) {
2016
- float32x4_t a_vec = vld1q_f32(a + i);
2017
- float32x4_t b_vec = vld1q_f32(b + i);
2018
- float32x4_t a_scaled_vec = vmulq_n_f32(a_vec, alpha_f32);
2019
- float32x4_t b_scaled_vec = vmulq_n_f32(b_vec, beta_f32);
2020
- float32x4_t sum_vec = vaddq_f32(a_scaled_vec, b_scaled_vec);
2021
- vst1q_f32(result + i, sum_vec);
2022
- }
2023
-
2024
- // The tail:
2025
- for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
2026
- }
2027
-
2028
- SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
2029
- simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
2030
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
2031
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
2032
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
2033
-
2034
- // The main loop:
2035
- simsimd_size_t i = 0;
2036
- for (; i + 4 <= n; i += 4) {
2037
- float32x4_t a_vec = vld1q_f32(a + i);
2038
- float32x4_t b_vec = vld1q_f32(b + i);
2039
- float32x4_t c_vec = vld1q_f32(c + i);
2040
- float32x4_t ab_vec = vmulq_f32(a_vec, b_vec);
2041
- float32x4_t ab_scaled_vec = vmulq_n_f32(ab_vec, alpha_f32);
2042
- float32x4_t sum_vec = vfmaq_n_f32(ab_scaled_vec, c_vec, beta_f32);
2043
- vst1q_f32(result + i, sum_vec);
2044
- }
2045
-
2046
- // The tail:
2047
- for (; i < n; ++i) result[i] = alpha_f32 * a[i] * b[i] + beta_f32 * c[i];
2048
- }
2049
-
2050
- #pragma clang attribute pop
2051
- #pragma GCC pop_options
2052
- #endif // SIMSIMD_TARGET_NEON
2053
-
2054
- #if SIMSIMD_TARGET_NEON_BF16
2055
- #pragma GCC push_options
2056
- #pragma GCC target("arch=armv8.6-a+simd+bf16")
2057
- #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
2058
-
2059
- SIMSIMD_PUBLIC void simsimd_sum_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
2060
- simsimd_bf16_t *result) {
2061
- // The main loop:
2062
- simsimd_size_t i = 0;
2063
- for (; i + 4 <= n; i += 4) {
2064
- float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
2065
- float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
2066
- float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
2067
- vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
2068
- }
2069
-
2070
- // The tail:
2071
- for (; i < n; ++i) simsimd_f32_to_bf16(simsimd_bf16_to_f32(a + i) + simsimd_bf16_to_f32(b + i), result + i);
2072
- }
2073
-
2074
- SIMSIMD_PUBLIC void simsimd_scale_bf16_neon(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2075
- simsimd_bf16_t *result) {
2076
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
2077
-
2078
- // The main loop:
2079
- simsimd_size_t i = 0;
2080
- for (; i + 4 <= n; i += 4) {
2081
- float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
2082
- float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
2083
- vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
2084
- }
2085
-
2086
- // The tail:
2087
- for (; i < n; ++i) simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i), result + i);
2088
- }
2089
-
2090
- SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
2091
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
2092
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
2093
-
2094
- // There are are several special cases we may want to implement:
2095
- // 1. Simple addition, when both weights are equal to 1.0.
2096
- if (alpha == 1 && beta == 1) {
2097
- // In this case we can avoid expensive multiplications.
2098
- simsimd_sum_bf16_neon(a, b, n, result);
2099
- return;
2100
- }
2101
- // 2. Just scaling, when one of the weights is equal to zero.
2102
- else if (alpha == 0 || beta == 0) {
2103
- // In this case we can avoid half of the load instructions.
2104
- if (beta == 0) { simsimd_scale_bf16_neon(a, n, alpha, result); }
2105
- else { simsimd_scale_bf16_neon(b, n, beta, result); }
2106
- return;
2107
- }
2108
-
2109
- // The general case.
2110
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
2111
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
2112
-
2113
- // The main loop:
2114
- simsimd_size_t i = 0;
2115
- for (; i + 4 <= n; i += 4) {
2116
- float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
2117
- float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
2118
- float32x4_t a_scaled_vec = vmulq_n_f32(a_vec, alpha_f32);
2119
- float32x4_t b_scaled_vec = vmulq_n_f32(b_vec, beta_f32);
2120
- float32x4_t sum_vec = vaddq_f32(a_scaled_vec, b_scaled_vec);
2121
- vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
2122
- }
2123
-
2124
- // The tail:
2125
- for (; i < n; ++i)
2126
- simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i) + beta_f32 * simsimd_bf16_to_f32(b + i), result + i);
2127
- }
2128
-
2129
- SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( //
2130
- simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
2131
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
2132
- simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
2133
- simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
2134
-
2135
- // The main loop:
2136
- simsimd_size_t i = 0;
2137
- for (; i + 4 <= n; i += 4) {
2138
- float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
2139
- float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
2140
- float32x4_t c_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)c + i));
2141
- float32x4_t ab_vec = vmulq_f32(a_vec, b_vec);
2142
- float32x4_t ab_scaled_vec = vmulq_n_f32(ab_vec, alpha_f32);
2143
- float32x4_t sum_vec = vfmaq_n_f32(ab_scaled_vec, c_vec, beta_f32);
2144
- vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
2145
- }
2146
-
2147
- // The tail:
2148
- for (; i < n; ++i)
2149
- simsimd_f32_to_bf16(
2150
- alpha_f32 * simsimd_bf16_to_f32(a + i) * simsimd_bf16_to_f32(b + i) + beta_f32 * simsimd_bf16_to_f32(c + i),
2151
- result + i);
2152
- }
2153
-
2154
- #pragma clang attribute pop
2155
- #pragma GCC pop_options
2156
- #endif // SIMSIMD_TARGET_NEON_BF16
2157
-
2158
- #if SIMSIMD_TARGET_NEON_F16
2159
- #pragma GCC push_options
2160
- #pragma GCC target("arch=armv8.2-a+simd+fp16")
2161
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
2162
-
2163
- SIMSIMD_PUBLIC void simsimd_sum_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
2164
- simsimd_f16_t *result) {
2165
- // The main loop:
2166
- simsimd_size_t i = 0;
2167
- for (; i + 8 <= n; i += 8) {
2168
- float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
2169
- float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
2170
- float16x8_t sum_vec = vaddq_f16(a_vec, b_vec);
2171
- vst1q_f16((float16_t *)result + i, sum_vec);
2172
- }
2173
-
2174
- // The tail:
2175
- for (; i < n; ++i) ((float16_t *)result)[i] = ((float16_t const *)a)[i] + ((float16_t const *)b)[i];
2176
- }
2177
-
2178
- SIMSIMD_PUBLIC void simsimd_scale_f16_neon(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2179
- simsimd_f16_t *result) {
2180
- float16_t alpha_f16 = (float16_t)alpha;
2181
-
2182
- // The main loop:
2183
- simsimd_size_t i = 0;
2184
- for (; i + 8 <= n; i += 8) {
2185
- float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
2186
- float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
2187
- vst1q_f16((float16_t *)result + i, sum_vec);
2188
- }
2189
-
2190
- // The tail:
2191
- for (; i < n; ++i) ((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i];
2192
- }
2193
-
2194
- SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
2195
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
2196
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
2197
- // There are are several special cases we may want to implement:
2198
- // 1. Simple addition, when both weights are equal to 1.0.
2199
- if (alpha == 1 && beta == 1) {
2200
- // In this case we can avoid expensive multiplications.
2201
- simsimd_sum_f16_neon(a, b, n, result);
2202
- return;
2203
- }
2204
- // 2. Just scaling, when one of the weights is equal to zero.
2205
- else if (alpha == 0 || beta == 0) {
2206
- // In this case we can avoid half of the load instructions.
2207
- if (beta == 0) { simsimd_scale_f16_neon(a, n, alpha, result); }
2208
- else { simsimd_scale_f16_neon(b, n, beta, result); }
2209
- return;
2210
- }
2211
-
2212
- // The general case.
2213
- float16_t alpha_f16 = (float16_t)alpha;
2214
- float16_t beta_f16 = (float16_t)beta;
2215
-
2216
- // The main loop:
2217
- simsimd_size_t i = 0;
2218
- for (; i + 8 <= n; i += 8) {
2219
- float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
2220
- float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
2221
- float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
2222
- float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
2223
- float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
2224
- vst1q_f16((float16_t *)result + i, sum_vec);
2225
- }
2226
-
2227
- // The tail:
2228
- for (; i < n; ++i)
2229
- ((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i] + beta_f16 * ((float16_t const *)b)[i];
2230
- }
2231
-
2232
- SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
2233
- simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
2234
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
2235
- float16_t alpha_f16 = (float16_t)alpha;
2236
- float16_t beta_f16 = (float16_t)beta;
2237
-
2238
- // The main loop:
2239
- simsimd_size_t i = 0;
2240
- for (; i + 8 <= n; i += 8) {
2241
- float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
2242
- float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
2243
- float16x8_t c_vec = vld1q_f16((float16_t const *)c + i);
2244
- float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
2245
- float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
2246
- float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
2247
- vst1q_f16((float16_t *)result + i, sum_vec);
2248
- }
2249
-
2250
- // The tail:
2251
- for (; i < n; ++i)
2252
- ((float16_t *)result)[i] =
2253
- alpha_f16 * ((float16_t const *)a)[i] * ((float16_t const *)b)[i] + beta_f16 * ((float16_t const *)c)[i];
2254
- }
2255
-
2256
- SIMSIMD_PUBLIC void simsimd_sum_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
2257
- simsimd_u8_t *result) {
2258
- // The main loop:
2259
- simsimd_size_t i = 0;
2260
- for (; i + 16 <= n; i += 16) {
2261
- uint8x16_t a_vec = vld1q_u8(a + i);
2262
- uint8x16_t b_vec = vld1q_u8(b + i);
2263
- uint8x16_t sum_vec = vqaddq_u8(a_vec, b_vec);
2264
- vst1q_u8(result + i, sum_vec);
2265
- }
2266
-
2267
- // The tail:
2268
- for (; i < n; ++i) { SIMSIMD_F32_TO_U8(a[i] + b[i], result + i); }
2269
- }
2270
-
2271
- SIMSIMD_PUBLIC void simsimd_scale_u8_neon(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2272
- simsimd_u8_t *result) {
2273
- float16_t alpha_f16 = (float16_t)alpha;
2274
-
2275
- // The main loop:
2276
- simsimd_size_t i = 0;
2277
- for (; i + 8 <= n; i += 8) {
2278
- uint8x8_t a_u8_vec = vld1_u8(a + i);
2279
- float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
2280
- float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
2281
- uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
2282
- vst1_u8(result + i, sum_u8_vec);
2283
- }
2284
-
2285
- // The tail:
2286
- for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i], result + i); }
2287
- }
2288
-
2289
- SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
2290
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
2291
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
2292
-
2293
- // There are are several special cases we may want to implement:
2294
- // 1. Simple addition, when both weights are equal to 1.0.
2295
- if (alpha == 1 && beta == 1) {
2296
- // In this case we can avoid expensive multiplications.
2297
- simsimd_sum_u8_neon(a, b, n, result);
2298
- return;
2299
- }
2300
- // 2. Just scaling, when one of the weights is equal to zero.
2301
- else if (alpha == 0 || beta == 0) {
2302
- // In this case we can avoid half of the load instructions.
2303
- if (beta == 0) { simsimd_scale_u8_neon(a, n, alpha, result); }
2304
- else { simsimd_scale_u8_neon(b, n, beta, result); }
2305
- return;
2306
- }
2307
-
2308
- // The general case.
2309
- float16_t alpha_f16 = (float16_t)alpha;
2310
- float16_t beta_f16 = (float16_t)beta;
2311
-
2312
- // The main loop:
2313
- simsimd_size_t i = 0;
2314
- for (; i + 8 <= n; i += 8) {
2315
- uint8x8_t a_u8_vec = vld1_u8(a + i);
2316
- uint8x8_t b_u8_vec = vld1_u8(b + i);
2317
- float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
2318
- float16x8_t b_vec = vcvtq_f16_u16(vmovl_u8(b_u8_vec));
2319
- float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
2320
- float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
2321
- float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
2322
- uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
2323
- vst1_u8(result + i, sum_u8_vec);
2324
- }
2325
-
2326
- // The tail:
2327
- for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] + beta_f16 * b[i], result + i); }
2328
- }
2329
-
2330
- SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
2331
- simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, //
2332
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
2333
- float16_t alpha_f16 = (float16_t)alpha;
2334
- float16_t beta_f16 = (float16_t)beta;
2335
-
2336
- // The main loop:
2337
- simsimd_size_t i = 0;
2338
- for (; i + 8 <= n; i += 8) {
2339
- uint8x8_t a_u8_vec = vld1_u8(a + i);
2340
- uint8x8_t b_u8_vec = vld1_u8(b + i);
2341
- uint8x8_t c_u8_vec = vld1_u8(c + i);
2342
- float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
2343
- float16x8_t b_vec = vcvtq_f16_u16(vmovl_u8(b_u8_vec));
2344
- float16x8_t c_vec = vcvtq_f16_u16(vmovl_u8(c_u8_vec));
2345
- float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
2346
- float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
2347
- float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
2348
- uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
2349
- vst1_u8(result + i, sum_u8_vec);
2350
- }
2351
-
2352
- // The tail:
2353
- for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
2354
- }
2355
-
2356
- SIMSIMD_PUBLIC void simsimd_sum_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
2357
- simsimd_i8_t *result) {
2358
- // The main loop:
2359
- simsimd_size_t i = 0;
2360
- for (; i + 16 <= n; i += 16) {
2361
- int8x16_t a_vec = vld1q_s8(a + i);
2362
- int8x16_t b_vec = vld1q_s8(b + i);
2363
- int8x16_t sum_vec = vqaddq_s8(a_vec, b_vec);
2364
- vst1q_s8(result + i, sum_vec);
2365
- }
2366
-
2367
- // The tail:
2368
- for (; i < n; ++i) { SIMSIMD_F32_TO_I8(a[i] + b[i], result + i); }
2369
- }
2370
-
2371
- SIMSIMD_PUBLIC void simsimd_scale_i8_neon(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2372
- simsimd_i8_t *result) {
2373
- float16_t alpha_f16 = (float16_t)alpha;
2374
-
2375
- // The main loop:
2376
- simsimd_size_t i = 0;
2377
- for (; i + 8 <= n; i += 8) {
2378
- int8x8_t a_i8_vec = vld1_s8(a + i);
2379
- float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
2380
- float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
2381
- int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
2382
- vst1_s8(result + i, sum_i8_vec);
2383
- }
2384
-
2385
- // The tail:
2386
- for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i], result + i); }
2387
- }
2388
-
2389
- SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
2390
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
2391
- simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
2392
-
2393
- // There are are several special cases we may want to implement:
2394
- // 1. Simple addition, when both weights are equal to 1.0.
2395
- if (alpha == 1 && beta == 1) {
2396
- // In this case we can avoid expensive multiplications.
2397
- simsimd_sum_i8_neon(a, b, n, result);
2398
- return;
2399
- }
2400
- // 2. Just scaling, when one of the weights is equal to zero.
2401
- else if (alpha == 0 || beta == 0) {
2402
- // In this case we can avoid half of the load instructions.
2403
- if (beta == 0) { simsimd_scale_i8_neon(a, n, alpha, result); }
2404
- else { simsimd_scale_i8_neon(b, n, beta, result); }
2405
- return;
2406
- }
2407
-
2408
- // The general case.
2409
- float16_t alpha_f16 = (float16_t)alpha;
2410
- float16_t beta_f16 = (float16_t)beta;
2411
-
2412
- // The main loop:
2413
- simsimd_size_t i = 0;
2414
- for (; i + 8 <= n; i += 8) {
2415
- int8x8_t a_i8_vec = vld1_s8(a + i);
2416
- int8x8_t b_i8_vec = vld1_s8(b + i);
2417
- float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
2418
- float16x8_t b_vec = vcvtq_f16_s16(vmovl_s8(b_i8_vec));
2419
- float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
2420
- float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
2421
- float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
2422
- int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
2423
- vst1_s8(result + i, sum_i8_vec);
2424
- }
2425
-
2426
- // The tail:
2427
- for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i] + beta_f16 * b[i], result + i); }
2428
- }
2429
-
2430
- SIMSIMD_PUBLIC void simsimd_fma_i8_neon( //
2431
- simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, //
2432
- simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
2433
- float16_t alpha_f16 = (float16_t)alpha;
2434
- float16_t beta_f16 = (float16_t)beta;
2435
-
2436
- // The main loop:
2437
- simsimd_size_t i = 0;
2438
- for (; i + 8 <= n; i += 8) {
2439
- int8x8_t a_i8_vec = vld1_s8(a + i);
2440
- int8x8_t b_i8_vec = vld1_s8(b + i);
2441
- int8x8_t c_i8_vec = vld1_s8(c + i);
2442
- float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
2443
- float16x8_t b_vec = vcvtq_f16_s16(vmovl_s8(b_i8_vec));
2444
- float16x8_t c_vec = vcvtq_f16_s16(vmovl_s8(c_i8_vec));
2445
- float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
2446
- float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
2447
- float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
2448
- int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
2449
- vst1_s8(result + i, sum_i8_vec);
2450
- }
2451
-
2452
- // The tail:
2453
- for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
2454
- }
2455
-
2456
- #pragma clang attribute pop
2457
- #pragma GCC pop_options
2458
- #endif // SIMSIMD_TARGET_NEON_F16
2459
- #endif // _SIMSIMD_TARGET_ARM
2460
-
2461
- #ifdef __cplusplus
2462
- }
2463
- #endif
2464
-
2465
- #endif