usearch 2.21.4 → 2.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (604) hide show
  1. package/README.md +65 -69
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +8 -8
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +124 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +224 -48
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/README.md +1 -1
  16. package/javascript/dist/cjs/usearch.d.ts +8 -3
  17. package/javascript/dist/cjs/usearch.js +12 -4
  18. package/javascript/dist/esm/usearch.d.ts +8 -3
  19. package/javascript/dist/esm/usearch.js +12 -4
  20. package/javascript/lib.cpp +17 -0
  21. package/javascript/usearch.test.js +117 -94
  22. package/javascript/usearch.ts +14 -6
  23. package/numkong/.clang-tidy +30 -0
  24. package/{simsimd → numkong}/.cmake-format.py +18 -0
  25. package/numkong/.editorconfig +12 -0
  26. package/numkong/.git-blame-ignore-revs +123 -0
  27. package/numkong/.mailmap +15 -0
  28. package/numkong/.spi.yml +11 -0
  29. package/numkong/.swift-format +14 -0
  30. package/numkong/CITATION.cff +81 -0
  31. package/numkong/CONTRIBUTING.md +500 -0
  32. package/numkong/Cargo.toml +54 -0
  33. package/numkong/MANIFEST.in +8 -0
  34. package/numkong/Package.swift +84 -0
  35. package/numkong/README.md +569 -0
  36. package/numkong/bench/README.md +217 -0
  37. package/numkong/bench/bench-browser.html +323 -0
  38. package/numkong/bench/bench.cpp +344 -0
  39. package/numkong/bench/bench.hpp +816 -0
  40. package/numkong/bench/bench.js +148 -0
  41. package/numkong/bench/bench.mjs +616 -0
  42. package/numkong/bench/bench_attention.cpp +113 -0
  43. package/numkong/bench/bench_cast.cpp +264 -0
  44. package/numkong/bench/bench_cross_amx.cpp +62 -0
  45. package/numkong/bench/bench_cross_arm.cpp +245 -0
  46. package/numkong/bench/bench_cross_blas.cpp +309 -0
  47. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  48. package/numkong/bench/bench_cross_power.cpp +84 -0
  49. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  50. package/numkong/bench/bench_cross_serial.cpp +151 -0
  51. package/numkong/bench/bench_cross_sme.cpp +163 -0
  52. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  53. package/numkong/bench/bench_cross_x86.cpp +455 -0
  54. package/numkong/bench/bench_curved.cpp +210 -0
  55. package/numkong/bench/bench_dot.cpp +315 -0
  56. package/numkong/bench/bench_each.cpp +503 -0
  57. package/numkong/bench/bench_geospatial.cpp +112 -0
  58. package/numkong/bench/bench_maxsim.cpp +154 -0
  59. package/numkong/bench/bench_mesh.cpp +148 -0
  60. package/numkong/bench/bench_probability.cpp +60 -0
  61. package/numkong/bench/bench_reduce.cpp +371 -0
  62. package/numkong/bench/bench_set.cpp +73 -0
  63. package/numkong/bench/bench_sparse.cpp +198 -0
  64. package/numkong/bench/bench_spatial.cpp +347 -0
  65. package/numkong/bench/bench_trigonometry.cpp +137 -0
  66. package/numkong/bench/swift/Bench.swift +506 -0
  67. package/numkong/binding.gyp +169 -0
  68. package/numkong/build.rs +484 -0
  69. package/numkong/c/dispatch.h +512 -0
  70. package/numkong/c/dispatch_bf16.c +447 -0
  71. package/numkong/c/dispatch_bf16c.c +52 -0
  72. package/numkong/c/dispatch_e2m3.c +296 -0
  73. package/numkong/c/dispatch_e3m2.c +284 -0
  74. package/numkong/c/dispatch_e4m3.c +318 -0
  75. package/numkong/c/dispatch_e5m2.c +316 -0
  76. package/numkong/c/dispatch_f16.c +452 -0
  77. package/numkong/c/dispatch_f16c.c +58 -0
  78. package/numkong/c/dispatch_f32.c +434 -0
  79. package/numkong/c/dispatch_f32c.c +99 -0
  80. package/numkong/c/dispatch_f64.c +348 -0
  81. package/numkong/c/dispatch_f64c.c +98 -0
  82. package/numkong/c/dispatch_i16.c +96 -0
  83. package/numkong/c/dispatch_i32.c +89 -0
  84. package/numkong/c/dispatch_i4.c +153 -0
  85. package/numkong/c/dispatch_i64.c +86 -0
  86. package/numkong/c/dispatch_i8.c +348 -0
  87. package/numkong/c/dispatch_other.c +348 -0
  88. package/numkong/c/dispatch_u1.c +193 -0
  89. package/numkong/c/dispatch_u16.c +124 -0
  90. package/numkong/c/dispatch_u32.c +118 -0
  91. package/numkong/c/dispatch_u4.c +153 -0
  92. package/numkong/c/dispatch_u64.c +102 -0
  93. package/numkong/c/dispatch_u8.c +364 -0
  94. package/numkong/c/numkong.c +940 -0
  95. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  96. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  97. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_math_lib.cmake +46 -0
  99. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  100. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  101. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  102. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  103. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  104. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  105. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  106. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  107. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  108. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  109. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  110. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  111. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  112. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  113. package/numkong/include/README.md +643 -0
  114. package/numkong/include/module.modulemap +129 -0
  115. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  116. package/numkong/include/numkong/attention/sme.h +2075 -0
  117. package/numkong/include/numkong/attention.h +49 -0
  118. package/numkong/include/numkong/capabilities.h +930 -0
  119. package/numkong/include/numkong/cast/README.md +262 -0
  120. package/numkong/include/numkong/cast/diamond.h +64 -0
  121. package/numkong/include/numkong/cast/haswell.h +823 -0
  122. package/numkong/include/numkong/cast/icelake.h +475 -0
  123. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  124. package/numkong/include/numkong/cast/neon.h +1174 -0
  125. package/numkong/include/numkong/cast/powervsx.h +449 -0
  126. package/numkong/include/numkong/cast/rvv.h +970 -0
  127. package/numkong/include/numkong/cast/sapphire.h +262 -0
  128. package/numkong/include/numkong/cast/serial.h +2336 -0
  129. package/numkong/include/numkong/cast/skylake.h +915 -0
  130. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  131. package/numkong/include/numkong/cast.h +254 -0
  132. package/numkong/include/numkong/cast.hpp +44 -0
  133. package/numkong/include/numkong/curved/README.md +223 -0
  134. package/numkong/include/numkong/curved/genoa.h +182 -0
  135. package/numkong/include/numkong/curved/haswell.h +276 -0
  136. package/numkong/include/numkong/curved/neon.h +329 -0
  137. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  138. package/numkong/include/numkong/curved/rvv.h +305 -0
  139. package/numkong/include/numkong/curved/serial.h +207 -0
  140. package/numkong/include/numkong/curved/skylake.h +457 -0
  141. package/numkong/include/numkong/curved/smef64.h +530 -0
  142. package/numkong/include/numkong/curved.h +513 -0
  143. package/numkong/include/numkong/curved.hpp +144 -0
  144. package/numkong/include/numkong/dot/README.md +442 -0
  145. package/numkong/include/numkong/dot/alder.h +563 -0
  146. package/numkong/include/numkong/dot/diamond.h +158 -0
  147. package/numkong/include/numkong/dot/genoa.h +289 -0
  148. package/numkong/include/numkong/dot/haswell.h +1723 -0
  149. package/numkong/include/numkong/dot/icelake.h +997 -0
  150. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  151. package/numkong/include/numkong/dot/neon.h +869 -0
  152. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  153. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  154. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  155. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  156. package/numkong/include/numkong/dot/powervsx.h +752 -0
  157. package/numkong/include/numkong/dot/rvv.h +722 -0
  158. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  159. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  160. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  161. package/numkong/include/numkong/dot/sapphire.h +141 -0
  162. package/numkong/include/numkong/dot/serial.h +874 -0
  163. package/numkong/include/numkong/dot/sierra.h +405 -0
  164. package/numkong/include/numkong/dot/skylake.h +1116 -0
  165. package/numkong/include/numkong/dot/sve.h +420 -0
  166. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  167. package/numkong/include/numkong/dot/svehalf.h +168 -0
  168. package/numkong/include/numkong/dot/svesdot.h +90 -0
  169. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  170. package/numkong/include/numkong/dot.h +1136 -0
  171. package/numkong/include/numkong/dot.hpp +94 -0
  172. package/numkong/include/numkong/dots/README.md +496 -0
  173. package/numkong/include/numkong/dots/alder.h +117 -0
  174. package/numkong/include/numkong/dots/diamond.h +86 -0
  175. package/numkong/include/numkong/dots/genoa.h +100 -0
  176. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  177. package/numkong/include/numkong/dots/haswell.h +310 -0
  178. package/numkong/include/numkong/dots/icelake.h +180 -0
  179. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  180. package/numkong/include/numkong/dots/neon.h +123 -0
  181. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  182. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  183. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  184. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  185. package/numkong/include/numkong/dots/powervsx.h +194 -0
  186. package/numkong/include/numkong/dots/rvv.h +2593 -0
  187. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  188. package/numkong/include/numkong/dots/serial.h +2867 -0
  189. package/numkong/include/numkong/dots/sierra.h +100 -0
  190. package/numkong/include/numkong/dots/skylake.h +207 -0
  191. package/numkong/include/numkong/dots/sme.h +5041 -0
  192. package/numkong/include/numkong/dots/smebi32.h +477 -0
  193. package/numkong/include/numkong/dots/smef64.h +1328 -0
  194. package/numkong/include/numkong/dots/smehalf.h +47 -0
  195. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  196. package/numkong/include/numkong/dots.h +3110 -0
  197. package/numkong/include/numkong/dots.hpp +641 -0
  198. package/numkong/include/numkong/each/README.md +465 -0
  199. package/numkong/include/numkong/each/haswell.h +1658 -0
  200. package/numkong/include/numkong/each/icelake.h +272 -0
  201. package/numkong/include/numkong/each/neon.h +1138 -0
  202. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  203. package/numkong/include/numkong/each/neonhalf.h +318 -0
  204. package/numkong/include/numkong/each/rvv.h +1121 -0
  205. package/numkong/include/numkong/each/sapphire.h +343 -0
  206. package/numkong/include/numkong/each/serial.h +282 -0
  207. package/numkong/include/numkong/each/skylake.h +1562 -0
  208. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  209. package/numkong/include/numkong/each.h +2232 -0
  210. package/numkong/include/numkong/each.hpp +434 -0
  211. package/numkong/include/numkong/geospatial/README.md +147 -0
  212. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  213. package/numkong/include/numkong/geospatial/neon.h +615 -0
  214. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  215. package/numkong/include/numkong/geospatial/serial.h +309 -0
  216. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  217. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  218. package/numkong/include/numkong/geospatial.h +453 -0
  219. package/numkong/include/numkong/geospatial.hpp +235 -0
  220. package/numkong/include/numkong/matrix.hpp +336 -0
  221. package/numkong/include/numkong/maxsim/README.md +187 -0
  222. package/numkong/include/numkong/maxsim/alder.h +511 -0
  223. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  224. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  225. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  226. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  227. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  228. package/numkong/include/numkong/maxsim/serial.h +490 -0
  229. package/numkong/include/numkong/maxsim/sme.h +993 -0
  230. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  231. package/numkong/include/numkong/maxsim.h +571 -0
  232. package/numkong/include/numkong/maxsim.hpp +133 -0
  233. package/numkong/include/numkong/mesh/README.md +219 -0
  234. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  235. package/numkong/include/numkong/mesh/neon.h +1821 -0
  236. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  237. package/numkong/include/numkong/mesh/rvv.h +936 -0
  238. package/numkong/include/numkong/mesh/serial.h +721 -0
  239. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  240. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  241. package/numkong/include/numkong/mesh.h +666 -0
  242. package/numkong/include/numkong/mesh.hpp +701 -0
  243. package/numkong/include/numkong/numkong.h +78 -0
  244. package/numkong/include/numkong/numkong.hpp +58 -0
  245. package/numkong/include/numkong/probability/README.md +177 -0
  246. package/numkong/include/numkong/probability/haswell.h +267 -0
  247. package/numkong/include/numkong/probability/neon.h +255 -0
  248. package/numkong/include/numkong/probability/rvv.h +414 -0
  249. package/numkong/include/numkong/probability/serial.h +181 -0
  250. package/numkong/include/numkong/probability/skylake.h +324 -0
  251. package/numkong/include/numkong/probability.h +380 -0
  252. package/numkong/include/numkong/probability.hpp +120 -0
  253. package/numkong/include/numkong/random.h +50 -0
  254. package/numkong/include/numkong/random.hpp +285 -0
  255. package/numkong/include/numkong/reduce/README.md +552 -0
  256. package/numkong/include/numkong/reduce/alder.h +636 -0
  257. package/numkong/include/numkong/reduce/genoa.h +201 -0
  258. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  259. package/numkong/include/numkong/reduce/icelake.h +549 -0
  260. package/numkong/include/numkong/reduce/neon.h +3980 -0
  261. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  262. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  263. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  264. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  265. package/numkong/include/numkong/reduce/serial.h +753 -0
  266. package/numkong/include/numkong/reduce/sierra.h +338 -0
  267. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  268. package/numkong/include/numkong/reduce/sve.h +52 -0
  269. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  270. package/numkong/include/numkong/reduce.h +1582 -0
  271. package/numkong/include/numkong/reduce.hpp +778 -0
  272. package/numkong/include/numkong/scalar/README.md +89 -0
  273. package/numkong/include/numkong/scalar/haswell.h +122 -0
  274. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  275. package/numkong/include/numkong/scalar/neon.h +122 -0
  276. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  277. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  278. package/numkong/include/numkong/scalar/rvv.h +211 -0
  279. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  280. package/numkong/include/numkong/scalar/serial.h +332 -0
  281. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  282. package/numkong/include/numkong/scalar.h +696 -0
  283. package/numkong/include/numkong/set/README.md +179 -0
  284. package/numkong/include/numkong/set/haswell.h +334 -0
  285. package/numkong/include/numkong/set/icelake.h +485 -0
  286. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  287. package/numkong/include/numkong/set/neon.h +363 -0
  288. package/numkong/include/numkong/set/powervsx.h +326 -0
  289. package/numkong/include/numkong/set/rvv.h +226 -0
  290. package/numkong/include/numkong/set/rvvbb.h +117 -0
  291. package/numkong/include/numkong/set/serial.h +174 -0
  292. package/numkong/include/numkong/set/sve.h +187 -0
  293. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  294. package/numkong/include/numkong/set.h +471 -0
  295. package/numkong/include/numkong/set.hpp +114 -0
  296. package/numkong/include/numkong/sets/README.md +149 -0
  297. package/numkong/include/numkong/sets/haswell.h +63 -0
  298. package/numkong/include/numkong/sets/icelake.h +66 -0
  299. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  300. package/numkong/include/numkong/sets/neon.h +61 -0
  301. package/numkong/include/numkong/sets/powervsx.h +65 -0
  302. package/numkong/include/numkong/sets/serial.h +43 -0
  303. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  304. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  305. package/numkong/include/numkong/sets.h +382 -0
  306. package/numkong/include/numkong/sparse/README.md +156 -0
  307. package/numkong/include/numkong/sparse/icelake.h +463 -0
  308. package/numkong/include/numkong/sparse/neon.h +288 -0
  309. package/numkong/include/numkong/sparse/serial.h +117 -0
  310. package/numkong/include/numkong/sparse/sve2.h +504 -0
  311. package/numkong/include/numkong/sparse/turin.h +322 -0
  312. package/numkong/include/numkong/sparse.h +363 -0
  313. package/numkong/include/numkong/sparse.hpp +113 -0
  314. package/numkong/include/numkong/spatial/README.md +439 -0
  315. package/numkong/include/numkong/spatial/alder.h +604 -0
  316. package/numkong/include/numkong/spatial/diamond.h +240 -0
  317. package/numkong/include/numkong/spatial/genoa.h +222 -0
  318. package/numkong/include/numkong/spatial/haswell.h +979 -0
  319. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  320. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  321. package/numkong/include/numkong/spatial/neon.h +846 -0
  322. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  323. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  324. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  325. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  326. package/numkong/include/numkong/spatial/rvv.h +991 -0
  327. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  328. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  329. package/numkong/include/numkong/spatial/serial.h +347 -0
  330. package/numkong/include/numkong/spatial/sierra.h +516 -0
  331. package/numkong/include/numkong/spatial/skylake.h +605 -0
  332. package/numkong/include/numkong/spatial/sve.h +246 -0
  333. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  334. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  335. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  336. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  337. package/numkong/include/numkong/spatial.h +1681 -0
  338. package/numkong/include/numkong/spatial.hpp +183 -0
  339. package/numkong/include/numkong/spatials/README.md +580 -0
  340. package/numkong/include/numkong/spatials/alder.h +94 -0
  341. package/numkong/include/numkong/spatials/diamond.h +82 -0
  342. package/numkong/include/numkong/spatials/genoa.h +94 -0
  343. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  344. package/numkong/include/numkong/spatials/haswell.h +219 -0
  345. package/numkong/include/numkong/spatials/icelake.h +113 -0
  346. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  347. package/numkong/include/numkong/spatials/neon.h +109 -0
  348. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  349. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  350. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  351. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  352. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  353. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  354. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  355. package/numkong/include/numkong/spatials/serial.h +248 -0
  356. package/numkong/include/numkong/spatials/sierra.h +96 -0
  357. package/numkong/include/numkong/spatials/skylake.h +184 -0
  358. package/numkong/include/numkong/spatials/sme.h +1927 -0
  359. package/numkong/include/numkong/spatials/smef64.h +483 -0
  360. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  361. package/numkong/include/numkong/spatials.h +3374 -0
  362. package/numkong/include/numkong/spatials.hpp +509 -0
  363. package/numkong/include/numkong/tensor.hpp +1981 -0
  364. package/numkong/include/numkong/trigonometry/README.md +184 -0
  365. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  366. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  367. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  368. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  369. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  370. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  371. package/numkong/include/numkong/trigonometry.h +467 -0
  372. package/numkong/include/numkong/trigonometry.hpp +166 -0
  373. package/numkong/include/numkong/types.h +1695 -0
  374. package/numkong/include/numkong/types.hpp +6027 -0
  375. package/numkong/include/numkong/vector.hpp +766 -0
  376. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  377. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  378. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  380. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  381. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  382. package/numkong/javascript/README.md +254 -0
  383. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  384. package/numkong/javascript/numkong-browser.ts +40 -0
  385. package/numkong/javascript/numkong-wasm.ts +750 -0
  386. package/numkong/javascript/numkong.c +712 -0
  387. package/numkong/javascript/numkong.ts +613 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  389. package/numkong/javascript/tsconfig-cjs.json +13 -0
  390. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  391. package/numkong/javascript/types.ts +784 -0
  392. package/numkong/numkong.gypi +104 -0
  393. package/numkong/package.json +108 -0
  394. package/numkong/probes/arm_neon.c +8 -0
  395. package/numkong/probes/arm_neon_bfdot.c +9 -0
  396. package/numkong/probes/arm_neon_fhm.c +9 -0
  397. package/numkong/probes/arm_neon_half.c +8 -0
  398. package/numkong/probes/arm_neon_sdot.c +9 -0
  399. package/numkong/probes/arm_neonfp8.c +9 -0
  400. package/numkong/probes/arm_sme.c +16 -0
  401. package/numkong/probes/arm_sme2.c +16 -0
  402. package/numkong/probes/arm_sme2p1.c +16 -0
  403. package/numkong/probes/arm_sme_bf16.c +16 -0
  404. package/numkong/probes/arm_sme_bi32.c +16 -0
  405. package/numkong/probes/arm_sme_f64.c +16 -0
  406. package/numkong/probes/arm_sme_fa64.c +14 -0
  407. package/numkong/probes/arm_sme_half.c +16 -0
  408. package/numkong/probes/arm_sme_lut2.c +15 -0
  409. package/numkong/probes/arm_sve.c +18 -0
  410. package/numkong/probes/arm_sve2.c +20 -0
  411. package/numkong/probes/arm_sve2p1.c +18 -0
  412. package/numkong/probes/arm_sve_bfdot.c +20 -0
  413. package/numkong/probes/arm_sve_half.c +18 -0
  414. package/numkong/probes/arm_sve_sdot.c +21 -0
  415. package/numkong/probes/loongarch_lasx.c +12 -0
  416. package/numkong/probes/power_vsx.c +12 -0
  417. package/numkong/probes/probe.js +127 -0
  418. package/numkong/probes/riscv_rvv.c +14 -0
  419. package/numkong/probes/riscv_rvv_bb.c +15 -0
  420. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  421. package/numkong/probes/riscv_rvv_half.c +14 -0
  422. package/numkong/probes/wasm_v128relaxed.c +11 -0
  423. package/numkong/probes/x86_alder.c +17 -0
  424. package/numkong/probes/x86_diamond.c +17 -0
  425. package/numkong/probes/x86_genoa.c +17 -0
  426. package/numkong/probes/x86_graniteamx.c +19 -0
  427. package/numkong/probes/x86_haswell.c +11 -0
  428. package/numkong/probes/x86_icelake.c +17 -0
  429. package/numkong/probes/x86_sapphire.c +16 -0
  430. package/numkong/probes/x86_sapphireamx.c +18 -0
  431. package/numkong/probes/x86_sierra.c +17 -0
  432. package/numkong/probes/x86_skylake.c +15 -0
  433. package/numkong/probes/x86_turin.c +17 -0
  434. package/numkong/rust/README.md +698 -0
  435. package/numkong/rust/capabilities.rs +96 -0
  436. package/numkong/rust/cast.rs +314 -0
  437. package/numkong/rust/curved.rs +391 -0
  438. package/numkong/rust/each.rs +2721 -0
  439. package/numkong/rust/geospatial.rs +343 -0
  440. package/numkong/rust/matrix.rs +5424 -0
  441. package/numkong/rust/maxsim.rs +540 -0
  442. package/numkong/rust/mesh.rs +837 -0
  443. package/numkong/rust/numkong.rs +409 -0
  444. package/numkong/rust/probability.rs +272 -0
  445. package/numkong/rust/reduce.rs +1249 -0
  446. package/numkong/rust/set.rs +194 -0
  447. package/numkong/rust/sparse.rs +513 -0
  448. package/numkong/rust/spatial.rs +1815 -0
  449. package/numkong/rust/tensor.rs +7565 -0
  450. package/numkong/rust/types.rs +3666 -0
  451. package/numkong/rust/vector.rs +1899 -0
  452. package/numkong/rustfmt.toml +2 -0
  453. package/numkong/swift/Matrix.swift +1388 -0
  454. package/numkong/swift/MaxSim.swift +151 -0
  455. package/numkong/swift/NumKong.swift +283 -0
  456. package/numkong/swift/Numerics.swift +329 -0
  457. package/numkong/swift/README.md +433 -0
  458. package/numkong/swift/Scalar.swift +741 -0
  459. package/numkong/swift/Tensor.swift +209 -0
  460. package/numkong/test/README.md +315 -0
  461. package/numkong/test/playwright.config.ts +16 -0
  462. package/numkong/test/swift/Test.swift +534 -0
  463. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  464. package/numkong/test/test-browser-bundle.html +188 -0
  465. package/numkong/test/test-browser-runner.mjs +43 -0
  466. package/numkong/test/test-browser.html +305 -0
  467. package/numkong/test/test-dtypes.mjs +453 -0
  468. package/numkong/test/test-wasm.mjs +267 -0
  469. package/numkong/test/test.cpp +442 -0
  470. package/numkong/test/test.hpp +580 -0
  471. package/numkong/test/test.mjs +363 -0
  472. package/numkong/test/test_base.py +1041 -0
  473. package/numkong/test/test_cast.cpp +242 -0
  474. package/numkong/test/test_cross.hpp +551 -0
  475. package/numkong/test/test_cross.py +467 -0
  476. package/numkong/test/test_cross_amx.cpp +50 -0
  477. package/numkong/test/test_cross_arm.cpp +225 -0
  478. package/numkong/test/test_cross_blas.cpp +354 -0
  479. package/numkong/test/test_cross_loongarch.cpp +90 -0
  480. package/numkong/test/test_cross_power.cpp +73 -0
  481. package/numkong/test/test_cross_rvv.cpp +109 -0
  482. package/numkong/test/test_cross_serial.cpp +181 -0
  483. package/numkong/test/test_cross_sme.cpp +148 -0
  484. package/numkong/test/test_cross_wasm.cpp +141 -0
  485. package/numkong/test/test_cross_x86.cpp +442 -0
  486. package/numkong/test/test_curved.cpp +182 -0
  487. package/numkong/test/test_curved.py +207 -0
  488. package/numkong/test/test_dot.cpp +353 -0
  489. package/numkong/test/test_dot.py +296 -0
  490. package/numkong/test/test_each.cpp +437 -0
  491. package/numkong/test/test_each.py +756 -0
  492. package/numkong/test/test_geospatial.cpp +155 -0
  493. package/numkong/test/test_geospatial.py +241 -0
  494. package/numkong/test/test_maxsim.cpp +138 -0
  495. package/numkong/test/test_maxsim.py +209 -0
  496. package/numkong/test/test_mesh.cpp +203 -0
  497. package/numkong/test/test_mesh.py +394 -0
  498. package/numkong/test/test_probability.cpp +136 -0
  499. package/numkong/test/test_probability.py +166 -0
  500. package/numkong/test/test_reduce.cpp +563 -0
  501. package/numkong/test/test_reduce.py +781 -0
  502. package/numkong/test/test_set.cpp +151 -0
  503. package/numkong/test/test_set.py +120 -0
  504. package/numkong/test/test_similarities.py +579 -0
  505. package/numkong/test/test_sparse.cpp +121 -0
  506. package/numkong/test/test_sparse.py +118 -0
  507. package/numkong/test/test_spatial.cpp +518 -0
  508. package/numkong/test/test_spatial.py +269 -0
  509. package/numkong/test/test_tensor.cpp +884 -0
  510. package/numkong/test/test_tensor.py +1417 -0
  511. package/numkong/test/test_trigonometry.cpp +156 -0
  512. package/numkong/test/test_trigonometry.py +180 -0
  513. package/package.json +7 -8
  514. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  515. package/prebuilds/linux-arm64/usearch.node +0 -0
  516. package/prebuilds/linux-x64/usearch.node +0 -0
  517. package/fp16/CMakeLists.txt +0 -176
  518. package/fp16/LICENSE +0 -11
  519. package/fp16/README.md +0 -20
  520. package/fp16/bench/alt-element.cc +0 -89
  521. package/fp16/bench/from-alt-array.cc +0 -175
  522. package/fp16/bench/from-ieee-array.cc +0 -399
  523. package/fp16/bench/ieee-element.cc +0 -272
  524. package/fp16/bench/to-alt-array.cc +0 -91
  525. package/fp16/bench/to-ieee-array.cc +0 -303
  526. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  527. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  528. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  529. package/fp16/configure.py +0 -65
  530. package/fp16/confu.yaml +0 -8
  531. package/fp16/include/fp16/__init__.py +0 -0
  532. package/fp16/include/fp16/avx.py +0 -50
  533. package/fp16/include/fp16/avx2.py +0 -53
  534. package/fp16/include/fp16/bitcasts.h +0 -92
  535. package/fp16/include/fp16/fp16.h +0 -451
  536. package/fp16/include/fp16/psimd.h +0 -131
  537. package/fp16/include/fp16.h +0 -11
  538. package/fp16/jni/Android.mk +0 -6
  539. package/fp16/jni/Application.mk +0 -4
  540. package/fp16/test/alt-from-fp32-value.cc +0 -498
  541. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  542. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  543. package/fp16/test/alt-to-fp32-value.cc +0 -348
  544. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  545. package/fp16/test/bitcasts.cc +0 -56
  546. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  547. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  548. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  549. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  550. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  551. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  552. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  553. package/fp16/test/peachpy/stubs.py +0 -38
  554. package/fp16/test/tables.cc +0 -530
  555. package/fp16/test/tables.h +0 -13
  556. package/fp16/third-party/THHalf.h +0 -92
  557. package/fp16/third-party/eigen-half.h +0 -170
  558. package/fp16/third-party/float16-compressor.h +0 -80
  559. package/fp16/third-party/half.hpp +0 -2916
  560. package/fp16/third-party/npy-halffloat.h +0 -328
  561. package/simsimd/.git-blame-ignore-revs +0 -2
  562. package/simsimd/CONTRIBUTING.md +0 -321
  563. package/simsimd/Cargo.toml +0 -49
  564. package/simsimd/MANIFEST.in +0 -4
  565. package/simsimd/Package.swift +0 -28
  566. package/simsimd/README.md +0 -1493
  567. package/simsimd/binding.gyp +0 -35
  568. package/simsimd/build.rs +0 -70
  569. package/simsimd/c/lib.c +0 -374
  570. package/simsimd/include/module.modulemap +0 -4
  571. package/simsimd/include/simsimd/binary.h +0 -486
  572. package/simsimd/include/simsimd/curved.h +0 -1541
  573. package/simsimd/include/simsimd/dot.h +0 -1853
  574. package/simsimd/include/simsimd/elementwise.h +0 -2465
  575. package/simsimd/include/simsimd/geospatial.h +0 -43
  576. package/simsimd/include/simsimd/mesh.h +0 -69
  577. package/simsimd/include/simsimd/probability.h +0 -606
  578. package/simsimd/include/simsimd/simsimd.h +0 -2567
  579. package/simsimd/include/simsimd/sparse.h +0 -1384
  580. package/simsimd/include/simsimd/spatial.h +0 -2335
  581. package/simsimd/include/simsimd/types.h +0 -666
  582. package/simsimd/javascript/fallback.ts +0 -233
  583. package/simsimd/javascript/lib.c +0 -125
  584. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  585. package/simsimd/javascript/simsimd.ts +0 -179
  586. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  587. package/simsimd/package-lock.json +0 -1761
  588. package/simsimd/package.json +0 -61
  589. package/simsimd/rust/lib.rs +0 -1821
  590. package/simsimd/scripts/bench.cxx +0 -1170
  591. package/simsimd/scripts/bench.ipynb +0 -187
  592. package/simsimd/scripts/bench.js +0 -127
  593. package/simsimd/scripts/bench.rs +0 -876
  594. package/simsimd/scripts/bench_vectors.py +0 -734
  595. package/simsimd/scripts/bench_vectors_live.py +0 -213
  596. package/simsimd/scripts/test.c +0 -187
  597. package/simsimd/scripts/test.mjs +0 -220
  598. package/simsimd/scripts/test.py +0 -1505
  599. package/simsimd/swift/SimSIMD.swift +0 -149
  600. package/simsimd/swift/Test.swift +0 -128
  601. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  602. /package/{simsimd → numkong}/LICENSE +0 -0
  603. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  604. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -1,1541 +0,0 @@
1
- /**
2
- * @file curved.h
3
- * @brief SIMD-accelerated Similarity Measures for curved spaces.
4
- * @author Ash Vardanian
5
- * @date August 27, 2024
6
- *
7
- * Contains:
8
- * - Mahalanobis distance
9
- * - Bilinear form multiplication
10
- * - Bilinear form multiplication over complex numbers
11
- *
12
- * For datatypes:
13
- * - 64-bit floating point numbers
14
- * - 32-bit floating point numbers
15
- * - 16-bit floating point numbers
16
- * - 16-bit brain-floating point numbers
17
- *
18
- * For hardware architectures:
19
- * - Arm: NEON
20
- * - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
21
- *
22
- * Most kernels in this file are designed for BLAS level 2 operations, where the operands are
23
- * a combination of matrices and vectors, generally forming a chain of multiplications.
24
- * Most kernels exploit the fact that matrix multiplication is associative, and the order of
25
- * operations can be changed to minimize the number of operations: `(A * B) * C = A * (B * C)`.
26
- * To optimize the performance, we minimize the number of memory accesses, and maximize the
27
- * number of arithmetic operations, by using SIMD instructions.
28
- *
29
- * When A and C are vectors, and B is a matrix, we can load every element in B just once, and
30
- * reuse it for every element in A and C.
31
- *
32
- * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
33
- * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
34
- */
35
- #ifndef SIMSIMD_CURVED_H
36
- #define SIMSIMD_CURVED_H
37
-
38
- #include "types.h"
39
-
40
- #include "dot.h" // `_simsimd_partial_load_f16x4_neon` and friends
41
- #include "spatial.h" // `_simsimd_substract_bf16x32_genoa`
42
-
43
- #ifdef __cplusplus
44
- extern "C" {
45
- #endif
46
-
47
- // clang-format off
48
-
49
- /* Serial backends for all numeric types.
50
- * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
51
- * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
52
- */
53
- SIMSIMD_PUBLIC void simsimd_bilinear_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
54
- SIMSIMD_PUBLIC void simsimd_bilinear_f64c_serial(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
55
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
56
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
57
- SIMSIMD_PUBLIC void simsimd_bilinear_f32c_serial(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
58
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
59
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
60
- SIMSIMD_PUBLIC void simsimd_bilinear_f16c_serial(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
61
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
62
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
63
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_serial(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
64
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
65
-
66
- /* Double-precision serial backends for all numeric types.
67
- * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
68
- */
69
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
70
- SIMSIMD_PUBLIC void simsimd_bilinear_f32c_accurate(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
71
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
72
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
73
- SIMSIMD_PUBLIC void simsimd_bilinear_f16c_accurate(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
74
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
75
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
76
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_accurate(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
77
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
78
-
79
- /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
80
- * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
81
- * server CPUs produced before 2023.
82
- */
83
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
84
- SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
85
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
86
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
87
- SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
88
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
89
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
90
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
91
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
92
-
93
- /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
94
- * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
95
- * Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
96
- * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
97
- * properly vectorized by recent compilers.
98
- */
99
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
100
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
101
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
102
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
103
-
104
- /* SIMD-powered backends for various generations of AVX512 CPUs.
105
- * Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
106
- * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
107
- * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
108
- */
109
- SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
110
- SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
111
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
112
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
113
- SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
114
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
115
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
116
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
117
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
118
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
119
- SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
120
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
121
- // clang-format on
122
-
123
- #define SIMSIMD_MAKE_BILINEAR(name, input_type, accumulator_type, load_and_convert) \
124
- SIMSIMD_PUBLIC void simsimd_bilinear_##input_type##_##name( \
125
- simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
126
- simsimd_size_t n, simsimd_distance_t *result) { \
127
- simsimd_##accumulator_type##_t sum = 0; \
128
- for (simsimd_size_t i = 0; i != n; ++i) { \
129
- simsimd_##accumulator_type##_t cb_j = 0; \
130
- simsimd_##accumulator_type##_t a_i = load_and_convert(a + i); \
131
- for (simsimd_size_t j = 0; j != n; ++j) { \
132
- simsimd_##accumulator_type##_t b_j = load_and_convert(b + j); \
133
- simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
134
- cb_j += c_ij * b_j; \
135
- } \
136
- sum += a_i * cb_j; \
137
- } \
138
- *result = (simsimd_distance_t)sum; \
139
- }
140
-
141
- #define SIMSIMD_MAKE_COMPLEX_BILINEAR(name, input_type, accumulator_type, load_and_convert) \
142
- SIMSIMD_PUBLIC void simsimd_bilinear_##input_type##_##name( \
143
- simsimd_##input_type##_t const *a_pairs, simsimd_##input_type##_t const *b_pairs, \
144
- simsimd_##input_type##_t const *c_pairs, simsimd_size_t n, simsimd_distance_t *results) { \
145
- simsimd_##accumulator_type##_t sum_real = 0; \
146
- simsimd_##accumulator_type##_t sum_imag = 0; \
147
- for (simsimd_size_t i = 0; i != n; ++i) { \
148
- simsimd_##accumulator_type##_t cb_j_real = 0; \
149
- simsimd_##accumulator_type##_t cb_j_imag = 0; \
150
- simsimd_##accumulator_type##_t a_i_real = load_and_convert(&(a_pairs + i)->real); \
151
- simsimd_##accumulator_type##_t a_i_imag = load_and_convert(&(a_pairs + i)->imag); \
152
- for (simsimd_size_t j = 0; j != n; ++j) { \
153
- simsimd_##accumulator_type##_t b_j_real = load_and_convert(&(b_pairs + j)->real); \
154
- simsimd_##accumulator_type##_t b_j_imag = load_and_convert(&(b_pairs + j)->imag); \
155
- simsimd_##accumulator_type##_t c_ij_real = load_and_convert(&(c_pairs + i * n + j)->real); \
156
- simsimd_##accumulator_type##_t c_ij_imag = load_and_convert(&(c_pairs + i * n + j)->imag); \
157
- /* Complex multiplication: (c_ij * b_j) */ \
158
- cb_j_real += c_ij_real * b_j_real - c_ij_imag * b_j_imag; \
159
- cb_j_imag += c_ij_real * b_j_imag + c_ij_imag * b_j_real; \
160
- } \
161
- /* Complex multiplication: (a_i * cb_j) */ \
162
- sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag; \
163
- sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real; \
164
- } \
165
- results[0] = (simsimd_distance_t)sum_real; \
166
- results[1] = (simsimd_distance_t)sum_imag; \
167
- }
168
-
169
- #define SIMSIMD_MAKE_MAHALANOBIS(name, input_type, accumulator_type, load_and_convert) \
170
- SIMSIMD_PUBLIC void simsimd_mahalanobis_##input_type##_##name( \
171
- simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
172
- simsimd_size_t n, simsimd_distance_t *result) { \
173
- simsimd_##accumulator_type##_t sum = 0; \
174
- for (simsimd_size_t i = 0; i != n; ++i) { \
175
- simsimd_##accumulator_type##_t cdiff_j = 0; \
176
- simsimd_##accumulator_type##_t diff_i = load_and_convert(a + i) - load_and_convert(b + i); \
177
- for (simsimd_size_t j = 0; j != n; ++j) { \
178
- simsimd_##accumulator_type##_t diff_j = load_and_convert(a + j) - load_and_convert(b + j); \
179
- simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
180
- cdiff_j += c_ij * diff_j; \
181
- } \
182
- sum += diff_i * cdiff_j; \
183
- } \
184
- *result = (simsimd_distance_t)SIMSIMD_SQRT(sum); \
185
- }
186
-
187
- SIMSIMD_MAKE_BILINEAR(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64_serial
188
- SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f64c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64c_serial
189
- SIMSIMD_MAKE_MAHALANOBIS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f64_serial
190
-
191
- SIMSIMD_MAKE_BILINEAR(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_serial
192
- SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f32c, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_serial
193
- SIMSIMD_MAKE_MAHALANOBIS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_serial
194
-
195
- SIMSIMD_MAKE_BILINEAR(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_serial
196
- SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f16c, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_serial
197
- SIMSIMD_MAKE_MAHALANOBIS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_serial
198
-
199
- SIMSIMD_MAKE_BILINEAR(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_serial
200
- SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, bf16c, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_serial
201
- SIMSIMD_MAKE_MAHALANOBIS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_serial
202
-
203
- SIMSIMD_MAKE_BILINEAR(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_accurate
204
- SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f32c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_accurate
205
- SIMSIMD_MAKE_MAHALANOBIS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_accurate
206
-
207
- SIMSIMD_MAKE_BILINEAR(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_accurate
208
- SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f16c, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_accurate
209
- SIMSIMD_MAKE_MAHALANOBIS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_accurate
210
-
211
- SIMSIMD_MAKE_BILINEAR(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_accurate
212
- SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, bf16c, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_accurate
213
- SIMSIMD_MAKE_MAHALANOBIS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_accurate
214
-
215
- #if _SIMSIMD_TARGET_ARM
216
- #if SIMSIMD_TARGET_NEON
217
- #pragma GCC push_options
218
- #pragma GCC target("arch=armv8.2-a+simd")
219
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
220
-
221
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
222
- simsimd_size_t n, simsimd_distance_t *result) {
223
- float32x4_t sum_vec = vdupq_n_f32(0);
224
- for (simsimd_size_t i = 0; i != n; ++i) {
225
- float32x4_t a_vec = vdupq_n_f32(a[i]);
226
- float32x4_t cb_j_vec = vdupq_n_f32(0);
227
- for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
228
- float32x4_t b_vec = vld1q_f32(b + j);
229
- float32x4_t c_vec = vld1q_f32(c + i * n + j);
230
- cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
231
- }
232
- sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
233
- }
234
-
235
- // Handle the tail of every row
236
- simsimd_f64_t sum = vaddvq_f32(sum_vec);
237
- simsimd_size_t const tail_length = n % 4;
238
- simsimd_size_t const tail_start = n - tail_length;
239
- if (tail_length) {
240
- for (simsimd_size_t i = 0; i != n; ++i) {
241
- simsimd_f32_t a_i = a[i];
242
- simsimd_f32_t cb_j = 0;
243
- for (simsimd_size_t j = tail_start; j != n; ++j) cb_j += b[j] * c[i * n + j];
244
- sum += a[i] * cb_j;
245
- }
246
- }
247
-
248
- *result = sum;
249
- }
250
-
251
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
252
- simsimd_size_t n, simsimd_distance_t *result) {
253
- float32x4_t sum_vec = vdupq_n_f32(0);
254
- for (simsimd_size_t i = 0; i != n; ++i) {
255
- float32x4_t diff_i_vec = vdupq_n_f32(a[i] - b[i]);
256
- float32x4_t cdiff_j_vec = vdupq_n_f32(0);
257
- for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
258
- float32x4_t diff_j_vec = vsubq_f32(vld1q_f32(a + j), vld1q_f32(b + j));
259
- float32x4_t c_vec = vld1q_f32(c + i * n + j);
260
- cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
261
- }
262
-
263
- sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
264
- }
265
-
266
- // Handle the tail of every row
267
- simsimd_f64_t sum = vaddvq_f32(sum_vec);
268
- simsimd_size_t const tail_length = n % 4;
269
- simsimd_size_t const tail_start = n - tail_length;
270
- if (tail_length) {
271
- for (simsimd_size_t i = 0; i != n; ++i) {
272
- simsimd_f32_t diff_i = a[i] - b[i];
273
- simsimd_f32_t cdiff_j = 0;
274
- for (simsimd_size_t j = tail_start; j != n; ++j) {
275
- simsimd_f32_t diff_j = a[j] - b[j];
276
- cdiff_j += diff_j * c[i * n + j];
277
- }
278
- sum += diff_i * cdiff_j;
279
- }
280
- }
281
-
282
- *result = _simsimd_sqrt_f64_neon(sum);
283
- }
284
-
285
- SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
286
- simsimd_f32c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
287
- simsimd_f32_t sum_real = 0;
288
- simsimd_f32_t sum_imag = 0;
289
- for (simsimd_size_t i = 0; i != n; ++i) {
290
- simsimd_f32c_t a_i = a[i];
291
- simsimd_f32c_t cb_j;
292
- float32x4_t cb_j_real_vec = vdupq_n_f32(0);
293
- float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
294
- for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
295
- // Unpack the input arrays into real and imaginary parts:
296
- float32x4x2_t b_vec = vld2q_f32((simsimd_f32_t const *)(b + j));
297
- float32x4x2_t c_vec = vld2q_f32((simsimd_f32_t const *)(c + i * n + j));
298
- float32x4_t b_real_vec = b_vec.val[0];
299
- float32x4_t b_imag_vec = b_vec.val[1];
300
- float32x4_t c_real_vec = c_vec.val[0];
301
- float32x4_t c_imag_vec = c_vec.val[1];
302
-
303
- // Compute the dot product:
304
- cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
305
- cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
306
- cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
307
- cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
308
- }
309
- cb_j.real = vaddvq_f32(cb_j_real_vec);
310
- cb_j.imag = vaddvq_f32(cb_j_imag_vec);
311
- sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
312
- sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
313
- }
314
-
315
- // Handle the tail of every row
316
- simsimd_size_t const tail_length = n % 4;
317
- simsimd_size_t const tail_start = n - tail_length;
318
- if (tail_length) {
319
- for (simsimd_size_t i = 0; i != n; ++i) {
320
- simsimd_f32c_t a_i = a[i];
321
- simsimd_f32c_t cb_j = {0, 0};
322
- for (simsimd_size_t j = tail_start; j != n; ++j) {
323
- simsimd_f32c_t b_j = b[j];
324
- simsimd_f32c_t c_ij = c[i * n + j];
325
- cb_j.real += b_j.real * c_ij.real - b_j.imag * c_ij.imag;
326
- cb_j.imag += b_j.real * c_ij.imag + b_j.imag * c_ij.real;
327
- }
328
- sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
329
- sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
330
- }
331
- }
332
-
333
- results[0] = sum_real;
334
- results[1] = sum_imag;
335
- }
336
-
337
- #pragma clang attribute pop
338
- #pragma GCC pop_options
339
- #endif // SIMSIMD_TARGET_NEON
340
-
341
- #if SIMSIMD_TARGET_NEON_F16
342
- #pragma GCC push_options
343
- #pragma GCC target("arch=armv8.2-a+simd+fp16")
344
- #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
345
-
346
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c,
347
- simsimd_size_t n, simsimd_distance_t *result) {
348
- float32x4_t sum_vec = vdupq_n_f32(0);
349
- for (simsimd_size_t i = 0; i != n; ++i) {
350
- // MSVC doesn't recognize `vdup_n_f16` as a valid intrinsic
351
- float32x4_t a_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
352
- float32x4_t cb_j_vec = vdupq_n_f32(0);
353
- for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
354
- float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
355
- float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
356
- cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
357
- }
358
- sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
359
- }
360
-
361
- // Handle the tail of every row
362
- simsimd_f64_t sum = vaddvq_f32(sum_vec);
363
- simsimd_size_t const tail_length = n % 4;
364
- simsimd_size_t const tail_start = n - tail_length;
365
- if (tail_length) {
366
- for (simsimd_size_t i = 0; i != n; ++i) {
367
- simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
368
- float32x4_t b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
369
- float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
370
- simsimd_f32_t cb_j = vaddvq_f32(vmulq_f32(b_vec, c_vec));
371
- sum += a_i * cb_j;
372
- }
373
- }
374
-
375
- *result = sum;
376
- }
377
-
378
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c,
379
- simsimd_size_t n, simsimd_distance_t *result) {
380
- float32x4_t sum_vec = vdupq_n_f32(0);
381
- for (simsimd_size_t i = 0; i != n; ++i) {
382
- // MSVC doesn't recognize `vdup_n_f16` as a valid intrinsic
383
- float32x4_t a_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
384
- float32x4_t b_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(b + i))));
385
- float32x4_t diff_i_vec = vsubq_f32(a_i_vec, b_i_vec);
386
- float32x4_t cdiff_j_vec = vdupq_n_f32(0);
387
- for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
388
- float32x4_t a_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(a + j)));
389
- float32x4_t b_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
390
- float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
391
- float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
392
- cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
393
- }
394
- sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
395
- }
396
-
397
- // Handle the tail of every row
398
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
399
- simsimd_size_t const tail_length = n % 4;
400
- simsimd_size_t const tail_start = n - tail_length;
401
- if (tail_length) {
402
- for (simsimd_size_t i = 0; i != n; ++i) {
403
- simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
404
- simsimd_f32_t b_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + i, 1)));
405
- simsimd_f32_t diff_i = a_i - b_i;
406
- float32x4_t a_j_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + tail_start, tail_length));
407
- float32x4_t b_j_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
408
- float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
409
- float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
410
- simsimd_f32_t cdiff_j = vaddvq_f32(vmulq_f32(diff_j_vec, c_vec));
411
- sum += diff_i * cdiff_j;
412
- }
413
- }
414
-
415
- *result = _simsimd_sqrt_f32_neon(sum);
416
- }
417
-
418
- SIMSIMD_INTERNAL simsimd_f32_t _simsimd_reduce_f16x8_neon(float16x8_t vec) {
419
- // Split the 8-element vector into two 4-element vectors
420
- float16x4_t low = vget_low_f16(vec); // Lower 4 elements
421
- float16x4_t high = vget_high_f16(vec); // Upper 4 elements
422
-
423
- // Add the lower and upper parts
424
- float16x4_t sum = vadd_f16(low, high);
425
-
426
- // Perform pairwise addition to reduce 4 elements to 2, then to 1
427
- sum = vpadd_f16(sum, sum); // First reduction: 4 -> 2
428
- sum = vpadd_f16(sum, sum); // Second reduction: 2 -> 1
429
-
430
- // Convert the remaining half-precision value to single-precision and return
431
- return vgetq_lane_f32(vcvt_f32_f16(sum), 0);
432
- }
433
-
434
- SIMSIMD_INTERNAL float16x8x2_t _simsimd_partial_load_f16x8x2_neon(simsimd_f16c_t const *x, simsimd_size_t n) {
435
- union {
436
- float16x8x2_t vecs;
437
- simsimd_f16_t scalars[2][8];
438
- } result;
439
- simsimd_size_t i = 0;
440
- for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
441
- for (; i < 8; ++i) result.scalars[0][i] = 0, result.scalars[1][i] = 0;
442
- return result.vecs;
443
- }
444
-
445
- SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
446
- simsimd_f16c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
447
- simsimd_f32_t sum_real = 0;
448
- simsimd_f32_t sum_imag = 0;
449
- simsimd_size_t const tail_length = n % 8;
450
- simsimd_size_t const tail_start = n - tail_length;
451
- for (simsimd_size_t i = 0; i != n; ++i) {
452
- simsimd_f32c_t a_i = {simsimd_f16_to_f32(&a[i].real), simsimd_f16_to_f32(&a[i].imag)};
453
- float16x8_t cb_j_real_vec = vdupq_n_f16(0);
454
- float16x8_t cb_j_imag_vec = vdupq_n_f16(0);
455
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
456
- // Unpack the input arrays into real and imaginary parts:
457
- float16x8x2_t b_vec = vld2q_f16((float16_t const *)(b + j));
458
- float16x8x2_t c_vec = vld2q_f16((float16_t const *)(c + i * n + j));
459
- float16x8_t b_real_vec = b_vec.val[0];
460
- float16x8_t b_imag_vec = b_vec.val[1];
461
- float16x8_t c_real_vec = c_vec.val[0];
462
- float16x8_t c_imag_vec = c_vec.val[1];
463
-
464
- // Compute the dot product:
465
- cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
466
- cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
467
- cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
468
- cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
469
- }
470
- // Handle row tails
471
- if (tail_length) {
472
- // Unpack the input arrays into real and imaginary parts:
473
- float16x8x2_t b_vec = _simsimd_partial_load_f16x8x2_neon(b + tail_start, tail_length);
474
- float16x8x2_t c_vec = _simsimd_partial_load_f16x8x2_neon(c + i * n + tail_start, tail_length);
475
- float16x8_t b_real_vec = b_vec.val[0];
476
- float16x8_t b_imag_vec = b_vec.val[1];
477
- float16x8_t c_real_vec = c_vec.val[0];
478
- float16x8_t c_imag_vec = c_vec.val[1];
479
-
480
- // Compute the dot product:
481
- cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
482
- cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
483
- cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
484
- cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
485
- }
486
-
487
- simsimd_f32c_t cb_j;
488
- cb_j.real = _simsimd_reduce_f16x8_neon(cb_j_real_vec);
489
- cb_j.imag = _simsimd_reduce_f16x8_neon(cb_j_imag_vec);
490
- sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
491
- sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
492
- }
493
-
494
- results[0] = sum_real;
495
- results[1] = sum_imag;
496
- }
497
-
498
- #pragma clang attribute pop
499
- #pragma GCC pop_options
500
- #endif // SIMSIMD_TARGET_NEON_F16
501
-
502
- #if SIMSIMD_TARGET_NEON_BF16
503
- #pragma GCC push_options
504
- #pragma GCC target("arch=armv8.6-a+simd+bf16")
505
- #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
506
-
507
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
508
- simsimd_bf16_t const *c, simsimd_size_t n, simsimd_distance_t *result) {
509
- float32x4_t sum_vec = vdupq_n_f32(0);
510
- for (simsimd_size_t i = 0; i != n; ++i) {
511
- float32x4_t a_vec = vdupq_n_f32(simsimd_bf16_to_f32(a + i));
512
- float32x4_t cb_j_vec = vdupq_n_f32(0);
513
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
514
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
515
- bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
516
- cb_j_vec = vbfdotq_f32(cb_j_vec, b_vec, c_vec);
517
- }
518
- sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
519
- }
520
-
521
- // Handle the tail of every row
522
- simsimd_f64_t sum = vaddvq_f32(sum_vec);
523
- simsimd_size_t const tail_length = n % 8;
524
- simsimd_size_t const tail_start = n - tail_length;
525
- if (tail_length) {
526
- for (simsimd_size_t i = 0; i != n; ++i) {
527
- simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
528
- bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b + tail_start, tail_length);
529
- bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
530
- simsimd_f32_t cb_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), b_vec, c_vec));
531
- sum += a_i * cb_j;
532
- }
533
- }
534
-
535
- *result = sum;
536
- }
537
-
538
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
539
- simsimd_bf16_t const *c, simsimd_size_t n,
540
- simsimd_distance_t *result) {
541
- float32x4_t sum_vec = vdupq_n_f32(0);
542
- for (simsimd_size_t i = 0; i != n; ++i) {
543
- simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
544
- simsimd_f32_t b_i = simsimd_bf16_to_f32(b + i);
545
- float32x4_t diff_i_vec = vdupq_n_f32(a_i - b_i);
546
- float32x4_t cdiff_j_vec = vdupq_n_f32(0);
547
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
548
- bfloat16x8_t a_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(a + j));
549
- bfloat16x8_t b_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
550
-
551
- // Arm NEON does not have a native subtraction instruction for `bf16`,
552
- // so we need to convert to `f32` first, subtract, and only then get back to `bf16`
553
- // for multiplication.
554
- float32x4_t a_j_vec_high = vcvt_f32_bf16(vget_high_bf16(a_j_vec));
555
- float32x4_t a_j_vec_low = vcvt_f32_bf16(vget_low_bf16(a_j_vec));
556
- float32x4_t b_j_vec_high = vcvt_f32_bf16(vget_high_bf16(b_j_vec));
557
- float32x4_t b_j_vec_low = vcvt_f32_bf16(vget_low_bf16(b_j_vec));
558
- float32x4_t diff_j_vec_high = vsubq_f32(a_j_vec_high, b_j_vec_high);
559
- float32x4_t diff_j_vec_low = vsubq_f32(a_j_vec_low, b_j_vec_low);
560
- bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
561
-
562
- bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
563
- cdiff_j_vec = vbfdotq_f32(cdiff_j_vec, diff_j_vec, c_vec);
564
- }
565
- sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
566
- }
567
-
568
- // Handle the tail of every row
569
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
570
- simsimd_size_t const tail_length = n % 8;
571
- simsimd_size_t const tail_start = n - tail_length;
572
- if (tail_length) {
573
- for (simsimd_size_t i = 0; i != n; ++i) {
574
- simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
575
- simsimd_f32_t b_i = simsimd_bf16_to_f32(b + i);
576
- simsimd_f32_t diff_i = a_i - b_i;
577
- bfloat16x8_t a_j_vec = _simsimd_partial_load_bf16x8_neon(a + tail_start, tail_length);
578
- bfloat16x8_t b_j_vec = _simsimd_partial_load_bf16x8_neon(b + tail_start, tail_length);
579
-
580
- // Again, upcast for subtraction
581
- float32x4_t a_j_vec_high = vcvt_f32_bf16(vget_high_bf16(a_j_vec));
582
- float32x4_t a_j_vec_low = vcvt_f32_bf16(vget_low_bf16(a_j_vec));
583
- float32x4_t b_j_vec_high = vcvt_f32_bf16(vget_high_bf16(b_j_vec));
584
- float32x4_t b_j_vec_low = vcvt_f32_bf16(vget_low_bf16(b_j_vec));
585
- float32x4_t diff_j_vec_high = vsubq_f32(a_j_vec_high, b_j_vec_high);
586
- float32x4_t diff_j_vec_low = vsubq_f32(a_j_vec_low, b_j_vec_low);
587
- bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
588
-
589
- bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
590
- simsimd_f32_t cdiff_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), diff_j_vec, c_vec));
591
- sum += diff_i * cdiff_j;
592
- }
593
- }
594
-
595
- *result = _simsimd_sqrt_f32_neon(sum);
596
- }
597
-
598
- SIMSIMD_INTERNAL int16x4x2_t _simsimd_partial_load_bf16x4x2_neon(simsimd_bf16c_t const *x, simsimd_size_t n) {
599
- union {
600
- int16x4x2_t vec;
601
- simsimd_bf16_t scalars[2][4];
602
- } result;
603
- simsimd_size_t i = 0;
604
- for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
605
- for (; i < 4; ++i) result.scalars[1][i] = 0, result.scalars[1][i] = 0;
606
- return result.vec;
607
- }
608
-
609
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
610
- simsimd_bf16c_t const *c, simsimd_size_t n,
611
- simsimd_distance_t *results) {
612
- simsimd_f32_t sum_real = 0;
613
- simsimd_f32_t sum_imag = 0;
614
- simsimd_size_t const tail_length = n % 4;
615
- simsimd_size_t const tail_start = n - tail_length;
616
- for (simsimd_size_t i = 0; i != n; ++i) {
617
- simsimd_f32c_t a_i = {simsimd_bf16_to_f32(&a[i].real), simsimd_bf16_to_f32(&a[i].imag)};
618
- // A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
619
- // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
620
- // at once. That's how the original implementation worked, but compiling it was a nightmare :)
621
- float32x4_t cb_j_real_vec = vdupq_n_f32(0);
622
- float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
623
- for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
624
- // Unpack the input arrays into real and imaginary parts.
625
- // MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
626
- // integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
627
- int16x4x2_t b_vec = vld2_s16((short const *)(b + j));
628
- int16x4x2_t c_vec = vld2_s16((short const *)(c + i * n + j));
629
- float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
630
- float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
631
- float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
632
- float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
633
-
634
- // Compute the dot product:
635
- cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
636
- cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
637
- cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
638
- cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
639
- }
640
- // Handle row tails
641
- if (tail_length) {
642
- // Unpack the input arrays into real and imaginary parts:
643
- int16x4x2_t b_vec = _simsimd_partial_load_bf16x4x2_neon(b + tail_start, tail_length);
644
- int16x4x2_t c_vec = _simsimd_partial_load_bf16x4x2_neon(c + i * n + tail_start, tail_length);
645
- float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
646
- float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
647
- float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
648
- float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
649
-
650
- // Compute the dot product:
651
- cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
652
- cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
653
- cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
654
- cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
655
- }
656
-
657
- simsimd_f32c_t cb_j;
658
- cb_j.real = vaddvq_f32(cb_j_real_vec);
659
- cb_j.imag = vaddvq_f32(cb_j_imag_vec);
660
- sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
661
- sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
662
- }
663
-
664
- results[0] = sum_real;
665
- results[1] = sum_imag;
666
- }
667
-
668
- #pragma clang attribute pop
669
- #pragma GCC pop_options
670
- #endif // SIMSIMD_TARGET_NEON_BF16
671
-
672
- #endif // _SIMSIMD_TARGET_ARM
673
-
674
- #if _SIMSIMD_TARGET_X86
675
- #if SIMSIMD_TARGET_HASWELL
676
- #pragma GCC push_options
677
- #pragma GCC target("avx2", "f16c", "fma")
678
- #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
679
-
680
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c,
681
- simsimd_size_t n, simsimd_distance_t *result) {
682
- __m256 sum_vec = _mm256_setzero_ps();
683
- for (simsimd_size_t i = 0; i != n; ++i) {
684
- __m256 a_vec = _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i)));
685
- __m256 cb_j_vec = _mm256_setzero_ps();
686
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
687
- __m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j)));
688
- __m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
689
- cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
690
- }
691
- sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
692
- }
693
-
694
- // Handle the tail of every row
695
- simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
696
- simsimd_size_t const tail_length = n % 8;
697
- simsimd_size_t const tail_start = n - tail_length;
698
- if (tail_length) {
699
- for (simsimd_size_t i = 0; i != n; ++i) {
700
- simsimd_f32_t a_i = _mm256_cvtss_f32(_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))));
701
- __m256 b_vec = _simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length);
702
- __m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
703
- simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
704
- sum += a_i * cb_j;
705
- }
706
- }
707
-
708
- *result = sum;
709
- }
710
-
711
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b,
712
- simsimd_f16_t const *c, simsimd_size_t n,
713
- simsimd_distance_t *result) {
714
- __m256 sum_vec = _mm256_setzero_ps();
715
- for (simsimd_size_t i = 0; i != n; ++i) {
716
- __m256 diff_i_vec = _mm256_sub_ps( //
717
- _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))), //
718
- _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(b + i))));
719
- __m256 cdiff_j_vec = _mm256_setzero_ps();
720
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
721
- __m256 diff_j_vec = _mm256_sub_ps( //
722
- _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(a + j))),
723
- _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j))));
724
- __m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
725
- cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
726
- }
727
- sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
728
- }
729
-
730
- // Handle the tail of every row
731
- simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
732
- simsimd_size_t const tail_length = n % 8;
733
- simsimd_size_t const tail_start = n - tail_length;
734
- if (tail_length) {
735
- for (simsimd_size_t i = 0; i != n; ++i) {
736
- simsimd_f32_t diff_i = _mm256_cvtss_f32(_mm256_sub_ps( //
737
- _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))), //
738
- _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(b + i)))));
739
- __m256 diff_j_vec = _mm256_sub_ps( //
740
- _simsimd_partial_load_f16x8_haswell(a + tail_start, tail_length),
741
- _simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length));
742
- __m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
743
- simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
744
- sum += diff_i * cdiff_j;
745
- }
746
- }
747
-
748
- *result = _simsimd_sqrt_f32_haswell(sum);
749
- }
750
-
751
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
752
- simsimd_bf16_t const *c, simsimd_size_t n,
753
- simsimd_distance_t *result) {
754
- __m256 sum_vec = _mm256_setzero_ps();
755
- for (simsimd_size_t i = 0; i != n; ++i) {
756
- // The `simsimd_bf16_to_f32` is cheaper than `_simsimd_bf16x8_to_f32x8_haswell`
757
- __m256 a_vec = _mm256_set1_ps(simsimd_bf16_to_f32(a + i));
758
- __m256 cb_j_vec = _mm256_setzero_ps();
759
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
760
- __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j)));
761
- __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
762
- cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
763
- }
764
- sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
765
- }
766
-
767
- // Handle the tail of every row
768
- simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
769
- simsimd_size_t const tail_length = n % 8;
770
- simsimd_size_t const tail_start = n - tail_length;
771
- if (tail_length) {
772
- for (simsimd_size_t i = 0; i != n; ++i) {
773
- simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
774
- __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell( //
775
- _simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length));
776
- __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell( //
777
- _simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
778
- simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
779
- sum += a_i * cb_j;
780
- }
781
- }
782
-
783
- *result = sum;
784
- }
785
-
786
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
787
- simsimd_bf16_t const *c, simsimd_size_t n,
788
- simsimd_distance_t *result) {
789
- __m256 sum_vec = _mm256_setzero_ps();
790
- for (simsimd_size_t i = 0; i != n; ++i) {
791
- __m256 diff_i_vec = _mm256_sub_ps( //
792
- _mm256_set1_ps(simsimd_bf16_to_f32(a + i)), //
793
- _mm256_set1_ps(simsimd_bf16_to_f32(b + i)));
794
- __m256 cdiff_j_vec = _mm256_setzero_ps();
795
- for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
796
- __m256 diff_j_vec = _mm256_sub_ps( //
797
- _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(a + j))), //
798
- _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j))));
799
- __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
800
- cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
801
- }
802
- sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
803
- }
804
-
805
- // Handle the tail of every row
806
- simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
807
- simsimd_size_t const tail_length = n % 8;
808
- simsimd_size_t const tail_start = n - tail_length;
809
- if (tail_length) {
810
- for (simsimd_size_t i = 0; i != n; ++i) {
811
- simsimd_f32_t diff_i = simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i);
812
- __m256 diff_j_vec = _mm256_sub_ps( //
813
- _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a + tail_start, tail_length)),
814
- _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length)));
815
- __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(
816
- _simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
817
- simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
818
- sum += diff_i * cdiff_j;
819
- }
820
- }
821
-
822
- *result = _simsimd_sqrt_f32_haswell(sum);
823
- }
824
-
825
- #pragma clang attribute pop
826
- #pragma GCC pop_options
827
- #endif // SIMSIMD_TARGET_HASWELL
828
-
829
- #if SIMSIMD_TARGET_SKYLAKE
830
- #pragma GCC push_options
831
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2")
832
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2"))), apply_to = function)
833
-
834
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake_under16unrolled(simsimd_f32_t const *a, simsimd_f32_t const *b,
835
- simsimd_f32_t const *c, simsimd_size_t n,
836
- simsimd_distance_t *result) {
837
- // The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
838
- // until the very end of the computation.
839
- __mmask16 const mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
840
- __m512 const b_vec = _mm512_maskz_loadu_ps(mask, b);
841
-
842
- __m512 cb_j1 = _mm512_setzero_ps();
843
- __m512 cb_j2 = _mm512_setzero_ps();
844
- __m512 cb_j3 = _mm512_setzero_ps();
845
- __m512 cb_j4 = _mm512_setzero_ps();
846
-
847
- // Unroll the loop to process 4x ZMM registers at a time.
848
- simsimd_size_t i = 0;
849
- for (; i + 4 <= n; i += 4) {
850
- cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
851
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
852
- cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
853
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
854
- cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
855
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
856
- cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
857
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
858
- }
859
-
860
- if (i + 0 < n)
861
- cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
862
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
863
- if (i + 1 < n)
864
- cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
865
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
866
- if (i + 2 < n)
867
- cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
868
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
869
- if (i + 3 < n)
870
- cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
871
- _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
872
-
873
- // Combine cb_j sums
874
- __m512 sum_vec = _mm512_add_ps( //
875
- _mm512_add_ps(cb_j1, cb_j2), //
876
- _mm512_add_ps(cb_j3, cb_j4));
877
- *result = _mm512_reduce_add_ps(sum_vec);
878
- }
879
-
880
- SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
881
- simsimd_size_t n, simsimd_distance_t *result) {
882
-
883
- // On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
884
- // per row and 16 rows at a time, still keeping enough register space for temporaries.
885
- if (n <= 16) {
886
- simsimd_bilinear_f32_skylake_under16unrolled(a, b, c, n, result);
887
- return;
888
- }
889
-
890
- // Default case for arbitrary size `n`
891
- simsimd_size_t const tail_length = n % 16;
892
- simsimd_size_t const tail_start = n - tail_length;
893
- __m512 sum_vec = _mm512_setzero_ps();
894
- __mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
895
-
896
- for (simsimd_size_t i = 0; i != n; ++i) {
897
- __m512 a_vec = _mm512_set1_ps(a[i]);
898
- __m512 cb_j_vec = _mm512_setzero_ps();
899
- __m512 b_vec, c_vec;
900
- simsimd_size_t j = 0;
901
-
902
- simsimd_bilinear_f32_skylake_cycle:
903
- if (j + 16 <= n) {
904
- b_vec = _mm512_loadu_ps(b + j);
905
- c_vec = _mm512_loadu_ps(c + i * n + j);
906
- }
907
- else {
908
- b_vec = _mm512_maskz_loadu_ps(tail_mask, b + tail_start);
909
- c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
910
- }
911
- cb_j_vec = _mm512_fmadd_ps(b_vec, c_vec, cb_j_vec);
912
- j += 16;
913
- if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
914
- sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
915
- }
916
-
917
- *result = _mm512_reduce_add_ps(sum_vec);
918
- }
919
-
920
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b,
921
- simsimd_f32_t const *c, simsimd_size_t n,
922
- simsimd_distance_t *result) {
923
- simsimd_size_t const tail_length = n % 16;
924
- simsimd_size_t const tail_start = n - tail_length;
925
- __m512 sum_vec = _mm512_setzero_ps();
926
- __mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
927
-
928
- for (simsimd_size_t i = 0; i != n; ++i) {
929
- __m512 diff_i_vec = _mm512_set1_ps(a[i] - b[i]);
930
- __m512 cdiff_j_vec = _mm512_setzero_ps(), cdiff_j_bot_vec = _mm512_setzero_ps();
931
- __m512 a_j_vec, b_j_vec, diff_j_vec, c_vec;
932
- simsimd_size_t j = 0;
933
-
934
- // The nested loop is cleaner to implement with a `goto` in this case:
935
- simsimd_bilinear_f32_skylake_cycle:
936
- if (j + 16 <= n) {
937
- a_j_vec = _mm512_loadu_ps(a + j);
938
- b_j_vec = _mm512_loadu_ps(b + j);
939
- c_vec = _mm512_loadu_ps(c + i * n + j);
940
- }
941
- else {
942
- a_j_vec = _mm512_maskz_loadu_ps(tail_mask, a + tail_start);
943
- b_j_vec = _mm512_maskz_loadu_ps(tail_mask, b + tail_start);
944
- c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
945
- }
946
- diff_j_vec = _mm512_sub_ps(a_j_vec, b_j_vec);
947
- cdiff_j_vec = _mm512_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
948
- j += 16;
949
- if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
950
- sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
951
- }
952
-
953
- *result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_ps(sum_vec));
954
- }
955
-
956
- SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
957
- simsimd_f32c_t const *c, simsimd_size_t n,
958
- simsimd_distance_t *results) {
959
-
960
- // We take into account, that FMS is the same as FMA with a negative multiplier.
961
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
962
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
963
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
964
- // one of the vectors.
965
- __m512i const sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
966
-
967
- // Default case for arbitrary size `n`
968
- simsimd_size_t const tail_length = n % 8;
969
- simsimd_size_t const tail_start = n - tail_length;
970
- __mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
971
- simsimd_f32_t sum_real = 0;
972
- simsimd_f32_t sum_imag = 0;
973
-
974
- for (simsimd_size_t i = 0; i != n; ++i) {
975
- simsimd_f32_t const a_i_real = a[i].real;
976
- simsimd_f32_t const a_i_imag = a[i].imag;
977
- __m512 cb_j_real_vec = _mm512_setzero_ps();
978
- __m512 cb_j_imag_vec = _mm512_setzero_ps();
979
- __m512 b_vec, c_vec;
980
- simsimd_size_t j = 0;
981
-
982
- simsimd_bilinear_f32c_skylake_cycle:
983
- if (j + 8 <= n) {
984
- b_vec = _mm512_loadu_ps((simsimd_f32_t const *)(b + j));
985
- c_vec = _mm512_loadu_ps((simsimd_f32_t const *)(c + i * n + j));
986
- }
987
- else {
988
- b_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(b + tail_start));
989
- c_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(c + i * n + tail_start));
990
- }
991
- // The real part of the product: b.real * c.real - b.imag * c.imag.
992
- // The subtraction will be performed later with a sign flip.
993
- cb_j_real_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_real_vec);
994
- // The imaginary part of the product: b.real * c.imag + b.imag * c.real.
995
- // Swap the imaginary and real parts of `c` before multiplication:
996
- c_vec = _mm512_permute_ps(c_vec, 0xB1); //? Swap adjacent entries within each pair
997
- cb_j_imag_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_imag_vec);
998
- j += 8;
999
- if (j < n) goto simsimd_bilinear_f32c_skylake_cycle;
1000
- // Flip the sign bit in every second scalar before accumulation:
1001
- cb_j_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(cb_j_real_vec), sign_flip_vec));
1002
- // Horizontal sums are the expensive part of the computation:
1003
- simsimd_f32_t const cb_j_real = _mm512_reduce_add_ps(cb_j_real_vec);
1004
- simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ps(cb_j_imag_vec);
1005
- sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1006
- sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1007
- }
1008
-
1009
- // Reduce horizontal sums:
1010
- results[0] = sum_real;
1011
- results[1] = sum_imag;
1012
- }
1013
-
1014
- SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake_under8unrolled(simsimd_f64_t const *a, simsimd_f64_t const *b,
1015
- simsimd_f64_t const *c, simsimd_size_t n,
1016
- simsimd_distance_t *result) {
1017
-
1018
- // The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
1019
- // until the very end of the computation.
1020
- __mmask8 const row_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1021
- __m512d const b_vec = _mm512_maskz_loadu_pd(row_mask, b);
1022
-
1023
- __m512d cb_j1 = _mm512_setzero_pd();
1024
- __m512d cb_j2 = _mm512_setzero_pd();
1025
- __m512d cb_j3 = _mm512_setzero_pd();
1026
- __m512d cb_j4 = _mm512_setzero_pd();
1027
-
1028
- // clang-format off
1029
- if (n > 0) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 0), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[0])), cb_j1);
1030
- if (n > 1) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 1), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[1])), cb_j2);
1031
- if (n > 2) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 2), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[2])), cb_j3);
1032
- if (n > 3) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 3), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[3])), cb_j4);
1033
-
1034
- if (n > 4) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 4), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[4])), cb_j1);
1035
- if (n > 5) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 5), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[5])), cb_j2);
1036
- if (n > 6) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 6), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[6])), cb_j3);
1037
- if (n > 7) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 7), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[7])), cb_j4);
1038
- // clang-format on
1039
-
1040
- // Combine cb_j sums
1041
- __m512d sum_vec = _mm512_add_pd( //
1042
- _mm512_add_pd(cb_j1, cb_j2), //
1043
- _mm512_add_pd(cb_j3, cb_j4));
1044
- *result = _mm512_reduce_add_pd(sum_vec);
1045
- }
1046
-
1047
- SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c,
1048
- simsimd_size_t n, simsimd_distance_t *result) {
1049
-
1050
- // On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
1051
- // per row and 8 rows at a time, still keeping enough register space for temporaries.
1052
- if (n <= 8) {
1053
- simsimd_bilinear_f64_skylake_under8unrolled(a, b, c, n, result);
1054
- return;
1055
- }
1056
-
1057
- // Default case for arbitrary size `n`
1058
- simsimd_size_t const tail_length = n % 8;
1059
- simsimd_size_t const tail_start = n - tail_length;
1060
- __m512d sum_vec = _mm512_setzero_pd();
1061
- __mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
1062
-
1063
- for (simsimd_size_t i = 0; i != n; ++i) {
1064
- __m512d a_vec = _mm512_set1_pd(a[i]);
1065
- __m512d cb_j_vec = _mm512_setzero_pd();
1066
- __m512d b_vec, c_vec;
1067
- simsimd_size_t j = 0;
1068
-
1069
- simsimd_bilinear_f64_skylake_cycle:
1070
- if (j + 8 <= n) {
1071
- b_vec = _mm512_loadu_pd(b + j);
1072
- c_vec = _mm512_loadu_pd(c + i * n + j);
1073
- }
1074
- else {
1075
- b_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
1076
- c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
1077
- }
1078
- cb_j_vec = _mm512_fmadd_pd(b_vec, c_vec, cb_j_vec);
1079
- j += 8;
1080
- if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
1081
- sum_vec = _mm512_fmadd_pd(a_vec, cb_j_vec, sum_vec);
1082
- }
1083
-
1084
- *result = _mm512_reduce_add_pd(sum_vec);
1085
- }
1086
-
1087
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b,
1088
- simsimd_f64_t const *c, simsimd_size_t n,
1089
- simsimd_distance_t *result) {
1090
- simsimd_size_t const tail_length = n % 8;
1091
- simsimd_size_t const tail_start = n - tail_length;
1092
- __mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
1093
- __m512d sum_vec = _mm512_setzero_pd();
1094
-
1095
- for (simsimd_size_t i = 0; i != n; ++i) {
1096
- __m512d diff_i_vec = _mm512_set1_pd(a[i] - b[i]);
1097
- __m512d cdiff_j_vec = _mm512_setzero_pd();
1098
- __m512d a_j_vec, b_j_vec, diff_j_vec, c_vec;
1099
- simsimd_size_t j = 0;
1100
-
1101
- // The nested loop is cleaner to implement with a `goto` in this case:
1102
- simsimd_bilinear_f64_skylake_cycle:
1103
- if (j + 8 <= n) {
1104
- a_j_vec = _mm512_loadu_pd(a + j);
1105
- b_j_vec = _mm512_loadu_pd(b + j);
1106
- c_vec = _mm512_loadu_pd(c + i * n + j);
1107
- }
1108
- else {
1109
- a_j_vec = _mm512_maskz_loadu_pd(tail_mask, a + tail_start);
1110
- b_j_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
1111
- c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
1112
- }
1113
- diff_j_vec = _mm512_sub_pd(a_j_vec, b_j_vec);
1114
- cdiff_j_vec = _mm512_fmadd_pd(diff_j_vec, c_vec, cdiff_j_vec);
1115
- j += 8;
1116
- if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
1117
- sum_vec = _mm512_fmadd_pd(diff_i_vec, cdiff_j_vec, sum_vec);
1118
- }
1119
-
1120
- *result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_pd(sum_vec));
1121
- }
1122
-
1123
- SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const *a, simsimd_f64c_t const *b,
1124
- simsimd_f64c_t const *c, simsimd_size_t n,
1125
- simsimd_distance_t *results) {
1126
-
1127
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1128
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1129
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1130
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1131
- // one of the vectors.
1132
- __m512i const sign_flip_vec = _mm512_set_epi64( //
1133
- 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
1134
- 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
1135
- );
1136
-
1137
- // Default case for arbitrary size `n`
1138
- simsimd_size_t const tail_length = n % 4;
1139
- simsimd_size_t const tail_start = n - tail_length;
1140
- __mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
1141
- simsimd_f64_t sum_real = 0;
1142
- simsimd_f64_t sum_imag = 0;
1143
-
1144
- for (simsimd_size_t i = 0; i != n; ++i) {
1145
- simsimd_f64_t const a_i_real = a[i].real;
1146
- simsimd_f64_t const a_i_imag = a[i].imag;
1147
- __m512d cb_j_real_vec = _mm512_setzero_pd();
1148
- __m512d cb_j_imag_vec = _mm512_setzero_pd();
1149
- __m512d b_vec, c_vec;
1150
- simsimd_size_t j = 0;
1151
-
1152
- simsimd_bilinear_f64c_skylake_cycle:
1153
- if (j + 4 <= n) {
1154
- b_vec = _mm512_loadu_pd((simsimd_f64_t const *)(b + j));
1155
- c_vec = _mm512_loadu_pd((simsimd_f64_t const *)(c + i * n + j));
1156
- }
1157
- else {
1158
- b_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(b + tail_start));
1159
- c_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(c + i * n + tail_start));
1160
- }
1161
- // The real part of the product: b.real * c.real - b.imag * c.imag.
1162
- // The subtraction will be performed later with a sign flip.
1163
- cb_j_real_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_real_vec);
1164
- // The imaginary part of the product: b.real * c.imag + b.imag * c.real.
1165
- // Swap the imaginary and real parts of `c` before multiplication:
1166
- c_vec = _mm512_permute_pd(c_vec, 0x55); //? Same as 0b01010101.
1167
- cb_j_imag_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_imag_vec);
1168
- j += 4;
1169
- if (j < n) goto simsimd_bilinear_f64c_skylake_cycle;
1170
- // Flip the sign bit in every second scalar before accumulation:
1171
- cb_j_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(cb_j_real_vec), sign_flip_vec));
1172
- // Horizontal sums are the expensive part of the computation:
1173
- simsimd_f64_t const cb_j_real = _mm512_reduce_add_pd(cb_j_real_vec);
1174
- simsimd_f64_t const cb_j_imag = _mm512_reduce_add_pd(cb_j_imag_vec);
1175
- sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1176
- sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1177
- }
1178
-
1179
- // Reduce horizontal sums:
1180
- results[0] = sum_real;
1181
- results[1] = sum_imag;
1182
- }
1183
-
1184
- #pragma clang attribute pop
1185
- #pragma GCC pop_options
1186
- #endif // SIMSIMD_TARGET_SKYLAKE
1187
-
1188
- #if SIMSIMD_TARGET_GENOA
1189
- #pragma GCC push_options
1190
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1191
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1192
- apply_to = function)
1193
-
1194
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
1195
- simsimd_bf16_t const *c, simsimd_size_t n, simsimd_distance_t *result) {
1196
- simsimd_size_t const tail_length = n % 32;
1197
- simsimd_size_t const tail_start = n - tail_length;
1198
- __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
1199
- __m512 sum_vec = _mm512_setzero_ps();
1200
-
1201
- for (simsimd_size_t i = 0; i != n; ++i) {
1202
- __m512 a_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i));
1203
- __m512 cb_j_vec = _mm512_setzero_ps();
1204
- __m512i b_vec, c_vec;
1205
- simsimd_size_t j = 0;
1206
-
1207
- simsimd_bilinear_bf16_genoa_cycle:
1208
- if (j + 32 <= n) {
1209
- b_vec = _mm512_loadu_epi16(b + j);
1210
- c_vec = _mm512_loadu_epi16(c + i * n + j);
1211
- }
1212
- else {
1213
- b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
1214
- c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
1215
- }
1216
- cb_j_vec = _mm512_dpbf16_ps(cb_j_vec, (__m512bh)(b_vec), (__m512bh)(c_vec));
1217
- j += 32;
1218
- if (j < n) goto simsimd_bilinear_bf16_genoa_cycle;
1219
- sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
1220
- }
1221
-
1222
- *result = _mm512_reduce_add_ps(sum_vec);
1223
- }
1224
-
1225
- SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
1226
- simsimd_bf16_t const *c, simsimd_size_t n,
1227
- simsimd_distance_t *result) {
1228
- simsimd_size_t const tail_length = n % 32;
1229
- simsimd_size_t const tail_start = n - tail_length;
1230
- __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
1231
- __m512 sum_vec = _mm512_setzero_ps();
1232
-
1233
- for (simsimd_size_t i = 0; i != n; ++i) {
1234
- __m512 diff_i_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i));
1235
- __m512 cdiff_j_vec = _mm512_setzero_ps();
1236
- __m512i a_j_vec, b_j_vec, diff_j_vec, c_vec;
1237
- simsimd_size_t j = 0;
1238
-
1239
- // The nested loop is cleaner to implement with a `goto` in this case:
1240
- simsimd_mahalanobis_bf16_genoa_cycle:
1241
- if (j + 32 <= n) {
1242
- a_j_vec = _mm512_loadu_epi16(a + j);
1243
- b_j_vec = _mm512_loadu_epi16(b + j);
1244
- c_vec = _mm512_loadu_epi16(c + i * n + j);
1245
- }
1246
- else {
1247
- a_j_vec = _mm512_maskz_loadu_epi16(tail_mask, a + tail_start);
1248
- b_j_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
1249
- c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
1250
- }
1251
- diff_j_vec = _simsimd_substract_bf16x32_genoa(a_j_vec, b_j_vec);
1252
- cdiff_j_vec = _mm512_dpbf16_ps(cdiff_j_vec, (__m512bh)(diff_j_vec), (__m512bh)(c_vec));
1253
- j += 32;
1254
- if (j < n) goto simsimd_mahalanobis_bf16_genoa_cycle;
1255
- sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
1256
- }
1257
-
1258
- *result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ps(sum_vec));
1259
- }
1260
-
1261
- SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
1262
- simsimd_bf16c_t const *c, simsimd_size_t n,
1263
- simsimd_distance_t *results) {
1264
-
1265
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1266
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1267
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1268
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1269
- // one of the vectors.
1270
- __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1271
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1272
- 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1273
- 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1274
- 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1275
- 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1276
- );
1277
-
1278
- // Default case for arbitrary size `n`
1279
- simsimd_size_t const tail_length = n % 16;
1280
- simsimd_size_t const tail_start = n - tail_length;
1281
- __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
1282
- simsimd_f64_t sum_real = 0;
1283
- simsimd_f64_t sum_imag = 0;
1284
-
1285
- for (simsimd_size_t i = 0; i != n; ++i) {
1286
- simsimd_f32_t const a_i_real = a[i].real;
1287
- simsimd_f32_t const a_i_imag = a[i].imag;
1288
- __m512 cb_j_real_vec = _mm512_setzero_ps();
1289
- __m512 cb_j_imag_vec = _mm512_setzero_ps();
1290
- __m512i b_vec, c_vec;
1291
- simsimd_size_t j = 0;
1292
-
1293
- simsimd_bilinear_bf16c_skylake_cycle:
1294
- if (j + 16 <= n) {
1295
- b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
1296
- c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
1297
- }
1298
- else {
1299
- b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
1300
- c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
1301
- }
1302
- cb_j_real_vec = _mm512_dpbf16_ps( //
1303
- cb_j_real_vec, //
1304
- (__m512bh)(_mm512_xor_si512(c_vec, sign_flip_vec)), //
1305
- (__m512bh)b_vec);
1306
- cb_j_imag_vec = _mm512_dpbf16_ps( //
1307
- cb_j_imag_vec, //
1308
- (__m512bh)(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
1309
- (__m512bh)b_vec);
1310
- j += 16;
1311
- if (j < n) goto simsimd_bilinear_bf16c_skylake_cycle;
1312
- // Horizontal sums are the expensive part of the computation:
1313
- simsimd_f64_t const cb_j_real = _simsimd_reduce_f32x16_skylake(cb_j_real_vec);
1314
- simsimd_f64_t const cb_j_imag = _simsimd_reduce_f32x16_skylake(cb_j_imag_vec);
1315
- sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1316
- sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1317
- }
1318
-
1319
- // Reduce horizontal sums:
1320
- results[0] = sum_real;
1321
- results[1] = sum_imag;
1322
- }
1323
-
1324
- #pragma clang attribute pop
1325
- #pragma GCC pop_options
1326
- #endif // SIMSIMD_TARGET_GENOA
1327
-
1328
- #if SIMSIMD_TARGET_SAPPHIRE
1329
- #pragma GCC push_options
1330
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
1331
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
1332
- apply_to = function)
1333
-
1334
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire_under32unrolled(simsimd_f16_t const *a, simsimd_f16_t const *b,
1335
- simsimd_f16_t const *c, simsimd_size_t const n,
1336
- simsimd_distance_t *result) {
1337
- // The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
1338
- // until the very end of the computation.
1339
- __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1340
- __m512h const b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
1341
-
1342
- // Independently accumulate the partial sums into separate variables to avoid data-dependencies.
1343
- __m512h cb_j1 = _mm512_setzero_ph();
1344
- __m512h cb_j2 = _mm512_setzero_ph();
1345
- __m512h cb_j3 = _mm512_setzero_ph();
1346
- __m512h cb_j4 = _mm512_setzero_ph();
1347
-
1348
- // Unroll the loop to process 4x ZMM registers at a time.
1349
- simsimd_size_t i = 0;
1350
- for (; i + 4 <= n; i += 4) {
1351
- // If the code is compiled without native support for `_Float16`, we need a workaround
1352
- // to avoid implicit casts from out `simsimd_f16_t` to `_Float16`.
1353
- cb_j1 = _mm512_fmadd_ph(
1354
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
1355
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
1356
- cb_j2 = _mm512_fmadd_ph(
1357
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
1358
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
1359
- cb_j3 = _mm512_fmadd_ph(
1360
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
1361
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
1362
- cb_j4 = _mm512_fmadd_ph(
1363
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
1364
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
1365
- }
1366
-
1367
- // Handle the tail of the loop:
1368
- if (i + 0 < n)
1369
- cb_j1 = _mm512_fmadd_ph(
1370
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
1371
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
1372
- if (i + 1 < n)
1373
- cb_j2 = _mm512_fmadd_ph(
1374
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
1375
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
1376
- if (i + 2 < n)
1377
- cb_j3 = _mm512_fmadd_ph(
1378
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
1379
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
1380
- if (i + 3 < n)
1381
- cb_j4 = _mm512_fmadd_ph(
1382
- _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
1383
- _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
1384
-
1385
- // Combine cb_j sums
1386
- __m512h sum_vec = _mm512_add_ph( //
1387
- _mm512_add_ph(cb_j1, cb_j2), //
1388
- _mm512_add_ph(cb_j3, cb_j4));
1389
- *result = _mm512_reduce_add_ph(sum_vec);
1390
- }
1391
-
1392
- SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
1393
- simsimd_f16_t const *c, simsimd_size_t n,
1394
- simsimd_distance_t *result) {
1395
-
1396
- // On modern x86 CPUs we have enough register space to load fairly large matrices with up to 32 cells
1397
- // per row and 32 rows at a time, still keeping enough register space for temporaries.
1398
- if (n <= 32) {
1399
- simsimd_bilinear_f16_sapphire_under32unrolled(a, b, c, n, result);
1400
- return;
1401
- }
1402
-
1403
- simsimd_size_t const tail_length = n % 32;
1404
- simsimd_size_t const tail_start = n - tail_length;
1405
- __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
1406
- __m512h sum_vec = _mm512_setzero_ph();
1407
-
1408
- for (simsimd_size_t i = 0; i != n; ++i) {
1409
- __m512h a_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
1410
- __m512h cb_j_vec = _mm512_setzero_ph();
1411
- __m512i b_vec, c_vec;
1412
- simsimd_size_t j = 0;
1413
-
1414
- simsimd_bilinear_f16_sapphire_cycle:
1415
- if (j + 32 <= n) {
1416
- b_vec = _mm512_loadu_epi16(b + j);
1417
- c_vec = _mm512_loadu_epi16(c + i * n + j);
1418
- }
1419
- else {
1420
- b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
1421
- c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
1422
- }
1423
- cb_j_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(c_vec), cb_j_vec);
1424
- j += 32;
1425
- if (j < n) goto simsimd_bilinear_f16_sapphire_cycle;
1426
- sum_vec = _mm512_fmadd_ph(a_vec, cb_j_vec, sum_vec);
1427
- }
1428
-
1429
- *result = _mm512_reduce_add_ph(sum_vec);
1430
- }
1431
-
1432
- SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
1433
- simsimd_f16_t const *c, simsimd_size_t n,
1434
- simsimd_distance_t *result) {
1435
- simsimd_size_t const tail_length = n % 32;
1436
- simsimd_size_t const tail_start = n - tail_length;
1437
- __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
1438
- __m512h sum_vec = _mm512_setzero_ph();
1439
-
1440
- for (simsimd_size_t i = 0; i != n; ++i) {
1441
- __m512h a_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
1442
- __m512h b_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(b + i)));
1443
- __m512h diff_i_vec = _mm512_sub_ph(a_i_vec, b_i_vec);
1444
- __m512h cdiff_j_vec = _mm512_setzero_ph();
1445
- __m512h diff_j_vec;
1446
- __m512i a_j_vec, b_j_vec, c_vec;
1447
- simsimd_size_t j = 0;
1448
-
1449
- // The nested loop is cleaner to implement with a `goto` in this case:
1450
- simsimd_mahalanobis_f16_sapphire_cycle:
1451
- if (j + 32 <= n) {
1452
- a_j_vec = _mm512_loadu_epi16(a + j);
1453
- b_j_vec = _mm512_loadu_epi16(b + j);
1454
- c_vec = _mm512_loadu_epi16(c + i * n + j);
1455
- }
1456
- else {
1457
- a_j_vec = _mm512_maskz_loadu_epi16(tail_mask, a + tail_start);
1458
- b_j_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
1459
- c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
1460
- }
1461
- diff_j_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_j_vec), _mm512_castsi512_ph(b_j_vec));
1462
- cdiff_j_vec = _mm512_fmadd_ph(diff_j_vec, _mm512_castsi512_ph(c_vec), cdiff_j_vec);
1463
- j += 32;
1464
- if (j < n) goto simsimd_mahalanobis_f16_sapphire_cycle;
1465
- sum_vec = _mm512_fmadd_ph(diff_i_vec, cdiff_j_vec, sum_vec);
1466
- }
1467
-
1468
- *result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ph(sum_vec));
1469
- }
1470
-
1471
- SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
1472
- simsimd_f16c_t const *c, simsimd_size_t n,
1473
- simsimd_distance_t *results) {
1474
-
1475
- // We take into account, that FMS is the same as FMA with a negative multiplier.
1476
- // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1477
- // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1478
- // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1479
- // one of the vectors.
1480
- __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1481
- __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1482
- 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1483
- 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1484
- 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1485
- 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1486
- );
1487
-
1488
- // Default case for arbitrary size `n`
1489
- simsimd_size_t const tail_length = n % 16;
1490
- simsimd_size_t const tail_start = n - tail_length;
1491
- __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
1492
- simsimd_f32_t sum_real = 0;
1493
- simsimd_f32_t sum_imag = 0;
1494
-
1495
- for (simsimd_size_t i = 0; i != n; ++i) {
1496
- simsimd_f32_t const a_i_real = a[i].real;
1497
- simsimd_f32_t const a_i_imag = a[i].imag;
1498
- __m512h cb_j_real_vec = _mm512_setzero_ph();
1499
- __m512h cb_j_imag_vec = _mm512_setzero_ph();
1500
- __m512i b_vec, c_vec;
1501
- simsimd_size_t j = 0;
1502
-
1503
- simsimd_bilinear_f16c_skylake_cycle:
1504
- if (j + 16 <= n) {
1505
- b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
1506
- c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
1507
- }
1508
- else {
1509
- b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
1510
- c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
1511
- }
1512
- cb_j_real_vec = _mm512_fmadd_ph( //
1513
- _mm512_castsi512_ph(_mm512_xor_si512(c_vec, sign_flip_vec)), //
1514
- _mm512_castsi512_ph(b_vec), cb_j_real_vec);
1515
- cb_j_imag_vec = _mm512_fmadd_ph( //
1516
- _mm512_castsi512_ph(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
1517
- _mm512_castsi512_ph(b_vec), cb_j_imag_vec);
1518
- j += 16;
1519
- if (j < n) goto simsimd_bilinear_f16c_skylake_cycle;
1520
- // Horizontal sums are the expensive part of the computation:
1521
- simsimd_f32_t const cb_j_real = _mm512_reduce_add_ph(cb_j_real_vec);
1522
- simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ph(cb_j_imag_vec);
1523
- sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1524
- sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1525
- }
1526
-
1527
- // Reduce horizontal sums:
1528
- results[0] = sum_real;
1529
- results[1] = sum_imag;
1530
- }
1531
-
1532
- #pragma clang attribute pop
1533
- #pragma GCC pop_options
1534
- #endif // SIMSIMD_TARGET_SAPPHIRE
1535
- #endif // _SIMSIMD_TARGET_X86
1536
-
1537
- #ifdef __cplusplus
1538
- }
1539
- #endif
1540
-
1541
- #endif