usearch 2.21.4 → 2.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (604) hide show
  1. package/README.md +65 -69
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +8 -8
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +124 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +224 -48
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/README.md +1 -1
  16. package/javascript/dist/cjs/usearch.d.ts +8 -3
  17. package/javascript/dist/cjs/usearch.js +12 -4
  18. package/javascript/dist/esm/usearch.d.ts +8 -3
  19. package/javascript/dist/esm/usearch.js +12 -4
  20. package/javascript/lib.cpp +17 -0
  21. package/javascript/usearch.test.js +117 -94
  22. package/javascript/usearch.ts +14 -6
  23. package/numkong/.clang-tidy +30 -0
  24. package/{simsimd → numkong}/.cmake-format.py +18 -0
  25. package/numkong/.editorconfig +12 -0
  26. package/numkong/.git-blame-ignore-revs +123 -0
  27. package/numkong/.mailmap +15 -0
  28. package/numkong/.spi.yml +11 -0
  29. package/numkong/.swift-format +14 -0
  30. package/numkong/CITATION.cff +81 -0
  31. package/numkong/CONTRIBUTING.md +500 -0
  32. package/numkong/Cargo.toml +54 -0
  33. package/numkong/MANIFEST.in +8 -0
  34. package/numkong/Package.swift +84 -0
  35. package/numkong/README.md +569 -0
  36. package/numkong/bench/README.md +217 -0
  37. package/numkong/bench/bench-browser.html +323 -0
  38. package/numkong/bench/bench.cpp +344 -0
  39. package/numkong/bench/bench.hpp +816 -0
  40. package/numkong/bench/bench.js +148 -0
  41. package/numkong/bench/bench.mjs +616 -0
  42. package/numkong/bench/bench_attention.cpp +113 -0
  43. package/numkong/bench/bench_cast.cpp +264 -0
  44. package/numkong/bench/bench_cross_amx.cpp +62 -0
  45. package/numkong/bench/bench_cross_arm.cpp +245 -0
  46. package/numkong/bench/bench_cross_blas.cpp +309 -0
  47. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  48. package/numkong/bench/bench_cross_power.cpp +84 -0
  49. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  50. package/numkong/bench/bench_cross_serial.cpp +151 -0
  51. package/numkong/bench/bench_cross_sme.cpp +163 -0
  52. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  53. package/numkong/bench/bench_cross_x86.cpp +455 -0
  54. package/numkong/bench/bench_curved.cpp +210 -0
  55. package/numkong/bench/bench_dot.cpp +315 -0
  56. package/numkong/bench/bench_each.cpp +503 -0
  57. package/numkong/bench/bench_geospatial.cpp +112 -0
  58. package/numkong/bench/bench_maxsim.cpp +154 -0
  59. package/numkong/bench/bench_mesh.cpp +148 -0
  60. package/numkong/bench/bench_probability.cpp +60 -0
  61. package/numkong/bench/bench_reduce.cpp +371 -0
  62. package/numkong/bench/bench_set.cpp +73 -0
  63. package/numkong/bench/bench_sparse.cpp +198 -0
  64. package/numkong/bench/bench_spatial.cpp +347 -0
  65. package/numkong/bench/bench_trigonometry.cpp +137 -0
  66. package/numkong/bench/swift/Bench.swift +506 -0
  67. package/numkong/binding.gyp +169 -0
  68. package/numkong/build.rs +484 -0
  69. package/numkong/c/dispatch.h +512 -0
  70. package/numkong/c/dispatch_bf16.c +447 -0
  71. package/numkong/c/dispatch_bf16c.c +52 -0
  72. package/numkong/c/dispatch_e2m3.c +296 -0
  73. package/numkong/c/dispatch_e3m2.c +284 -0
  74. package/numkong/c/dispatch_e4m3.c +318 -0
  75. package/numkong/c/dispatch_e5m2.c +316 -0
  76. package/numkong/c/dispatch_f16.c +452 -0
  77. package/numkong/c/dispatch_f16c.c +58 -0
  78. package/numkong/c/dispatch_f32.c +434 -0
  79. package/numkong/c/dispatch_f32c.c +99 -0
  80. package/numkong/c/dispatch_f64.c +348 -0
  81. package/numkong/c/dispatch_f64c.c +98 -0
  82. package/numkong/c/dispatch_i16.c +96 -0
  83. package/numkong/c/dispatch_i32.c +89 -0
  84. package/numkong/c/dispatch_i4.c +153 -0
  85. package/numkong/c/dispatch_i64.c +86 -0
  86. package/numkong/c/dispatch_i8.c +348 -0
  87. package/numkong/c/dispatch_other.c +348 -0
  88. package/numkong/c/dispatch_u1.c +193 -0
  89. package/numkong/c/dispatch_u16.c +124 -0
  90. package/numkong/c/dispatch_u32.c +118 -0
  91. package/numkong/c/dispatch_u4.c +153 -0
  92. package/numkong/c/dispatch_u64.c +102 -0
  93. package/numkong/c/dispatch_u8.c +364 -0
  94. package/numkong/c/numkong.c +940 -0
  95. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  96. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  97. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_math_lib.cmake +46 -0
  99. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  100. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  101. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  102. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  103. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  104. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  105. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  106. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  107. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  108. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  109. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  110. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  111. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  112. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  113. package/numkong/include/README.md +643 -0
  114. package/numkong/include/module.modulemap +129 -0
  115. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  116. package/numkong/include/numkong/attention/sme.h +2075 -0
  117. package/numkong/include/numkong/attention.h +49 -0
  118. package/numkong/include/numkong/capabilities.h +930 -0
  119. package/numkong/include/numkong/cast/README.md +262 -0
  120. package/numkong/include/numkong/cast/diamond.h +64 -0
  121. package/numkong/include/numkong/cast/haswell.h +823 -0
  122. package/numkong/include/numkong/cast/icelake.h +475 -0
  123. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  124. package/numkong/include/numkong/cast/neon.h +1174 -0
  125. package/numkong/include/numkong/cast/powervsx.h +449 -0
  126. package/numkong/include/numkong/cast/rvv.h +970 -0
  127. package/numkong/include/numkong/cast/sapphire.h +262 -0
  128. package/numkong/include/numkong/cast/serial.h +2336 -0
  129. package/numkong/include/numkong/cast/skylake.h +915 -0
  130. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  131. package/numkong/include/numkong/cast.h +254 -0
  132. package/numkong/include/numkong/cast.hpp +44 -0
  133. package/numkong/include/numkong/curved/README.md +223 -0
  134. package/numkong/include/numkong/curved/genoa.h +182 -0
  135. package/numkong/include/numkong/curved/haswell.h +276 -0
  136. package/numkong/include/numkong/curved/neon.h +329 -0
  137. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  138. package/numkong/include/numkong/curved/rvv.h +305 -0
  139. package/numkong/include/numkong/curved/serial.h +207 -0
  140. package/numkong/include/numkong/curved/skylake.h +457 -0
  141. package/numkong/include/numkong/curved/smef64.h +530 -0
  142. package/numkong/include/numkong/curved.h +513 -0
  143. package/numkong/include/numkong/curved.hpp +144 -0
  144. package/numkong/include/numkong/dot/README.md +442 -0
  145. package/numkong/include/numkong/dot/alder.h +563 -0
  146. package/numkong/include/numkong/dot/diamond.h +158 -0
  147. package/numkong/include/numkong/dot/genoa.h +289 -0
  148. package/numkong/include/numkong/dot/haswell.h +1723 -0
  149. package/numkong/include/numkong/dot/icelake.h +997 -0
  150. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  151. package/numkong/include/numkong/dot/neon.h +869 -0
  152. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  153. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  154. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  155. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  156. package/numkong/include/numkong/dot/powervsx.h +752 -0
  157. package/numkong/include/numkong/dot/rvv.h +722 -0
  158. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  159. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  160. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  161. package/numkong/include/numkong/dot/sapphire.h +141 -0
  162. package/numkong/include/numkong/dot/serial.h +874 -0
  163. package/numkong/include/numkong/dot/sierra.h +405 -0
  164. package/numkong/include/numkong/dot/skylake.h +1116 -0
  165. package/numkong/include/numkong/dot/sve.h +420 -0
  166. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  167. package/numkong/include/numkong/dot/svehalf.h +168 -0
  168. package/numkong/include/numkong/dot/svesdot.h +90 -0
  169. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  170. package/numkong/include/numkong/dot.h +1136 -0
  171. package/numkong/include/numkong/dot.hpp +94 -0
  172. package/numkong/include/numkong/dots/README.md +496 -0
  173. package/numkong/include/numkong/dots/alder.h +117 -0
  174. package/numkong/include/numkong/dots/diamond.h +86 -0
  175. package/numkong/include/numkong/dots/genoa.h +100 -0
  176. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  177. package/numkong/include/numkong/dots/haswell.h +310 -0
  178. package/numkong/include/numkong/dots/icelake.h +180 -0
  179. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  180. package/numkong/include/numkong/dots/neon.h +123 -0
  181. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  182. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  183. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  184. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  185. package/numkong/include/numkong/dots/powervsx.h +194 -0
  186. package/numkong/include/numkong/dots/rvv.h +2593 -0
  187. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  188. package/numkong/include/numkong/dots/serial.h +2867 -0
  189. package/numkong/include/numkong/dots/sierra.h +100 -0
  190. package/numkong/include/numkong/dots/skylake.h +207 -0
  191. package/numkong/include/numkong/dots/sme.h +5041 -0
  192. package/numkong/include/numkong/dots/smebi32.h +477 -0
  193. package/numkong/include/numkong/dots/smef64.h +1328 -0
  194. package/numkong/include/numkong/dots/smehalf.h +47 -0
  195. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  196. package/numkong/include/numkong/dots.h +3110 -0
  197. package/numkong/include/numkong/dots.hpp +641 -0
  198. package/numkong/include/numkong/each/README.md +465 -0
  199. package/numkong/include/numkong/each/haswell.h +1658 -0
  200. package/numkong/include/numkong/each/icelake.h +272 -0
  201. package/numkong/include/numkong/each/neon.h +1138 -0
  202. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  203. package/numkong/include/numkong/each/neonhalf.h +318 -0
  204. package/numkong/include/numkong/each/rvv.h +1121 -0
  205. package/numkong/include/numkong/each/sapphire.h +343 -0
  206. package/numkong/include/numkong/each/serial.h +282 -0
  207. package/numkong/include/numkong/each/skylake.h +1562 -0
  208. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  209. package/numkong/include/numkong/each.h +2232 -0
  210. package/numkong/include/numkong/each.hpp +434 -0
  211. package/numkong/include/numkong/geospatial/README.md +147 -0
  212. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  213. package/numkong/include/numkong/geospatial/neon.h +615 -0
  214. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  215. package/numkong/include/numkong/geospatial/serial.h +309 -0
  216. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  217. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  218. package/numkong/include/numkong/geospatial.h +453 -0
  219. package/numkong/include/numkong/geospatial.hpp +235 -0
  220. package/numkong/include/numkong/matrix.hpp +336 -0
  221. package/numkong/include/numkong/maxsim/README.md +187 -0
  222. package/numkong/include/numkong/maxsim/alder.h +511 -0
  223. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  224. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  225. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  226. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  227. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  228. package/numkong/include/numkong/maxsim/serial.h +490 -0
  229. package/numkong/include/numkong/maxsim/sme.h +993 -0
  230. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  231. package/numkong/include/numkong/maxsim.h +571 -0
  232. package/numkong/include/numkong/maxsim.hpp +133 -0
  233. package/numkong/include/numkong/mesh/README.md +219 -0
  234. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  235. package/numkong/include/numkong/mesh/neon.h +1821 -0
  236. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  237. package/numkong/include/numkong/mesh/rvv.h +936 -0
  238. package/numkong/include/numkong/mesh/serial.h +721 -0
  239. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  240. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  241. package/numkong/include/numkong/mesh.h +666 -0
  242. package/numkong/include/numkong/mesh.hpp +701 -0
  243. package/numkong/include/numkong/numkong.h +78 -0
  244. package/numkong/include/numkong/numkong.hpp +58 -0
  245. package/numkong/include/numkong/probability/README.md +177 -0
  246. package/numkong/include/numkong/probability/haswell.h +267 -0
  247. package/numkong/include/numkong/probability/neon.h +255 -0
  248. package/numkong/include/numkong/probability/rvv.h +414 -0
  249. package/numkong/include/numkong/probability/serial.h +181 -0
  250. package/numkong/include/numkong/probability/skylake.h +324 -0
  251. package/numkong/include/numkong/probability.h +380 -0
  252. package/numkong/include/numkong/probability.hpp +120 -0
  253. package/numkong/include/numkong/random.h +50 -0
  254. package/numkong/include/numkong/random.hpp +285 -0
  255. package/numkong/include/numkong/reduce/README.md +552 -0
  256. package/numkong/include/numkong/reduce/alder.h +636 -0
  257. package/numkong/include/numkong/reduce/genoa.h +201 -0
  258. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  259. package/numkong/include/numkong/reduce/icelake.h +549 -0
  260. package/numkong/include/numkong/reduce/neon.h +3980 -0
  261. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  262. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  263. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  264. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  265. package/numkong/include/numkong/reduce/serial.h +753 -0
  266. package/numkong/include/numkong/reduce/sierra.h +338 -0
  267. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  268. package/numkong/include/numkong/reduce/sve.h +52 -0
  269. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  270. package/numkong/include/numkong/reduce.h +1582 -0
  271. package/numkong/include/numkong/reduce.hpp +778 -0
  272. package/numkong/include/numkong/scalar/README.md +89 -0
  273. package/numkong/include/numkong/scalar/haswell.h +122 -0
  274. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  275. package/numkong/include/numkong/scalar/neon.h +122 -0
  276. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  277. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  278. package/numkong/include/numkong/scalar/rvv.h +211 -0
  279. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  280. package/numkong/include/numkong/scalar/serial.h +332 -0
  281. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  282. package/numkong/include/numkong/scalar.h +696 -0
  283. package/numkong/include/numkong/set/README.md +179 -0
  284. package/numkong/include/numkong/set/haswell.h +334 -0
  285. package/numkong/include/numkong/set/icelake.h +485 -0
  286. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  287. package/numkong/include/numkong/set/neon.h +363 -0
  288. package/numkong/include/numkong/set/powervsx.h +326 -0
  289. package/numkong/include/numkong/set/rvv.h +226 -0
  290. package/numkong/include/numkong/set/rvvbb.h +117 -0
  291. package/numkong/include/numkong/set/serial.h +174 -0
  292. package/numkong/include/numkong/set/sve.h +187 -0
  293. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  294. package/numkong/include/numkong/set.h +471 -0
  295. package/numkong/include/numkong/set.hpp +114 -0
  296. package/numkong/include/numkong/sets/README.md +149 -0
  297. package/numkong/include/numkong/sets/haswell.h +63 -0
  298. package/numkong/include/numkong/sets/icelake.h +66 -0
  299. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  300. package/numkong/include/numkong/sets/neon.h +61 -0
  301. package/numkong/include/numkong/sets/powervsx.h +65 -0
  302. package/numkong/include/numkong/sets/serial.h +43 -0
  303. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  304. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  305. package/numkong/include/numkong/sets.h +382 -0
  306. package/numkong/include/numkong/sparse/README.md +156 -0
  307. package/numkong/include/numkong/sparse/icelake.h +463 -0
  308. package/numkong/include/numkong/sparse/neon.h +288 -0
  309. package/numkong/include/numkong/sparse/serial.h +117 -0
  310. package/numkong/include/numkong/sparse/sve2.h +504 -0
  311. package/numkong/include/numkong/sparse/turin.h +322 -0
  312. package/numkong/include/numkong/sparse.h +363 -0
  313. package/numkong/include/numkong/sparse.hpp +113 -0
  314. package/numkong/include/numkong/spatial/README.md +439 -0
  315. package/numkong/include/numkong/spatial/alder.h +604 -0
  316. package/numkong/include/numkong/spatial/diamond.h +240 -0
  317. package/numkong/include/numkong/spatial/genoa.h +222 -0
  318. package/numkong/include/numkong/spatial/haswell.h +979 -0
  319. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  320. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  321. package/numkong/include/numkong/spatial/neon.h +846 -0
  322. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  323. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  324. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  325. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  326. package/numkong/include/numkong/spatial/rvv.h +991 -0
  327. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  328. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  329. package/numkong/include/numkong/spatial/serial.h +347 -0
  330. package/numkong/include/numkong/spatial/sierra.h +516 -0
  331. package/numkong/include/numkong/spatial/skylake.h +605 -0
  332. package/numkong/include/numkong/spatial/sve.h +246 -0
  333. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  334. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  335. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  336. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  337. package/numkong/include/numkong/spatial.h +1681 -0
  338. package/numkong/include/numkong/spatial.hpp +183 -0
  339. package/numkong/include/numkong/spatials/README.md +580 -0
  340. package/numkong/include/numkong/spatials/alder.h +94 -0
  341. package/numkong/include/numkong/spatials/diamond.h +82 -0
  342. package/numkong/include/numkong/spatials/genoa.h +94 -0
  343. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  344. package/numkong/include/numkong/spatials/haswell.h +219 -0
  345. package/numkong/include/numkong/spatials/icelake.h +113 -0
  346. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  347. package/numkong/include/numkong/spatials/neon.h +109 -0
  348. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  349. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  350. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  351. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  352. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  353. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  354. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  355. package/numkong/include/numkong/spatials/serial.h +248 -0
  356. package/numkong/include/numkong/spatials/sierra.h +96 -0
  357. package/numkong/include/numkong/spatials/skylake.h +184 -0
  358. package/numkong/include/numkong/spatials/sme.h +1927 -0
  359. package/numkong/include/numkong/spatials/smef64.h +483 -0
  360. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  361. package/numkong/include/numkong/spatials.h +3374 -0
  362. package/numkong/include/numkong/spatials.hpp +509 -0
  363. package/numkong/include/numkong/tensor.hpp +1981 -0
  364. package/numkong/include/numkong/trigonometry/README.md +184 -0
  365. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  366. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  367. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  368. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  369. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  370. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  371. package/numkong/include/numkong/trigonometry.h +467 -0
  372. package/numkong/include/numkong/trigonometry.hpp +166 -0
  373. package/numkong/include/numkong/types.h +1695 -0
  374. package/numkong/include/numkong/types.hpp +6027 -0
  375. package/numkong/include/numkong/vector.hpp +766 -0
  376. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  377. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  378. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  380. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  381. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  382. package/numkong/javascript/README.md +254 -0
  383. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  384. package/numkong/javascript/numkong-browser.ts +40 -0
  385. package/numkong/javascript/numkong-wasm.ts +750 -0
  386. package/numkong/javascript/numkong.c +712 -0
  387. package/numkong/javascript/numkong.ts +613 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  389. package/numkong/javascript/tsconfig-cjs.json +13 -0
  390. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  391. package/numkong/javascript/types.ts +784 -0
  392. package/numkong/numkong.gypi +104 -0
  393. package/numkong/package.json +108 -0
  394. package/numkong/probes/arm_neon.c +8 -0
  395. package/numkong/probes/arm_neon_bfdot.c +9 -0
  396. package/numkong/probes/arm_neon_fhm.c +9 -0
  397. package/numkong/probes/arm_neon_half.c +8 -0
  398. package/numkong/probes/arm_neon_sdot.c +9 -0
  399. package/numkong/probes/arm_neonfp8.c +9 -0
  400. package/numkong/probes/arm_sme.c +16 -0
  401. package/numkong/probes/arm_sme2.c +16 -0
  402. package/numkong/probes/arm_sme2p1.c +16 -0
  403. package/numkong/probes/arm_sme_bf16.c +16 -0
  404. package/numkong/probes/arm_sme_bi32.c +16 -0
  405. package/numkong/probes/arm_sme_f64.c +16 -0
  406. package/numkong/probes/arm_sme_fa64.c +14 -0
  407. package/numkong/probes/arm_sme_half.c +16 -0
  408. package/numkong/probes/arm_sme_lut2.c +15 -0
  409. package/numkong/probes/arm_sve.c +18 -0
  410. package/numkong/probes/arm_sve2.c +20 -0
  411. package/numkong/probes/arm_sve2p1.c +18 -0
  412. package/numkong/probes/arm_sve_bfdot.c +20 -0
  413. package/numkong/probes/arm_sve_half.c +18 -0
  414. package/numkong/probes/arm_sve_sdot.c +21 -0
  415. package/numkong/probes/loongarch_lasx.c +12 -0
  416. package/numkong/probes/power_vsx.c +12 -0
  417. package/numkong/probes/probe.js +127 -0
  418. package/numkong/probes/riscv_rvv.c +14 -0
  419. package/numkong/probes/riscv_rvv_bb.c +15 -0
  420. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  421. package/numkong/probes/riscv_rvv_half.c +14 -0
  422. package/numkong/probes/wasm_v128relaxed.c +11 -0
  423. package/numkong/probes/x86_alder.c +17 -0
  424. package/numkong/probes/x86_diamond.c +17 -0
  425. package/numkong/probes/x86_genoa.c +17 -0
  426. package/numkong/probes/x86_graniteamx.c +19 -0
  427. package/numkong/probes/x86_haswell.c +11 -0
  428. package/numkong/probes/x86_icelake.c +17 -0
  429. package/numkong/probes/x86_sapphire.c +16 -0
  430. package/numkong/probes/x86_sapphireamx.c +18 -0
  431. package/numkong/probes/x86_sierra.c +17 -0
  432. package/numkong/probes/x86_skylake.c +15 -0
  433. package/numkong/probes/x86_turin.c +17 -0
  434. package/numkong/rust/README.md +698 -0
  435. package/numkong/rust/capabilities.rs +96 -0
  436. package/numkong/rust/cast.rs +314 -0
  437. package/numkong/rust/curved.rs +391 -0
  438. package/numkong/rust/each.rs +2721 -0
  439. package/numkong/rust/geospatial.rs +343 -0
  440. package/numkong/rust/matrix.rs +5424 -0
  441. package/numkong/rust/maxsim.rs +540 -0
  442. package/numkong/rust/mesh.rs +837 -0
  443. package/numkong/rust/numkong.rs +409 -0
  444. package/numkong/rust/probability.rs +272 -0
  445. package/numkong/rust/reduce.rs +1249 -0
  446. package/numkong/rust/set.rs +194 -0
  447. package/numkong/rust/sparse.rs +513 -0
  448. package/numkong/rust/spatial.rs +1815 -0
  449. package/numkong/rust/tensor.rs +7565 -0
  450. package/numkong/rust/types.rs +3666 -0
  451. package/numkong/rust/vector.rs +1899 -0
  452. package/numkong/rustfmt.toml +2 -0
  453. package/numkong/swift/Matrix.swift +1388 -0
  454. package/numkong/swift/MaxSim.swift +151 -0
  455. package/numkong/swift/NumKong.swift +283 -0
  456. package/numkong/swift/Numerics.swift +329 -0
  457. package/numkong/swift/README.md +433 -0
  458. package/numkong/swift/Scalar.swift +741 -0
  459. package/numkong/swift/Tensor.swift +209 -0
  460. package/numkong/test/README.md +315 -0
  461. package/numkong/test/playwright.config.ts +16 -0
  462. package/numkong/test/swift/Test.swift +534 -0
  463. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  464. package/numkong/test/test-browser-bundle.html +188 -0
  465. package/numkong/test/test-browser-runner.mjs +43 -0
  466. package/numkong/test/test-browser.html +305 -0
  467. package/numkong/test/test-dtypes.mjs +453 -0
  468. package/numkong/test/test-wasm.mjs +267 -0
  469. package/numkong/test/test.cpp +442 -0
  470. package/numkong/test/test.hpp +580 -0
  471. package/numkong/test/test.mjs +363 -0
  472. package/numkong/test/test_base.py +1041 -0
  473. package/numkong/test/test_cast.cpp +242 -0
  474. package/numkong/test/test_cross.hpp +551 -0
  475. package/numkong/test/test_cross.py +467 -0
  476. package/numkong/test/test_cross_amx.cpp +50 -0
  477. package/numkong/test/test_cross_arm.cpp +225 -0
  478. package/numkong/test/test_cross_blas.cpp +354 -0
  479. package/numkong/test/test_cross_loongarch.cpp +90 -0
  480. package/numkong/test/test_cross_power.cpp +73 -0
  481. package/numkong/test/test_cross_rvv.cpp +109 -0
  482. package/numkong/test/test_cross_serial.cpp +181 -0
  483. package/numkong/test/test_cross_sme.cpp +148 -0
  484. package/numkong/test/test_cross_wasm.cpp +141 -0
  485. package/numkong/test/test_cross_x86.cpp +442 -0
  486. package/numkong/test/test_curved.cpp +182 -0
  487. package/numkong/test/test_curved.py +207 -0
  488. package/numkong/test/test_dot.cpp +353 -0
  489. package/numkong/test/test_dot.py +296 -0
  490. package/numkong/test/test_each.cpp +437 -0
  491. package/numkong/test/test_each.py +756 -0
  492. package/numkong/test/test_geospatial.cpp +155 -0
  493. package/numkong/test/test_geospatial.py +241 -0
  494. package/numkong/test/test_maxsim.cpp +138 -0
  495. package/numkong/test/test_maxsim.py +209 -0
  496. package/numkong/test/test_mesh.cpp +203 -0
  497. package/numkong/test/test_mesh.py +394 -0
  498. package/numkong/test/test_probability.cpp +136 -0
  499. package/numkong/test/test_probability.py +166 -0
  500. package/numkong/test/test_reduce.cpp +563 -0
  501. package/numkong/test/test_reduce.py +781 -0
  502. package/numkong/test/test_set.cpp +151 -0
  503. package/numkong/test/test_set.py +120 -0
  504. package/numkong/test/test_similarities.py +579 -0
  505. package/numkong/test/test_sparse.cpp +121 -0
  506. package/numkong/test/test_sparse.py +118 -0
  507. package/numkong/test/test_spatial.cpp +518 -0
  508. package/numkong/test/test_spatial.py +269 -0
  509. package/numkong/test/test_tensor.cpp +884 -0
  510. package/numkong/test/test_tensor.py +1417 -0
  511. package/numkong/test/test_trigonometry.cpp +156 -0
  512. package/numkong/test/test_trigonometry.py +180 -0
  513. package/package.json +7 -8
  514. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  515. package/prebuilds/linux-arm64/usearch.node +0 -0
  516. package/prebuilds/linux-x64/usearch.node +0 -0
  517. package/fp16/CMakeLists.txt +0 -176
  518. package/fp16/LICENSE +0 -11
  519. package/fp16/README.md +0 -20
  520. package/fp16/bench/alt-element.cc +0 -89
  521. package/fp16/bench/from-alt-array.cc +0 -175
  522. package/fp16/bench/from-ieee-array.cc +0 -399
  523. package/fp16/bench/ieee-element.cc +0 -272
  524. package/fp16/bench/to-alt-array.cc +0 -91
  525. package/fp16/bench/to-ieee-array.cc +0 -303
  526. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  527. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  528. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  529. package/fp16/configure.py +0 -65
  530. package/fp16/confu.yaml +0 -8
  531. package/fp16/include/fp16/__init__.py +0 -0
  532. package/fp16/include/fp16/avx.py +0 -50
  533. package/fp16/include/fp16/avx2.py +0 -53
  534. package/fp16/include/fp16/bitcasts.h +0 -92
  535. package/fp16/include/fp16/fp16.h +0 -451
  536. package/fp16/include/fp16/psimd.h +0 -131
  537. package/fp16/include/fp16.h +0 -11
  538. package/fp16/jni/Android.mk +0 -6
  539. package/fp16/jni/Application.mk +0 -4
  540. package/fp16/test/alt-from-fp32-value.cc +0 -498
  541. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  542. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  543. package/fp16/test/alt-to-fp32-value.cc +0 -348
  544. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  545. package/fp16/test/bitcasts.cc +0 -56
  546. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  547. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  548. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  549. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  550. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  551. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  552. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  553. package/fp16/test/peachpy/stubs.py +0 -38
  554. package/fp16/test/tables.cc +0 -530
  555. package/fp16/test/tables.h +0 -13
  556. package/fp16/third-party/THHalf.h +0 -92
  557. package/fp16/third-party/eigen-half.h +0 -170
  558. package/fp16/third-party/float16-compressor.h +0 -80
  559. package/fp16/third-party/half.hpp +0 -2916
  560. package/fp16/third-party/npy-halffloat.h +0 -328
  561. package/simsimd/.git-blame-ignore-revs +0 -2
  562. package/simsimd/CONTRIBUTING.md +0 -321
  563. package/simsimd/Cargo.toml +0 -49
  564. package/simsimd/MANIFEST.in +0 -4
  565. package/simsimd/Package.swift +0 -28
  566. package/simsimd/README.md +0 -1493
  567. package/simsimd/binding.gyp +0 -35
  568. package/simsimd/build.rs +0 -70
  569. package/simsimd/c/lib.c +0 -374
  570. package/simsimd/include/module.modulemap +0 -4
  571. package/simsimd/include/simsimd/binary.h +0 -486
  572. package/simsimd/include/simsimd/curved.h +0 -1541
  573. package/simsimd/include/simsimd/dot.h +0 -1853
  574. package/simsimd/include/simsimd/elementwise.h +0 -2465
  575. package/simsimd/include/simsimd/geospatial.h +0 -43
  576. package/simsimd/include/simsimd/mesh.h +0 -69
  577. package/simsimd/include/simsimd/probability.h +0 -606
  578. package/simsimd/include/simsimd/simsimd.h +0 -2567
  579. package/simsimd/include/simsimd/sparse.h +0 -1384
  580. package/simsimd/include/simsimd/spatial.h +0 -2335
  581. package/simsimd/include/simsimd/types.h +0 -666
  582. package/simsimd/javascript/fallback.ts +0 -233
  583. package/simsimd/javascript/lib.c +0 -125
  584. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  585. package/simsimd/javascript/simsimd.ts +0 -179
  586. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  587. package/simsimd/package-lock.json +0 -1761
  588. package/simsimd/package.json +0 -61
  589. package/simsimd/rust/lib.rs +0 -1821
  590. package/simsimd/scripts/bench.cxx +0 -1170
  591. package/simsimd/scripts/bench.ipynb +0 -187
  592. package/simsimd/scripts/bench.js +0 -127
  593. package/simsimd/scripts/bench.rs +0 -876
  594. package/simsimd/scripts/bench_vectors.py +0 -734
  595. package/simsimd/scripts/bench_vectors_live.py +0 -213
  596. package/simsimd/scripts/test.c +0 -187
  597. package/simsimd/scripts/test.mjs +0 -220
  598. package/simsimd/scripts/test.py +0 -1505
  599. package/simsimd/swift/SimSIMD.swift +0 -149
  600. package/simsimd/swift/Test.swift +0 -128
  601. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  602. /package/{simsimd → numkong}/LICENSE +0 -0
  603. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  604. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -0,0 +1,3374 @@
1
+ /**
2
+ * @brief SIMD-accelerated Batched Spatial Distances (Angular & Euclidean).
3
+ * @file include/numkong/spatials.h
4
+ * @author Ash Vardanian
5
+ * @date February 22, 2026
6
+ *
7
+ * This module provides efficient batched computation of angular and euclidean distances
8
+ * via a two-pass approach: compute dot products first, then post-process with spatial
9
+ * distance formulas using pre-computed norms stored in the packed buffer.
10
+ *
11
+ * For dtypes:
12
+ *
13
+ * - f64: 64-bit IEEE floating point numbers → 64-bit floats
14
+ * - f32: 32-bit IEEE floating point numbers → 64-bit floats
15
+ * - f16: 16-bit IEEE floating point numbers → 32-bit floats
16
+ * - bf16: 16-bit brain floating point numbers → 32-bit floats
17
+ * - e4m3: 8-bit e4m3 floating point numbers → 32-bit floats
18
+ * - e5m2: 8-bit e5m2 floating point numbers → 32-bit floats
19
+ * - e2m3: 8-bit e2m3 floating point numbers (MX) → 32-bit floats
20
+ * - e3m2: 8-bit e3m2 floating point numbers (MX) → 32-bit floats
21
+ *
22
+ * For hardware architectures:
23
+ *
24
+ * - Arm: NEON, NEON+HALF, NEON+FHM, NEON+BF16, NEON+SDOT, SME, SME+F64
25
+ * - x86: Haswell, Skylake, Ice Lake, Genoa, Sapphire Rapids (AMX), Sierra Forest
26
+ * - RISC-V: RVV
27
+ *
28
+ * @section numerical_stability Numerical Stability
29
+ *
30
+ * Inherits dot-product precision from nk_dots_packed_* and keeps packed payloads narrow. `f32` batched spatial
31
+ * kernels now normalize from widened `f64` dots and norms and store `f64` results directly.
32
+ *
33
+ * @section approach Two-Pass Approach
34
+ *
35
+ * 1. Pack B matrix using nk_dots_pack_* (norms are stored in the packed buffer footer)
36
+ * 2. Compute nk_angulars_packed_* or nk_euclideans_packed_*:
37
+ * a. Internally calls nk_dots_packed_* to fill result buffer with dot products
38
+ * b. Post-processes each result cell using angular/euclidean formula with pre-computed norms
39
+ *
40
+ * @section math Mathematical Foundation
41
+ *
42
+ * Angular distance: 1 - dot(a,b) / sqrt(sumsq(a) * sumsq(b))
43
+ * Euclidean distance: sqrt(max(0, sumsq(a) + sumsq(b) - 2*dot(a,b)))
44
+ *
45
+ * @section packing Packing
46
+ *
47
+ * Uses the SAME pack functions as dot products (nk_dots_packed_size_*, nk_dots_pack_*).
48
+ * The packed buffer includes norms appended after the data.
49
+ */
50
+
51
+ #ifndef NK_SPATIALS_H
52
+ #define NK_SPATIALS_H
53
+
54
+ #include "numkong/dots.h"
55
+ #include "numkong/types.h"
56
+
57
+ #if defined(__cplusplus)
58
+ extern "C" {
59
+ #endif
60
+
61
+ /**
62
+ * @brief Computes batched angular distances using a packed second matrix.
63
+ * @param[in] a Input A matrix in row-major order.
64
+ * @param[in] b_packed Packed B matrix (produced by nk_dots_pack_*), with norms in footer.
65
+ * @param[out] result Output matrix (rows x cols) of angular distances.
66
+ * @param[in] rows Number of rows in A.
67
+ * @param[in] cols Number of columns in B (packed).
68
+ * @param[in] depth Shared inner dimension (vector length).
69
+ * @param[in] a_stride_in_bytes Row stride in bytes for A.
70
+ * @param[in] r_stride_in_bytes Row stride in bytes for the result matrix.
71
+ */
72
+ NK_DYNAMIC void nk_angulars_packed_f32(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
73
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
74
+ nk_size_t r_stride_in_bytes);
75
+
76
+ /**
77
+ * @brief Computes symmetric angular distance matrix (Gram-style) for a set of vectors.
78
+ * @param[in] vectors Input matrix of row vectors in row-major order.
79
+ * @param[in] vectors_count Number of vectors (rows) in the input matrix.
80
+ * @param[in] depth Dimension of each vector (columns).
81
+ * @param[in] stride Row stride in bytes for the input matrix.
82
+ * @param[out] result Output symmetric matrix (vectors_count x vectors_count).
83
+ * @param[in] result_stride Row stride in bytes for the result matrix.
84
+ * @param[in] row_start Starting row offset of results to compute (for parallelism).
85
+ * @param[in] row_count Number of rows of results to compute (for parallelism).
86
+ */
87
+ NK_DYNAMIC void nk_angulars_symmetric_f32(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
88
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
89
+ nk_size_t row_start, nk_size_t row_count);
90
+
91
+ /**
92
+ * @brief Computes batched euclidean distances using a packed second matrix.
93
+ * @param[in] a Input A matrix in row-major order.
94
+ * @param[in] b_packed Packed B matrix (produced by nk_dots_pack_*), with norms in footer.
95
+ * @param[out] result Output matrix (rows x cols) of euclidean distances.
96
+ * @param[in] rows Number of rows in A.
97
+ * @param[in] cols Number of columns in B (packed).
98
+ * @param[in] depth Shared inner dimension (vector length).
99
+ * @param[in] a_stride_in_bytes Row stride in bytes for A.
100
+ * @param[in] r_stride_in_bytes Row stride in bytes for the result matrix.
101
+ */
102
+ NK_DYNAMIC void nk_euclideans_packed_f32(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
103
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
104
+ nk_size_t r_stride_in_bytes);
105
+
106
+ /**
107
+ * @brief Computes symmetric euclidean distance matrix (Gram-style) for a set of vectors.
108
+ * @param[in] vectors Input matrix of row vectors in row-major order.
109
+ * @param[in] vectors_count Number of vectors (rows) in the input matrix.
110
+ * @param[in] depth Dimension of each vector (columns).
111
+ * @param[in] stride Row stride in bytes for the input matrix.
112
+ * @param[out] result Output symmetric matrix (vectors_count x vectors_count).
113
+ * @param[in] result_stride Row stride in bytes for the result matrix.
114
+ * @param[in] row_start Starting row offset of results to compute (for parallelism).
115
+ * @param[in] row_count Number of rows of results to compute (for parallelism).
116
+ */
117
+ NK_DYNAMIC void nk_euclideans_symmetric_f32(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
118
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
119
+ nk_size_t row_start, nk_size_t row_count);
120
+
121
+ /** @copydoc nk_angulars_packed_f32 */
122
+ NK_DYNAMIC void nk_angulars_packed_f64(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
123
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
124
+ nk_size_t r_stride_in_bytes);
125
+ /** @copydoc nk_angulars_symmetric_f32 */
126
+ NK_DYNAMIC void nk_angulars_symmetric_f64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
127
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
128
+ nk_size_t row_start, nk_size_t row_count);
129
+ /** @copydoc nk_euclideans_packed_f32 */
130
+ NK_DYNAMIC void nk_euclideans_packed_f64(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
131
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
132
+ nk_size_t r_stride_in_bytes);
133
+ /** @copydoc nk_euclideans_symmetric_f32 */
134
+ NK_DYNAMIC void nk_euclideans_symmetric_f64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
135
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
136
+ nk_size_t row_start, nk_size_t row_count);
137
+
138
+ /** @copydoc nk_angulars_packed_f32 */
139
+ NK_DYNAMIC void nk_angulars_packed_f16(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
140
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
141
+ nk_size_t r_stride_in_bytes);
142
+ /** @copydoc nk_angulars_symmetric_f32 */
143
+ NK_DYNAMIC void nk_angulars_symmetric_f16(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
144
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
145
+ nk_size_t row_start, nk_size_t row_count);
146
+ /** @copydoc nk_euclideans_packed_f32 */
147
+ NK_DYNAMIC void nk_euclideans_packed_f16(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
148
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
149
+ nk_size_t r_stride_in_bytes);
150
+ /** @copydoc nk_euclideans_symmetric_f32 */
151
+ NK_DYNAMIC void nk_euclideans_symmetric_f16(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
152
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
153
+ nk_size_t row_start, nk_size_t row_count);
154
+
155
+ /** @copydoc nk_angulars_packed_f32 */
156
+ NK_DYNAMIC void nk_angulars_packed_bf16(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
157
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
158
+ nk_size_t r_stride_in_bytes);
159
+ /** @copydoc nk_angulars_symmetric_f32 */
160
+ NK_DYNAMIC void nk_angulars_symmetric_bf16(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
161
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
162
+ nk_size_t row_start, nk_size_t row_count);
163
+ /** @copydoc nk_euclideans_packed_f32 */
164
+ NK_DYNAMIC void nk_euclideans_packed_bf16(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
165
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
166
+ nk_size_t r_stride_in_bytes);
167
+ /** @copydoc nk_euclideans_symmetric_f32 */
168
+ NK_DYNAMIC void nk_euclideans_symmetric_bf16(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
169
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
170
+ nk_size_t row_start, nk_size_t row_count);
171
+
172
+ /** @copydoc nk_angulars_packed_f32 */
173
+ NK_DYNAMIC void nk_angulars_packed_e4m3(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
174
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
175
+ nk_size_t r_stride_in_bytes);
176
+ /** @copydoc nk_angulars_symmetric_f32 */
177
+ NK_DYNAMIC void nk_angulars_symmetric_e4m3(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
178
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
179
+ nk_size_t row_start, nk_size_t row_count);
180
+ /** @copydoc nk_euclideans_packed_f32 */
181
+ NK_DYNAMIC void nk_euclideans_packed_e4m3(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
182
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
183
+ nk_size_t r_stride_in_bytes);
184
+ /** @copydoc nk_euclideans_symmetric_f32 */
185
+ NK_DYNAMIC void nk_euclideans_symmetric_e4m3(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
186
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
187
+ nk_size_t row_start, nk_size_t row_count);
188
+
189
+ /** @copydoc nk_angulars_packed_f32 */
190
+ NK_DYNAMIC void nk_angulars_packed_e5m2(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
191
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
192
+ nk_size_t r_stride_in_bytes);
193
+ /** @copydoc nk_angulars_symmetric_f32 */
194
+ NK_DYNAMIC void nk_angulars_symmetric_e5m2(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
195
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
196
+ nk_size_t row_start, nk_size_t row_count);
197
+ /** @copydoc nk_euclideans_packed_f32 */
198
+ NK_DYNAMIC void nk_euclideans_packed_e5m2(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
199
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
200
+ nk_size_t r_stride_in_bytes);
201
+ /** @copydoc nk_euclideans_symmetric_f32 */
202
+ NK_DYNAMIC void nk_euclideans_symmetric_e5m2(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
203
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
204
+ nk_size_t row_start, nk_size_t row_count);
205
+
206
+ /** @copydoc nk_angulars_packed_f32 */
207
+ NK_DYNAMIC void nk_angulars_packed_e2m3(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
208
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
209
+ nk_size_t r_stride_in_bytes);
210
+ /** @copydoc nk_angulars_symmetric_f32 */
211
+ NK_DYNAMIC void nk_angulars_symmetric_e2m3(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
212
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
213
+ nk_size_t row_start, nk_size_t row_count);
214
+ /** @copydoc nk_euclideans_packed_f32 */
215
+ NK_DYNAMIC void nk_euclideans_packed_e2m3(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
216
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
217
+ nk_size_t r_stride_in_bytes);
218
+ /** @copydoc nk_euclideans_symmetric_f32 */
219
+ NK_DYNAMIC void nk_euclideans_symmetric_e2m3(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
220
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
221
+ nk_size_t row_start, nk_size_t row_count);
222
+
223
+ /** @copydoc nk_angulars_packed_f32 */
224
+ NK_DYNAMIC void nk_angulars_packed_e3m2(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
225
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
226
+ nk_size_t r_stride_in_bytes);
227
+ /** @copydoc nk_angulars_symmetric_f32 */
228
+ NK_DYNAMIC void nk_angulars_symmetric_e3m2(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
229
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
230
+ nk_size_t row_start, nk_size_t row_count);
231
+ /** @copydoc nk_euclideans_packed_f32 */
232
+ NK_DYNAMIC void nk_euclideans_packed_e3m2(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
233
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
234
+ nk_size_t r_stride_in_bytes);
235
+ /** @copydoc nk_euclideans_symmetric_f32 */
236
+ NK_DYNAMIC void nk_euclideans_symmetric_e3m2(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
237
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
238
+ nk_size_t row_start, nk_size_t row_count);
239
+
240
+ /** @copydoc nk_angulars_packed_f32 */
241
+ NK_DYNAMIC void nk_angulars_packed_i8(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
242
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
243
+ nk_size_t r_stride_in_bytes);
244
+ /** @copydoc nk_angulars_symmetric_f32 */
245
+ NK_DYNAMIC void nk_angulars_symmetric_i8(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
246
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
247
+ nk_size_t row_start, nk_size_t row_count);
248
+ /** @copydoc nk_euclideans_packed_f32 */
249
+ NK_DYNAMIC void nk_euclideans_packed_i8(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
250
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
251
+ nk_size_t r_stride_in_bytes);
252
+ /** @copydoc nk_euclideans_symmetric_f32 */
253
+ NK_DYNAMIC void nk_euclideans_symmetric_i8(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
254
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
255
+ nk_size_t row_start, nk_size_t row_count);
256
+
257
+ /** @copydoc nk_angulars_packed_f32 */
258
+ NK_DYNAMIC void nk_angulars_packed_u8(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
259
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
260
+ nk_size_t r_stride_in_bytes);
261
+ /** @copydoc nk_angulars_symmetric_f32 */
262
+ NK_DYNAMIC void nk_angulars_symmetric_u8(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
263
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
264
+ nk_size_t row_start, nk_size_t row_count);
265
+ /** @copydoc nk_euclideans_packed_f32 */
266
+ NK_DYNAMIC void nk_euclideans_packed_u8(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
267
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
268
+ nk_size_t r_stride_in_bytes);
269
+ /** @copydoc nk_euclideans_symmetric_f32 */
270
+ NK_DYNAMIC void nk_euclideans_symmetric_u8(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
271
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
272
+ nk_size_t row_start, nk_size_t row_count);
273
+
274
+ /** @copydoc nk_angulars_packed_f32 */
275
+ NK_DYNAMIC void nk_angulars_packed_i4(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
276
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
277
+ nk_size_t r_stride_in_bytes);
278
+ /** @copydoc nk_angulars_symmetric_f32 */
279
+ NK_DYNAMIC void nk_angulars_symmetric_i4(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
280
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
281
+ nk_size_t row_start, nk_size_t row_count);
282
+ /** @copydoc nk_euclideans_packed_f32 */
283
+ NK_DYNAMIC void nk_euclideans_packed_i4(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
284
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
285
+ nk_size_t r_stride_in_bytes);
286
+ /** @copydoc nk_euclideans_symmetric_f32 */
287
+ NK_DYNAMIC void nk_euclideans_symmetric_i4(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
288
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
289
+ nk_size_t row_start, nk_size_t row_count);
290
+
291
+ /** @copydoc nk_angulars_packed_f32 */
292
+ NK_DYNAMIC void nk_angulars_packed_u4(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
293
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
294
+ nk_size_t r_stride_in_bytes);
295
+ /** @copydoc nk_angulars_symmetric_f32 */
296
+ NK_DYNAMIC void nk_angulars_symmetric_u4(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
297
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
298
+ nk_size_t row_start, nk_size_t row_count);
299
+ /** @copydoc nk_euclideans_packed_f32 */
300
+ NK_DYNAMIC void nk_euclideans_packed_u4(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
301
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
302
+ nk_size_t r_stride_in_bytes);
303
+ /** @copydoc nk_euclideans_symmetric_f32 */
304
+ NK_DYNAMIC void nk_euclideans_symmetric_u4(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
305
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
306
+ nk_size_t row_start, nk_size_t row_count);
307
+
308
+ /** @copydoc nk_angulars_packed_f32 */
309
+ NK_PUBLIC void nk_angulars_packed_f32_serial(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
310
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
311
+ nk_size_t r_stride_in_bytes);
312
+ /** @copydoc nk_angulars_symmetric_f32 */
313
+ NK_PUBLIC void nk_angulars_symmetric_f32_serial(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
314
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
315
+ nk_size_t row_start, nk_size_t row_count);
316
+ /** @copydoc nk_euclideans_packed_f32 */
317
+ NK_PUBLIC void nk_euclideans_packed_f32_serial(nk_f32_t const *a, void const *b_packed, nk_f64_t *result,
318
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
319
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
320
+ /** @copydoc nk_euclideans_symmetric_f32 */
321
+ NK_PUBLIC void nk_euclideans_symmetric_f32_serial(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
322
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
323
+ nk_size_t row_start, nk_size_t row_count);
324
+
325
+ /** @copydoc nk_angulars_packed_f64 */
326
+ NK_PUBLIC void nk_angulars_packed_f64_serial(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
327
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
328
+ nk_size_t r_stride_in_bytes);
329
+ /** @copydoc nk_angulars_symmetric_f64 */
330
+ NK_PUBLIC void nk_angulars_symmetric_f64_serial(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
331
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
332
+ nk_size_t row_start, nk_size_t row_count);
333
+ /** @copydoc nk_euclideans_packed_f64 */
334
+ NK_PUBLIC void nk_euclideans_packed_f64_serial(nk_f64_t const *a, void const *b_packed, nk_f64_t *result,
335
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
336
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
337
+ /** @copydoc nk_euclideans_symmetric_f64 */
338
+ NK_PUBLIC void nk_euclideans_symmetric_f64_serial(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
339
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
340
+ nk_size_t row_start, nk_size_t row_count);
341
+
342
+ /** @copydoc nk_angulars_packed_f16 */
343
+ NK_PUBLIC void nk_angulars_packed_f16_serial(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
344
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
345
+ nk_size_t r_stride_in_bytes);
346
+ /** @copydoc nk_angulars_symmetric_f16 */
347
+ NK_PUBLIC void nk_angulars_symmetric_f16_serial(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
348
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
349
+ nk_size_t row_start, nk_size_t row_count);
350
+ /** @copydoc nk_euclideans_packed_f16 */
351
+ NK_PUBLIC void nk_euclideans_packed_f16_serial(nk_f16_t const *a, void const *b_packed, nk_f32_t *result,
352
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
353
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
354
+ /** @copydoc nk_euclideans_symmetric_f16 */
355
+ NK_PUBLIC void nk_euclideans_symmetric_f16_serial(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
356
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
357
+ nk_size_t row_start, nk_size_t row_count);
358
+
359
+ /** @copydoc nk_angulars_packed_bf16 */
360
+ NK_PUBLIC void nk_angulars_packed_bf16_serial(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
361
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
362
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
363
+ /** @copydoc nk_angulars_symmetric_bf16 */
364
+ NK_PUBLIC void nk_angulars_symmetric_bf16_serial(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
365
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
366
+ nk_size_t row_start, nk_size_t row_count);
367
+ /** @copydoc nk_euclideans_packed_bf16 */
368
+ NK_PUBLIC void nk_euclideans_packed_bf16_serial(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
369
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
370
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
371
+ /** @copydoc nk_euclideans_symmetric_bf16 */
372
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_serial(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
373
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
374
+ nk_size_t row_start, nk_size_t row_count);
375
+
376
+ /** @copydoc nk_angulars_packed_e4m3 */
377
+ NK_PUBLIC void nk_angulars_packed_e4m3_serial(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
378
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
379
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
380
+ /** @copydoc nk_angulars_symmetric_e4m3 */
381
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_serial(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
382
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
383
+ nk_size_t row_start, nk_size_t row_count);
384
+ /** @copydoc nk_euclideans_packed_e4m3 */
385
+ NK_PUBLIC void nk_euclideans_packed_e4m3_serial(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
386
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
387
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
388
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
389
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_serial(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
390
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
391
+ nk_size_t row_start, nk_size_t row_count);
392
+
393
+ /** @copydoc nk_angulars_packed_e5m2 */
394
+ NK_PUBLIC void nk_angulars_packed_e5m2_serial(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
395
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
396
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
397
+ /** @copydoc nk_angulars_symmetric_e5m2 */
398
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_serial(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
399
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
400
+ nk_size_t row_start, nk_size_t row_count);
401
+ /** @copydoc nk_euclideans_packed_e5m2 */
402
+ NK_PUBLIC void nk_euclideans_packed_e5m2_serial(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
403
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
404
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
405
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
406
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_serial(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
407
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
408
+ nk_size_t row_start, nk_size_t row_count);
409
+
410
+ /** @copydoc nk_angulars_packed_e2m3 */
411
+ NK_PUBLIC void nk_angulars_packed_e2m3_serial(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
412
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
413
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
414
+ /** @copydoc nk_angulars_symmetric_e2m3 */
415
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_serial(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
416
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
417
+ nk_size_t row_start, nk_size_t row_count);
418
+ /** @copydoc nk_euclideans_packed_e2m3 */
419
+ NK_PUBLIC void nk_euclideans_packed_e2m3_serial(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
420
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
421
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
422
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
423
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_serial(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
424
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
425
+ nk_size_t row_start, nk_size_t row_count);
426
+
427
+ /** @copydoc nk_angulars_packed_e3m2 */
428
+ NK_PUBLIC void nk_angulars_packed_e3m2_serial(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
429
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
430
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
431
+ /** @copydoc nk_angulars_symmetric_e3m2 */
432
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_serial(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
433
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
434
+ nk_size_t row_start, nk_size_t row_count);
435
+ /** @copydoc nk_euclideans_packed_e3m2 */
436
+ NK_PUBLIC void nk_euclideans_packed_e3m2_serial(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
437
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
438
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
439
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
440
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_serial(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
441
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
442
+ nk_size_t row_start, nk_size_t row_count);
443
+
444
+ /** @copydoc nk_angulars_packed_i8 */
445
+ NK_PUBLIC void nk_angulars_packed_i8_serial(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
446
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
447
+ nk_size_t r_stride_in_bytes);
448
+ /** @copydoc nk_angulars_symmetric_i8 */
449
+ NK_PUBLIC void nk_angulars_symmetric_i8_serial(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
450
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
451
+ nk_size_t row_start, nk_size_t row_count);
452
+ /** @copydoc nk_euclideans_packed_i8 */
453
+ NK_PUBLIC void nk_euclideans_packed_i8_serial(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
454
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
455
+ nk_size_t r_stride_in_bytes);
456
+ /** @copydoc nk_euclideans_symmetric_i8 */
457
+ NK_PUBLIC void nk_euclideans_symmetric_i8_serial(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
458
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
459
+ nk_size_t row_start, nk_size_t row_count);
460
+
461
+ /** @copydoc nk_angulars_packed_u8 */
462
+ NK_PUBLIC void nk_angulars_packed_u8_serial(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
463
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
464
+ nk_size_t r_stride_in_bytes);
465
+ /** @copydoc nk_angulars_symmetric_u8 */
466
+ NK_PUBLIC void nk_angulars_symmetric_u8_serial(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
467
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
468
+ nk_size_t row_start, nk_size_t row_count);
469
+ /** @copydoc nk_euclideans_packed_u8 */
470
+ NK_PUBLIC void nk_euclideans_packed_u8_serial(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
471
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
472
+ nk_size_t r_stride_in_bytes);
473
+ /** @copydoc nk_euclideans_symmetric_u8 */
474
+ NK_PUBLIC void nk_euclideans_symmetric_u8_serial(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
475
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
476
+ nk_size_t row_start, nk_size_t row_count);
477
+
478
+ /** @copydoc nk_angulars_packed_i4 */
479
+ NK_PUBLIC void nk_angulars_packed_i4_serial(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
480
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
481
+ nk_size_t r_stride_in_bytes);
482
+ /** @copydoc nk_angulars_symmetric_i4 */
483
+ NK_PUBLIC void nk_angulars_symmetric_i4_serial(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
484
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
485
+ nk_size_t row_start, nk_size_t row_count);
486
+ /** @copydoc nk_euclideans_packed_i4 */
487
+ NK_PUBLIC void nk_euclideans_packed_i4_serial(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result,
488
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
489
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
490
+ /** @copydoc nk_euclideans_symmetric_i4 */
491
+ NK_PUBLIC void nk_euclideans_symmetric_i4_serial(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
492
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
493
+ nk_size_t row_start, nk_size_t row_count);
494
+
495
+ /** @copydoc nk_angulars_packed_u4 */
496
+ NK_PUBLIC void nk_angulars_packed_u4_serial(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
497
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
498
+ nk_size_t r_stride_in_bytes);
499
+ /** @copydoc nk_angulars_symmetric_u4 */
500
+ NK_PUBLIC void nk_angulars_symmetric_u4_serial(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
501
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
502
+ nk_size_t row_start, nk_size_t row_count);
503
+ /** @copydoc nk_euclideans_packed_u4 */
504
+ NK_PUBLIC void nk_euclideans_packed_u4_serial(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result,
505
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
506
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
507
+ /** @copydoc nk_euclideans_symmetric_u4 */
508
+ NK_PUBLIC void nk_euclideans_symmetric_u4_serial(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
509
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
510
+ nk_size_t row_start, nk_size_t row_count);
511
+
512
+ /* Genoa backends using AVX-512 with BF16 extensions.
513
+ * These use VDPBF16PS for BF16 dot products.
514
+ * Packing interleaves elements for SIMD broadcast patterns.
515
+ */
516
+ #if NK_TARGET_GENOA
517
+ /** @copydoc nk_angulars_packed_bf16 */
518
+ NK_PUBLIC void nk_angulars_packed_bf16_genoa(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
519
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
520
+ nk_size_t r_stride_in_bytes);
521
+ /** @copydoc nk_angulars_symmetric_bf16 */
522
+ NK_PUBLIC void nk_angulars_symmetric_bf16_genoa(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
523
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
524
+ nk_size_t row_start, nk_size_t row_count);
525
+ /** @copydoc nk_euclideans_packed_bf16 */
526
+ NK_PUBLIC void nk_euclideans_packed_bf16_genoa(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
527
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
528
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
529
+ /** @copydoc nk_euclideans_symmetric_bf16 */
530
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_genoa(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
531
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
532
+ nk_size_t row_start, nk_size_t row_count);
533
+
534
+ /** @copydoc nk_angulars_packed_e4m3 */
535
+ NK_PUBLIC void nk_angulars_packed_e4m3_genoa(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
536
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
537
+ nk_size_t r_stride_in_bytes);
538
+ /** @copydoc nk_angulars_symmetric_e4m3 */
539
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_genoa(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
540
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
541
+ nk_size_t row_start, nk_size_t row_count);
542
+ /** @copydoc nk_euclideans_packed_e4m3 */
543
+ NK_PUBLIC void nk_euclideans_packed_e4m3_genoa(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
544
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
545
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
546
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
547
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_genoa(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
548
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
549
+ nk_size_t row_start, nk_size_t row_count);
550
+
551
+ /** @copydoc nk_angulars_packed_e5m2 */
552
+ NK_PUBLIC void nk_angulars_packed_e5m2_genoa(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
553
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
554
+ nk_size_t r_stride_in_bytes);
555
+ /** @copydoc nk_angulars_symmetric_e5m2 */
556
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_genoa(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
557
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
558
+ nk_size_t row_start, nk_size_t row_count);
559
+ /** @copydoc nk_euclideans_packed_e5m2 */
560
+ NK_PUBLIC void nk_euclideans_packed_e5m2_genoa(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
561
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
562
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
563
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
564
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_genoa(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
565
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
566
+ nk_size_t row_start, nk_size_t row_count);
567
+
568
+ #endif // NK_TARGET_GENOA
569
+
570
+ #if NK_TARGET_DIAMOND
571
+ /** @copydoc nk_angulars_packed_e4m3 */
572
+ NK_PUBLIC void nk_angulars_packed_e4m3_diamond(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
573
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
574
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
575
+ /** @copydoc nk_angulars_symmetric_e4m3 */
576
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_diamond(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
577
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
578
+ nk_size_t row_start, nk_size_t row_count);
579
+ /** @copydoc nk_euclideans_packed_e4m3 */
580
+ NK_PUBLIC void nk_euclideans_packed_e4m3_diamond(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
581
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
582
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
583
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
584
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_diamond(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
585
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
586
+ nk_size_t row_start, nk_size_t row_count);
587
+
588
+ /** @copydoc nk_angulars_packed_e5m2 */
589
+ NK_PUBLIC void nk_angulars_packed_e5m2_diamond(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
590
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
591
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
592
+ /** @copydoc nk_angulars_symmetric_e5m2 */
593
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_diamond(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
594
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
595
+ nk_size_t row_start, nk_size_t row_count);
596
+ /** @copydoc nk_euclideans_packed_e5m2 */
597
+ NK_PUBLIC void nk_euclideans_packed_e5m2_diamond(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
598
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
599
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
600
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
601
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_diamond(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
602
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
603
+ nk_size_t row_start, nk_size_t row_count);
604
+ #endif // NK_TARGET_DIAMOND
605
+
606
+ /* Sapphire Rapids backends using Intel AMX (Advanced Matrix Extensions).
607
+ * AMX provides 8 tile registers (TMM0-TMM7), each holding up to 1KB of data.
608
+ * Tiles are configured as 16 rows x 64 bytes, enabling (16 x 32) BF16 or (16 x 64) INT8 tiles.
609
+ * Packing arranges data into AMX-native tile layout with pair interleaving for TDPBF16PS.
610
+ */
611
+ #if NK_TARGET_SAPPHIREAMX
612
+ /** @copydoc nk_angulars_packed_bf16 */
613
+ NK_PUBLIC void nk_angulars_packed_bf16_sapphireamx(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
614
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
615
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
616
+ /** @copydoc nk_angulars_symmetric_bf16 */
617
+ NK_PUBLIC void nk_angulars_symmetric_bf16_sapphireamx(nk_bf16_t const *vectors, nk_size_t vectors_count,
618
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
619
+ nk_size_t result_stride, nk_size_t row_start,
620
+ nk_size_t row_count);
621
+ /** @copydoc nk_euclideans_packed_bf16 */
622
+ NK_PUBLIC void nk_euclideans_packed_bf16_sapphireamx(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
623
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
624
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
625
+ /** @copydoc nk_euclideans_symmetric_bf16 */
626
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_sapphireamx(nk_bf16_t const *vectors, nk_size_t vectors_count,
627
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
628
+ nk_size_t result_stride, nk_size_t row_start,
629
+ nk_size_t row_count);
630
+
631
+ /** @copydoc nk_angulars_packed_e4m3 */
632
+ NK_PUBLIC void nk_angulars_packed_e4m3_sapphireamx(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
633
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
634
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
635
+ /** @copydoc nk_angulars_symmetric_e4m3 */
636
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_sapphireamx(nk_e4m3_t const *vectors, nk_size_t vectors_count,
637
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
638
+ nk_size_t result_stride, nk_size_t row_start,
639
+ nk_size_t row_count);
640
+ /** @copydoc nk_euclideans_packed_e4m3 */
641
+ NK_PUBLIC void nk_euclideans_packed_e4m3_sapphireamx(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
642
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
643
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
644
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
645
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_sapphireamx(nk_e4m3_t const *vectors, nk_size_t vectors_count,
646
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
647
+ nk_size_t result_stride, nk_size_t row_start,
648
+ nk_size_t row_count);
649
+
650
+ /** @copydoc nk_angulars_packed_e5m2 */
651
+ NK_PUBLIC void nk_angulars_packed_e5m2_sapphireamx(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
652
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
653
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
654
+ /** @copydoc nk_angulars_symmetric_e5m2 */
655
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_sapphireamx(nk_e5m2_t const *vectors, nk_size_t vectors_count,
656
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
657
+ nk_size_t result_stride, nk_size_t row_start,
658
+ nk_size_t row_count);
659
+ /** @copydoc nk_euclideans_packed_e5m2 */
660
+ NK_PUBLIC void nk_euclideans_packed_e5m2_sapphireamx(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
661
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
662
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
663
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
664
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_sapphireamx(nk_e5m2_t const *vectors, nk_size_t vectors_count,
665
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
666
+ nk_size_t result_stride, nk_size_t row_start,
667
+ nk_size_t row_count);
668
+
669
+ /** @copydoc nk_angulars_packed_e2m3 */
670
+ NK_PUBLIC void nk_angulars_packed_e2m3_sapphireamx(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
671
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
672
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
673
+ /** @copydoc nk_angulars_symmetric_e2m3 */
674
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_sapphireamx(nk_e2m3_t const *vectors, nk_size_t vectors_count,
675
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
676
+ nk_size_t result_stride, nk_size_t row_start,
677
+ nk_size_t row_count);
678
+ /** @copydoc nk_euclideans_packed_e2m3 */
679
+ NK_PUBLIC void nk_euclideans_packed_e2m3_sapphireamx(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
680
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
681
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
682
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
683
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_sapphireamx(nk_e2m3_t const *vectors, nk_size_t vectors_count,
684
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
685
+ nk_size_t result_stride, nk_size_t row_start,
686
+ nk_size_t row_count);
687
+
688
+ /** @copydoc nk_angulars_packed_e3m2 */
689
+ NK_PUBLIC void nk_angulars_packed_e3m2_sapphireamx(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
690
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
691
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
692
+ /** @copydoc nk_angulars_symmetric_e3m2 */
693
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_sapphireamx(nk_e3m2_t const *vectors, nk_size_t vectors_count,
694
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
695
+ nk_size_t result_stride, nk_size_t row_start,
696
+ nk_size_t row_count);
697
+ /** @copydoc nk_euclideans_packed_e3m2 */
698
+ NK_PUBLIC void nk_euclideans_packed_e3m2_sapphireamx(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
699
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
700
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
701
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
702
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_sapphireamx(nk_e3m2_t const *vectors, nk_size_t vectors_count,
703
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
704
+ nk_size_t result_stride, nk_size_t row_start,
705
+ nk_size_t row_count);
706
+
707
+ /** @copydoc nk_angulars_packed_i8 */
708
+ NK_PUBLIC void nk_angulars_packed_i8_sapphireamx(nk_i8_t const *a, void const *b_packed, nk_f32_t *result,
709
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
710
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
711
+ /** @copydoc nk_angulars_symmetric_i8 */
712
+ NK_PUBLIC void nk_angulars_symmetric_i8_sapphireamx(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
713
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
714
+ nk_size_t row_start, nk_size_t row_count);
715
+ /** @copydoc nk_euclideans_packed_i8 */
716
+ NK_PUBLIC void nk_euclideans_packed_i8_sapphireamx(nk_i8_t const *a, void const *b_packed, nk_f32_t *result,
717
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
718
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
719
+ /** @copydoc nk_euclideans_symmetric_i8 */
720
+ NK_PUBLIC void nk_euclideans_symmetric_i8_sapphireamx(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
721
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
722
+ nk_size_t row_start, nk_size_t row_count);
723
+
724
+ /** @copydoc nk_angulars_packed_u8 */
725
+ NK_PUBLIC void nk_angulars_packed_u8_sapphireamx(nk_u8_t const *a, void const *b_packed, nk_f32_t *result,
726
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
727
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
728
+ /** @copydoc nk_angulars_symmetric_u8 */
729
+ NK_PUBLIC void nk_angulars_symmetric_u8_sapphireamx(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
730
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
731
+ nk_size_t row_start, nk_size_t row_count);
732
+ /** @copydoc nk_euclideans_packed_u8 */
733
+ NK_PUBLIC void nk_euclideans_packed_u8_sapphireamx(nk_u8_t const *a, void const *b_packed, nk_f32_t *result,
734
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
735
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
736
+ /** @copydoc nk_euclideans_symmetric_u8 */
737
+ NK_PUBLIC void nk_euclideans_symmetric_u8_sapphireamx(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
738
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
739
+ nk_size_t row_start, nk_size_t row_count);
740
+ #endif // NK_TARGET_SAPPHIREAMX
741
+
742
+ /* Granite Rapids backends using Intel AMX-FP16.
743
+ * Native FP16 spatial kernels.
744
+ */
745
+ #if NK_TARGET_GRANITEAMX
746
+ /** @copydoc nk_angulars_packed_f16 */
747
+ NK_PUBLIC void nk_angulars_packed_f16_graniteamx(nk_f16_t const *a, void const *b_packed, nk_f32_t *result,
748
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
749
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
750
+ /** @copydoc nk_angulars_symmetric_f16 */
751
+ NK_PUBLIC void nk_angulars_symmetric_f16_graniteamx(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
752
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
753
+ nk_size_t row_start, nk_size_t row_count);
754
+ /** @copydoc nk_euclideans_packed_f16 */
755
+ NK_PUBLIC void nk_euclideans_packed_f16_graniteamx(nk_f16_t const *a, void const *b_packed, nk_f32_t *result,
756
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
757
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
758
+ /** @copydoc nk_euclideans_symmetric_f16 */
759
+ NK_PUBLIC void nk_euclideans_symmetric_f16_graniteamx(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
760
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
761
+ nk_size_t row_start, nk_size_t row_count);
762
+ #endif // NK_TARGET_GRANITEAMX
763
+
764
+ /* ARM SME backends using Scalable Matrix Extension.
765
+ * SME provides ZA tile registers for outer product operations.
766
+ * F16/BF16/I8/U8/E4M3 use ZA32 tiles, F32/F64 use ZA64 tiles (FEAT_SME_F64F64).
767
+ */
768
+ #if NK_TARGET_SME
769
+ /** @copydoc nk_angulars_packed_f16 */
770
+ NK_PUBLIC void nk_angulars_packed_f16_sme(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
771
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
772
+ nk_size_t r_stride_in_bytes);
773
+ /** @copydoc nk_angulars_symmetric_f16 */
774
+ NK_PUBLIC void nk_angulars_symmetric_f16_sme(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
775
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
776
+ nk_size_t row_start, nk_size_t row_count);
777
+ /** @copydoc nk_euclideans_packed_f16 */
778
+ NK_PUBLIC void nk_euclideans_packed_f16_sme(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
779
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
780
+ nk_size_t r_stride_in_bytes);
781
+ /** @copydoc nk_euclideans_symmetric_f16 */
782
+ NK_PUBLIC void nk_euclideans_symmetric_f16_sme(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
783
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
784
+ nk_size_t row_start, nk_size_t row_count);
785
+
786
+ /** @copydoc nk_angulars_packed_bf16 */
787
+ NK_PUBLIC void nk_angulars_packed_bf16_sme(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
788
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
789
+ nk_size_t r_stride_in_bytes);
790
+ /** @copydoc nk_angulars_symmetric_bf16 */
791
+ NK_PUBLIC void nk_angulars_symmetric_bf16_sme(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
792
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
793
+ nk_size_t row_start, nk_size_t row_count);
794
+ /** @copydoc nk_euclideans_packed_bf16 */
795
+ NK_PUBLIC void nk_euclideans_packed_bf16_sme(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
796
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
797
+ nk_size_t r_stride_in_bytes);
798
+ /** @copydoc nk_euclideans_symmetric_bf16 */
799
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_sme(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
800
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
801
+ nk_size_t row_start, nk_size_t row_count);
802
+
803
+ /** @copydoc nk_angulars_packed_e4m3 */
804
+ NK_PUBLIC void nk_angulars_packed_e4m3_sme(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
805
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
806
+ nk_size_t r_stride_in_bytes);
807
+ /** @copydoc nk_angulars_symmetric_e4m3 */
808
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_sme(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
809
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
810
+ nk_size_t row_start, nk_size_t row_count);
811
+ /** @copydoc nk_euclideans_packed_e4m3 */
812
+ NK_PUBLIC void nk_euclideans_packed_e4m3_sme(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
813
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
814
+ nk_size_t r_stride_in_bytes);
815
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
816
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_sme(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
817
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
818
+ nk_size_t row_start, nk_size_t row_count);
819
+
820
+ /** @copydoc nk_angulars_packed_e5m2 */
821
+ NK_PUBLIC void nk_angulars_packed_e5m2_sme(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
822
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
823
+ nk_size_t r_stride_in_bytes);
824
+ /** @copydoc nk_angulars_symmetric_e5m2 */
825
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_sme(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
826
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
827
+ nk_size_t row_start, nk_size_t row_count);
828
+ /** @copydoc nk_euclideans_packed_e5m2 */
829
+ NK_PUBLIC void nk_euclideans_packed_e5m2_sme(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
830
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
831
+ nk_size_t r_stride_in_bytes);
832
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
833
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_sme(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
834
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
835
+ nk_size_t row_start, nk_size_t row_count);
836
+
837
+ /** @copydoc nk_angulars_packed_e2m3 */
838
+ NK_PUBLIC void nk_angulars_packed_e2m3_sme(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
839
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
840
+ nk_size_t r_stride_in_bytes);
841
+ /** @copydoc nk_angulars_symmetric_e2m3 */
842
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_sme(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
843
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
844
+ nk_size_t row_start, nk_size_t row_count);
845
+ /** @copydoc nk_euclideans_packed_e2m3 */
846
+ NK_PUBLIC void nk_euclideans_packed_e2m3_sme(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
847
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
848
+ nk_size_t r_stride_in_bytes);
849
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
850
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_sme(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
851
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
852
+ nk_size_t row_start, nk_size_t row_count);
853
+
854
+ /** @copydoc nk_angulars_packed_e3m2 */
855
+ NK_PUBLIC void nk_angulars_packed_e3m2_sme(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
856
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
857
+ nk_size_t r_stride_in_bytes);
858
+ /** @copydoc nk_angulars_symmetric_e3m2 */
859
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_sme(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
860
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
861
+ nk_size_t row_start, nk_size_t row_count);
862
+ /** @copydoc nk_euclideans_packed_e3m2 */
863
+ NK_PUBLIC void nk_euclideans_packed_e3m2_sme(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
864
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
865
+ nk_size_t r_stride_in_bytes);
866
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
867
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_sme(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
868
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
869
+ nk_size_t row_start, nk_size_t row_count);
870
+
871
+ /** @copydoc nk_angulars_packed_i8 */
872
+ NK_PUBLIC void nk_angulars_packed_i8_sme(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
873
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
874
+ nk_size_t r_stride_in_bytes);
875
+ /** @copydoc nk_angulars_symmetric_i8 */
876
+ NK_PUBLIC void nk_angulars_symmetric_i8_sme(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
877
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
878
+ nk_size_t row_start, nk_size_t row_count);
879
+ /** @copydoc nk_euclideans_packed_i8 */
880
+ NK_PUBLIC void nk_euclideans_packed_i8_sme(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
881
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
882
+ nk_size_t r_stride_in_bytes);
883
+ /** @copydoc nk_euclideans_symmetric_i8 */
884
+ NK_PUBLIC void nk_euclideans_symmetric_i8_sme(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
885
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
886
+ nk_size_t row_start, nk_size_t row_count);
887
+
888
+ /** @copydoc nk_angulars_packed_u8 */
889
+ NK_PUBLIC void nk_angulars_packed_u8_sme(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
890
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
891
+ nk_size_t r_stride_in_bytes);
892
+ /** @copydoc nk_angulars_symmetric_u8 */
893
+ NK_PUBLIC void nk_angulars_symmetric_u8_sme(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
894
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
895
+ nk_size_t row_start, nk_size_t row_count);
896
+ /** @copydoc nk_euclideans_packed_u8 */
897
+ NK_PUBLIC void nk_euclideans_packed_u8_sme(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
898
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
899
+ nk_size_t r_stride_in_bytes);
900
+ /** @copydoc nk_euclideans_symmetric_u8 */
901
+ NK_PUBLIC void nk_euclideans_symmetric_u8_sme(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
902
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
903
+ nk_size_t row_start, nk_size_t row_count);
904
+
905
+ /** @copydoc nk_angulars_packed_i4 */
906
+ NK_PUBLIC void nk_angulars_packed_i4_sme(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
907
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
908
+ nk_size_t r_stride_in_bytes);
909
+ /** @copydoc nk_angulars_symmetric_i4 */
910
+ NK_PUBLIC void nk_angulars_symmetric_i4_sme(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
911
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
912
+ nk_size_t row_start, nk_size_t row_count);
913
+ /** @copydoc nk_euclideans_packed_i4 */
914
+ NK_PUBLIC void nk_euclideans_packed_i4_sme(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
915
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
916
+ nk_size_t r_stride_in_bytes);
917
+ /** @copydoc nk_euclideans_symmetric_i4 */
918
+ NK_PUBLIC void nk_euclideans_symmetric_i4_sme(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
919
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
920
+ nk_size_t row_start, nk_size_t row_count);
921
+
922
+ /** @copydoc nk_angulars_packed_u4 */
923
+ NK_PUBLIC void nk_angulars_packed_u4_sme(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
924
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
925
+ nk_size_t r_stride_in_bytes);
926
+ /** @copydoc nk_angulars_symmetric_u4 */
927
+ NK_PUBLIC void nk_angulars_symmetric_u4_sme(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
928
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
929
+ nk_size_t row_start, nk_size_t row_count);
930
+ /** @copydoc nk_euclideans_packed_u4 */
931
+ NK_PUBLIC void nk_euclideans_packed_u4_sme(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
932
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
933
+ nk_size_t r_stride_in_bytes);
934
+ /** @copydoc nk_euclideans_symmetric_u4 */
935
+ NK_PUBLIC void nk_euclideans_symmetric_u4_sme(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
936
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
937
+ nk_size_t row_start, nk_size_t row_count);
938
+ #endif // NK_TARGET_SME
939
+
940
+ /* ARM SME with FEAT_SME_F64F64 (F32/F64 with F64 accumulators).
941
+ * Requires Apple M4 or equivalent with F64 outer product support.
942
+ */
943
+ #if NK_TARGET_SMEF64
944
+ /** @copydoc nk_angulars_packed_f32 */
945
+ NK_PUBLIC void nk_angulars_packed_f32_smef64(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
946
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
947
+ nk_size_t r_stride_in_bytes);
948
+ /** @copydoc nk_angulars_symmetric_f32 */
949
+ NK_PUBLIC void nk_angulars_symmetric_f32_smef64(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
950
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
951
+ nk_size_t row_start, nk_size_t row_count);
952
+ /** @copydoc nk_euclideans_packed_f32 */
953
+ NK_PUBLIC void nk_euclideans_packed_f32_smef64(nk_f32_t const *a, void const *b_packed, nk_f64_t *result,
954
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
955
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
956
+ /** @copydoc nk_euclideans_symmetric_f32 */
957
+ NK_PUBLIC void nk_euclideans_symmetric_f32_smef64(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
958
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
959
+ nk_size_t row_start, nk_size_t row_count);
960
+
961
+ /** @copydoc nk_angulars_packed_f64 */
962
+ NK_PUBLIC void nk_angulars_packed_f64_smef64(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
963
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
964
+ nk_size_t r_stride_in_bytes);
965
+ /** @copydoc nk_angulars_symmetric_f64 */
966
+ NK_PUBLIC void nk_angulars_symmetric_f64_smef64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
967
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
968
+ nk_size_t row_start, nk_size_t row_count);
969
+ /** @copydoc nk_euclideans_packed_f64 */
970
+ NK_PUBLIC void nk_euclideans_packed_f64_smef64(nk_f64_t const *a, void const *b_packed, nk_f64_t *result,
971
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
972
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
973
+ /** @copydoc nk_euclideans_symmetric_f64 */
974
+ NK_PUBLIC void nk_euclideans_symmetric_f64_smef64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
975
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
976
+ nk_size_t row_start, nk_size_t row_count);
977
+ #endif // NK_TARGET_SMEF64
978
+
979
+ /* Haswell backends using AVX2 (Intel Core 4th gen).
980
+ * Supports F32/F64 via FMA, F16/BF16/FP8 via software emulation, I8/U8 via VPMADDUBSW+VPADDD.
981
+ */
982
+ #if NK_TARGET_HASWELL
983
+ /** @copydoc nk_angulars_packed_f32 */
984
+ NK_PUBLIC void nk_angulars_packed_f32_haswell(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
985
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
986
+ nk_size_t r_stride_in_bytes);
987
+ /** @copydoc nk_angulars_symmetric_f32 */
988
+ NK_PUBLIC void nk_angulars_symmetric_f32_haswell(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
989
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
990
+ nk_size_t row_start, nk_size_t row_count);
991
+ /** @copydoc nk_euclideans_packed_f32 */
992
+ NK_PUBLIC void nk_euclideans_packed_f32_haswell(nk_f32_t const *a, void const *b_packed, nk_f64_t *result,
993
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
994
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
995
+ /** @copydoc nk_euclideans_symmetric_f32 */
996
+ NK_PUBLIC void nk_euclideans_symmetric_f32_haswell(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
997
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
998
+ nk_size_t row_start, nk_size_t row_count);
999
+
1000
+ /** @copydoc nk_angulars_packed_f64 */
1001
+ NK_PUBLIC void nk_angulars_packed_f64_haswell(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1002
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1003
+ nk_size_t r_stride_in_bytes);
1004
+ /** @copydoc nk_angulars_symmetric_f64 */
1005
+ NK_PUBLIC void nk_angulars_symmetric_f64_haswell(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1006
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1007
+ nk_size_t row_start, nk_size_t row_count);
1008
+ /** @copydoc nk_euclideans_packed_f64 */
1009
+ NK_PUBLIC void nk_euclideans_packed_f64_haswell(nk_f64_t const *a, void const *b_packed, nk_f64_t *result,
1010
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1011
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1012
+ /** @copydoc nk_euclideans_symmetric_f64 */
1013
+ NK_PUBLIC void nk_euclideans_symmetric_f64_haswell(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1014
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1015
+ nk_size_t row_start, nk_size_t row_count);
1016
+
1017
+ /** @copydoc nk_angulars_packed_f16 */
1018
+ NK_PUBLIC void nk_angulars_packed_f16_haswell(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1019
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1020
+ nk_size_t r_stride_in_bytes);
1021
+ /** @copydoc nk_angulars_symmetric_f16 */
1022
+ NK_PUBLIC void nk_angulars_symmetric_f16_haswell(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1023
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1024
+ nk_size_t row_start, nk_size_t row_count);
1025
+ /** @copydoc nk_euclideans_packed_f16 */
1026
+ NK_PUBLIC void nk_euclideans_packed_f16_haswell(nk_f16_t const *a, void const *b_packed, nk_f32_t *result,
1027
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1028
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1029
+ /** @copydoc nk_euclideans_symmetric_f16 */
1030
+ NK_PUBLIC void nk_euclideans_symmetric_f16_haswell(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1031
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1032
+ nk_size_t row_start, nk_size_t row_count);
1033
+
1034
+ /** @copydoc nk_angulars_packed_bf16 */
1035
+ NK_PUBLIC void nk_angulars_packed_bf16_haswell(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1036
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1037
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1038
+ /** @copydoc nk_angulars_symmetric_bf16 */
1039
+ NK_PUBLIC void nk_angulars_symmetric_bf16_haswell(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1040
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1041
+ nk_size_t row_start, nk_size_t row_count);
1042
+ /** @copydoc nk_euclideans_packed_bf16 */
1043
+ NK_PUBLIC void nk_euclideans_packed_bf16_haswell(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1044
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1045
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1046
+ /** @copydoc nk_euclideans_symmetric_bf16 */
1047
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_haswell(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1048
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1049
+ nk_size_t row_start, nk_size_t row_count);
1050
+
1051
+ /** @copydoc nk_angulars_packed_e4m3 */
1052
+ NK_PUBLIC void nk_angulars_packed_e4m3_haswell(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1053
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1054
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1055
+ /** @copydoc nk_angulars_symmetric_e4m3 */
1056
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_haswell(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1057
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1058
+ nk_size_t row_start, nk_size_t row_count);
1059
+ /** @copydoc nk_euclideans_packed_e4m3 */
1060
+ NK_PUBLIC void nk_euclideans_packed_e4m3_haswell(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1061
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1062
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1063
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
1064
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_haswell(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1065
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1066
+ nk_size_t row_start, nk_size_t row_count);
1067
+
1068
+ /** @copydoc nk_angulars_packed_e5m2 */
1069
+ NK_PUBLIC void nk_angulars_packed_e5m2_haswell(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1070
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1071
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1072
+ /** @copydoc nk_angulars_symmetric_e5m2 */
1073
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_haswell(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1074
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1075
+ nk_size_t row_start, nk_size_t row_count);
1076
+ /** @copydoc nk_euclideans_packed_e5m2 */
1077
+ NK_PUBLIC void nk_euclideans_packed_e5m2_haswell(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1078
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1079
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1080
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
1081
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_haswell(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1082
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1083
+ nk_size_t row_start, nk_size_t row_count);
1084
+
1085
+ /** @copydoc nk_angulars_packed_e2m3 */
1086
+ NK_PUBLIC void nk_angulars_packed_e2m3_haswell(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1087
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1088
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1089
+ /** @copydoc nk_angulars_symmetric_e2m3 */
1090
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_haswell(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1091
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1092
+ nk_size_t row_start, nk_size_t row_count);
1093
+ /** @copydoc nk_euclideans_packed_e2m3 */
1094
+ NK_PUBLIC void nk_euclideans_packed_e2m3_haswell(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1095
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1096
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1097
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
1098
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_haswell(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1099
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1100
+ nk_size_t row_start, nk_size_t row_count);
1101
+
1102
+ /** @copydoc nk_angulars_packed_e3m2 */
1103
+ NK_PUBLIC void nk_angulars_packed_e3m2_haswell(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
1104
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1105
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1106
+ /** @copydoc nk_angulars_symmetric_e3m2 */
1107
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_haswell(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1108
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1109
+ nk_size_t row_start, nk_size_t row_count);
1110
+ /** @copydoc nk_euclideans_packed_e3m2 */
1111
+ NK_PUBLIC void nk_euclideans_packed_e3m2_haswell(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
1112
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1113
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1114
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
1115
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_haswell(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1116
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1117
+ nk_size_t row_start, nk_size_t row_count);
1118
+ /** @copydoc nk_angulars_packed_i8 */
1119
+ NK_PUBLIC void nk_angulars_packed_i8_haswell(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1120
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1121
+ nk_size_t r_stride_in_bytes);
1122
+ /** @copydoc nk_angulars_symmetric_i8 */
1123
+ NK_PUBLIC void nk_angulars_symmetric_i8_haswell(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1124
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1125
+ nk_size_t row_start, nk_size_t row_count);
1126
+ /** @copydoc nk_euclideans_packed_i8 */
1127
+ NK_PUBLIC void nk_euclideans_packed_i8_haswell(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1128
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1129
+ nk_size_t r_stride_in_bytes);
1130
+ /** @copydoc nk_euclideans_symmetric_i8 */
1131
+ NK_PUBLIC void nk_euclideans_symmetric_i8_haswell(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1132
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1133
+ nk_size_t row_start, nk_size_t row_count);
1134
+ /** @copydoc nk_angulars_packed_u8 */
1135
+ NK_PUBLIC void nk_angulars_packed_u8_haswell(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1136
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1137
+ nk_size_t r_stride_in_bytes);
1138
+ /** @copydoc nk_angulars_symmetric_u8 */
1139
+ NK_PUBLIC void nk_angulars_symmetric_u8_haswell(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1140
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1141
+ nk_size_t row_start, nk_size_t row_count);
1142
+ /** @copydoc nk_euclideans_packed_u8 */
1143
+ NK_PUBLIC void nk_euclideans_packed_u8_haswell(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1144
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1145
+ nk_size_t r_stride_in_bytes);
1146
+ /** @copydoc nk_euclideans_symmetric_u8 */
1147
+ NK_PUBLIC void nk_euclideans_symmetric_u8_haswell(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1148
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1149
+ nk_size_t row_start, nk_size_t row_count);
1150
+ #endif // NK_TARGET_HASWELL
1151
+
1152
+ /* Skylake backends using AVX-512 (Intel Core 6th gen+).
1153
+ * Provides 512-bit vectors (16x f32, 8x f64), supporting F32/F64/F16/BF16/FP8 with FMA.
1154
+ */
1155
+ #if NK_TARGET_SKYLAKE
1156
+ /** @copydoc nk_angulars_packed_f32 */
1157
+ NK_PUBLIC void nk_angulars_packed_f32_skylake(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1158
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1159
+ nk_size_t r_stride_in_bytes);
1160
+ /** @copydoc nk_angulars_symmetric_f32 */
1161
+ NK_PUBLIC void nk_angulars_symmetric_f32_skylake(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1162
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1163
+ nk_size_t row_start, nk_size_t row_count);
1164
+ /** @copydoc nk_euclideans_packed_f32 */
1165
+ NK_PUBLIC void nk_euclideans_packed_f32_skylake(nk_f32_t const *a, void const *b_packed, nk_f64_t *result,
1166
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1167
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1168
+ /** @copydoc nk_euclideans_symmetric_f32 */
1169
+ NK_PUBLIC void nk_euclideans_symmetric_f32_skylake(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1170
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1171
+ nk_size_t row_start, nk_size_t row_count);
1172
+
1173
+ /** @copydoc nk_angulars_packed_f64 */
1174
+ NK_PUBLIC void nk_angulars_packed_f64_skylake(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1175
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1176
+ nk_size_t r_stride_in_bytes);
1177
+ /** @copydoc nk_angulars_symmetric_f64 */
1178
+ NK_PUBLIC void nk_angulars_symmetric_f64_skylake(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1179
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1180
+ nk_size_t row_start, nk_size_t row_count);
1181
+ /** @copydoc nk_euclideans_packed_f64 */
1182
+ NK_PUBLIC void nk_euclideans_packed_f64_skylake(nk_f64_t const *a, void const *b_packed, nk_f64_t *result,
1183
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1184
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1185
+ /** @copydoc nk_euclideans_symmetric_f64 */
1186
+ NK_PUBLIC void nk_euclideans_symmetric_f64_skylake(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1187
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1188
+ nk_size_t row_start, nk_size_t row_count);
1189
+
1190
+ /** @copydoc nk_angulars_packed_f16 */
1191
+ NK_PUBLIC void nk_angulars_packed_f16_skylake(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1192
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1193
+ nk_size_t r_stride_in_bytes);
1194
+ /** @copydoc nk_angulars_symmetric_f16 */
1195
+ NK_PUBLIC void nk_angulars_symmetric_f16_skylake(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1196
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1197
+ nk_size_t row_start, nk_size_t row_count);
1198
+ /** @copydoc nk_euclideans_packed_f16 */
1199
+ NK_PUBLIC void nk_euclideans_packed_f16_skylake(nk_f16_t const *a, void const *b_packed, nk_f32_t *result,
1200
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1201
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1202
+ /** @copydoc nk_euclideans_symmetric_f16 */
1203
+ NK_PUBLIC void nk_euclideans_symmetric_f16_skylake(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1204
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1205
+ nk_size_t row_start, nk_size_t row_count);
1206
+
1207
+ /** @copydoc nk_angulars_packed_bf16 */
1208
+ NK_PUBLIC void nk_angulars_packed_bf16_skylake(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1209
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1210
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1211
+ /** @copydoc nk_angulars_symmetric_bf16 */
1212
+ NK_PUBLIC void nk_angulars_symmetric_bf16_skylake(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1213
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1214
+ nk_size_t row_start, nk_size_t row_count);
1215
+ /** @copydoc nk_euclideans_packed_bf16 */
1216
+ NK_PUBLIC void nk_euclideans_packed_bf16_skylake(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1217
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1218
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1219
+ /** @copydoc nk_euclideans_symmetric_bf16 */
1220
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_skylake(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1221
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1222
+ nk_size_t row_start, nk_size_t row_count);
1223
+
1224
+ /** @copydoc nk_angulars_packed_e4m3 */
1225
+ NK_PUBLIC void nk_angulars_packed_e4m3_skylake(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1226
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1227
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1228
+ /** @copydoc nk_angulars_symmetric_e4m3 */
1229
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_skylake(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1230
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1231
+ nk_size_t row_start, nk_size_t row_count);
1232
+ /** @copydoc nk_euclideans_packed_e4m3 */
1233
+ NK_PUBLIC void nk_euclideans_packed_e4m3_skylake(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1234
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1235
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1236
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
1237
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_skylake(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1238
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1239
+ nk_size_t row_start, nk_size_t row_count);
1240
+
1241
+ /** @copydoc nk_angulars_packed_e5m2 */
1242
+ NK_PUBLIC void nk_angulars_packed_e5m2_skylake(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1243
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1244
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1245
+ /** @copydoc nk_angulars_symmetric_e5m2 */
1246
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_skylake(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1247
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1248
+ nk_size_t row_start, nk_size_t row_count);
1249
+ /** @copydoc nk_euclideans_packed_e5m2 */
1250
+ NK_PUBLIC void nk_euclideans_packed_e5m2_skylake(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1251
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1252
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1253
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
1254
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_skylake(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1255
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1256
+ nk_size_t row_start, nk_size_t row_count);
1257
+
1258
+ /** @copydoc nk_angulars_packed_e2m3 */
1259
+ NK_PUBLIC void nk_angulars_packed_e2m3_skylake(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1260
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1261
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1262
+ /** @copydoc nk_angulars_symmetric_e2m3 */
1263
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_skylake(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1264
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1265
+ nk_size_t row_start, nk_size_t row_count);
1266
+ /** @copydoc nk_euclideans_packed_e2m3 */
1267
+ NK_PUBLIC void nk_euclideans_packed_e2m3_skylake(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1268
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1269
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1270
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
1271
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_skylake(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1272
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1273
+ nk_size_t row_start, nk_size_t row_count);
1274
+
1275
+ /** @copydoc nk_angulars_packed_e3m2 */
1276
+ NK_PUBLIC void nk_angulars_packed_e3m2_skylake(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
1277
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1278
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1279
+ /** @copydoc nk_angulars_symmetric_e3m2 */
1280
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_skylake(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1281
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1282
+ nk_size_t row_start, nk_size_t row_count);
1283
+ /** @copydoc nk_euclideans_packed_e3m2 */
1284
+ NK_PUBLIC void nk_euclideans_packed_e3m2_skylake(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
1285
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1286
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1287
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
1288
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_skylake(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1289
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1290
+ nk_size_t row_start, nk_size_t row_count);
1291
+ #endif // NK_TARGET_SKYLAKE
1292
+
1293
+ /* Ice Lake backends using AVX-512 with VNNI (Vector Neural Network Instructions).
1294
+ * Adds VPDPBUSD for I8/U8, VPDPWSSD for I4/U4 with efficient dot products.
1295
+ */
1296
+ #if NK_TARGET_ICELAKE
1297
+ /** @copydoc nk_angulars_packed_i8 */
1298
+ NK_PUBLIC void nk_angulars_packed_i8_icelake(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1299
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1300
+ nk_size_t r_stride_in_bytes);
1301
+ /** @copydoc nk_angulars_symmetric_i8 */
1302
+ NK_PUBLIC void nk_angulars_symmetric_i8_icelake(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1303
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1304
+ nk_size_t row_start, nk_size_t row_count);
1305
+ /** @copydoc nk_euclideans_packed_i8 */
1306
+ NK_PUBLIC void nk_euclideans_packed_i8_icelake(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1307
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1308
+ nk_size_t r_stride_in_bytes);
1309
+ /** @copydoc nk_euclideans_symmetric_i8 */
1310
+ NK_PUBLIC void nk_euclideans_symmetric_i8_icelake(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1311
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1312
+ nk_size_t row_start, nk_size_t row_count);
1313
+
1314
+ /** @copydoc nk_angulars_packed_u8 */
1315
+ NK_PUBLIC void nk_angulars_packed_u8_icelake(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1316
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1317
+ nk_size_t r_stride_in_bytes);
1318
+ /** @copydoc nk_angulars_symmetric_u8 */
1319
+ NK_PUBLIC void nk_angulars_symmetric_u8_icelake(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1320
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1321
+ nk_size_t row_start, nk_size_t row_count);
1322
+ /** @copydoc nk_euclideans_packed_u8 */
1323
+ NK_PUBLIC void nk_euclideans_packed_u8_icelake(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1324
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1325
+ nk_size_t r_stride_in_bytes);
1326
+ /** @copydoc nk_euclideans_symmetric_u8 */
1327
+ NK_PUBLIC void nk_euclideans_symmetric_u8_icelake(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1328
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1329
+ nk_size_t row_start, nk_size_t row_count);
1330
+
1331
+ /** @copydoc nk_angulars_packed_i4 */
1332
+ NK_PUBLIC void nk_angulars_packed_i4_icelake(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1333
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1334
+ nk_size_t r_stride_in_bytes);
1335
+ /** @copydoc nk_angulars_symmetric_i4 */
1336
+ NK_PUBLIC void nk_angulars_symmetric_i4_icelake(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1337
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1338
+ nk_size_t row_start, nk_size_t row_count);
1339
+ /** @copydoc nk_euclideans_packed_i4 */
1340
+ NK_PUBLIC void nk_euclideans_packed_i4_icelake(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result,
1341
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1342
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1343
+ /** @copydoc nk_euclideans_symmetric_i4 */
1344
+ NK_PUBLIC void nk_euclideans_symmetric_i4_icelake(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1345
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1346
+ nk_size_t row_start, nk_size_t row_count);
1347
+
1348
+ /** @copydoc nk_angulars_packed_u4 */
1349
+ NK_PUBLIC void nk_angulars_packed_u4_icelake(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1350
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1351
+ nk_size_t r_stride_in_bytes);
1352
+ /** @copydoc nk_angulars_symmetric_u4 */
1353
+ NK_PUBLIC void nk_angulars_symmetric_u4_icelake(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1354
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1355
+ nk_size_t row_start, nk_size_t row_count);
1356
+ /** @copydoc nk_euclideans_packed_u4 */
1357
+ NK_PUBLIC void nk_euclideans_packed_u4_icelake(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result,
1358
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1359
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1360
+ /** @copydoc nk_euclideans_symmetric_u4 */
1361
+ NK_PUBLIC void nk_euclideans_symmetric_u4_icelake(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1362
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1363
+ nk_size_t row_start, nk_size_t row_count);
1364
+ #endif // NK_TARGET_ICELAKE
1365
+
1366
+ #if NK_TARGET_ALDER
1367
+ /** @copydoc nk_angulars_packed_i8 */
1368
+ NK_PUBLIC void nk_angulars_packed_i8_alder(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1369
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1370
+ nk_size_t r_stride_in_bytes);
1371
+ /** @copydoc nk_angulars_symmetric_i8 */
1372
+ NK_PUBLIC void nk_angulars_symmetric_i8_alder(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1373
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1374
+ nk_size_t row_start, nk_size_t row_count);
1375
+ /** @copydoc nk_euclideans_packed_i8 */
1376
+ NK_PUBLIC void nk_euclideans_packed_i8_alder(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1377
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1378
+ nk_size_t r_stride_in_bytes);
1379
+ /** @copydoc nk_euclideans_symmetric_i8 */
1380
+ NK_PUBLIC void nk_euclideans_symmetric_i8_alder(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1381
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1382
+ nk_size_t row_start, nk_size_t row_count);
1383
+ /** @copydoc nk_angulars_packed_u8 */
1384
+ NK_PUBLIC void nk_angulars_packed_u8_alder(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1385
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1386
+ nk_size_t r_stride_in_bytes);
1387
+ /** @copydoc nk_angulars_symmetric_u8 */
1388
+ NK_PUBLIC void nk_angulars_symmetric_u8_alder(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1389
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1390
+ nk_size_t row_start, nk_size_t row_count);
1391
+ /** @copydoc nk_euclideans_packed_u8 */
1392
+ NK_PUBLIC void nk_euclideans_packed_u8_alder(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1393
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1394
+ nk_size_t r_stride_in_bytes);
1395
+ /** @copydoc nk_euclideans_symmetric_u8 */
1396
+ NK_PUBLIC void nk_euclideans_symmetric_u8_alder(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1397
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1398
+ nk_size_t row_start, nk_size_t row_count);
1399
+ /** @copydoc nk_angulars_packed_e2m3 */
1400
+ NK_PUBLIC void nk_angulars_packed_e2m3_alder(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1401
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1402
+ nk_size_t r_stride_in_bytes);
1403
+ /** @copydoc nk_angulars_symmetric_e2m3 */
1404
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_alder(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1405
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1406
+ nk_size_t row_start, nk_size_t row_count);
1407
+ /** @copydoc nk_euclideans_packed_e2m3 */
1408
+ NK_PUBLIC void nk_euclideans_packed_e2m3_alder(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1409
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1410
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1411
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
1412
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_alder(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1413
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1414
+ nk_size_t row_start, nk_size_t row_count);
1415
+ #endif // NK_TARGET_ALDER
1416
+
1417
+ /* Sierra backends using AVX10.2 with VMPSADBW.
1418
+ * Optimized for I8/U8 via VMPSADBW (vector multiply-sum of absolute differences).
1419
+ */
1420
+ #if NK_TARGET_SIERRA
1421
+ /** @copydoc nk_angulars_packed_i8 */
1422
+ NK_PUBLIC void nk_angulars_packed_i8_sierra(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1423
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1424
+ nk_size_t r_stride_in_bytes);
1425
+ /** @copydoc nk_angulars_symmetric_i8 */
1426
+ NK_PUBLIC void nk_angulars_symmetric_i8_sierra(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1427
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1428
+ nk_size_t row_start, nk_size_t row_count);
1429
+ /** @copydoc nk_euclideans_packed_i8 */
1430
+ NK_PUBLIC void nk_euclideans_packed_i8_sierra(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1431
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1432
+ nk_size_t r_stride_in_bytes);
1433
+ /** @copydoc nk_euclideans_symmetric_i8 */
1434
+ NK_PUBLIC void nk_euclideans_symmetric_i8_sierra(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1435
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1436
+ nk_size_t row_start, nk_size_t row_count);
1437
+ /** @copydoc nk_angulars_packed_u8 */
1438
+ NK_PUBLIC void nk_angulars_packed_u8_sierra(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1439
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1440
+ nk_size_t r_stride_in_bytes);
1441
+ /** @copydoc nk_angulars_symmetric_u8 */
1442
+ NK_PUBLIC void nk_angulars_symmetric_u8_sierra(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1443
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1444
+ nk_size_t row_start, nk_size_t row_count);
1445
+ /** @copydoc nk_euclideans_packed_u8 */
1446
+ NK_PUBLIC void nk_euclideans_packed_u8_sierra(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1447
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1448
+ nk_size_t r_stride_in_bytes);
1449
+ /** @copydoc nk_euclideans_symmetric_u8 */
1450
+ NK_PUBLIC void nk_euclideans_symmetric_u8_sierra(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1451
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1452
+ nk_size_t row_start, nk_size_t row_count);
1453
+ /** @copydoc nk_angulars_packed_e2m3 */
1454
+ NK_PUBLIC void nk_angulars_packed_e2m3_sierra(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1455
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1456
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1457
+ /** @copydoc nk_angulars_symmetric_e2m3 */
1458
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_sierra(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1459
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1460
+ nk_size_t row_start, nk_size_t row_count);
1461
+ /** @copydoc nk_euclideans_packed_e2m3 */
1462
+ NK_PUBLIC void nk_euclideans_packed_e2m3_sierra(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1463
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1464
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1465
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
1466
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_sierra(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1467
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1468
+ nk_size_t row_start, nk_size_t row_count);
1469
+ #endif // NK_TARGET_SIERRA
1470
+
1471
+ /* WASM Relaxed SIMD backends for angular/euclidean distances.
1472
+ * Covers I8/U8/E2M3/BF16/F32/F64 spatial distance operations.
1473
+ */
1474
+ #if NK_TARGET_V128RELAXED
1475
+ /** @copydoc nk_angulars_packed_i8 */
1476
+ NK_PUBLIC void nk_angulars_packed_i8_v128relaxed(nk_i8_t const *a, void const *b_packed, nk_f32_t *result,
1477
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1478
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1479
+ /** @copydoc nk_angulars_symmetric_i8 */
1480
+ NK_PUBLIC void nk_angulars_symmetric_i8_v128relaxed(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1481
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1482
+ nk_size_t row_start, nk_size_t row_count);
1483
+ /** @copydoc nk_euclideans_packed_i8 */
1484
+ NK_PUBLIC void nk_euclideans_packed_i8_v128relaxed(nk_i8_t const *a, void const *b_packed, nk_f32_t *result,
1485
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1486
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1487
+ /** @copydoc nk_euclideans_symmetric_i8 */
1488
+ NK_PUBLIC void nk_euclideans_symmetric_i8_v128relaxed(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1489
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1490
+ nk_size_t row_start, nk_size_t row_count);
1491
+ /** @copydoc nk_angulars_packed_u8 */
1492
+ NK_PUBLIC void nk_angulars_packed_u8_v128relaxed(nk_u8_t const *a, void const *b_packed, nk_f32_t *result,
1493
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1494
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1495
+ /** @copydoc nk_angulars_symmetric_u8 */
1496
+ NK_PUBLIC void nk_angulars_symmetric_u8_v128relaxed(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1497
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1498
+ nk_size_t row_start, nk_size_t row_count);
1499
+ /** @copydoc nk_euclideans_packed_u8 */
1500
+ NK_PUBLIC void nk_euclideans_packed_u8_v128relaxed(nk_u8_t const *a, void const *b_packed, nk_f32_t *result,
1501
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1502
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1503
+ /** @copydoc nk_euclideans_symmetric_u8 */
1504
+ NK_PUBLIC void nk_euclideans_symmetric_u8_v128relaxed(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1505
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1506
+ nk_size_t row_start, nk_size_t row_count);
1507
+ /** @copydoc nk_angulars_packed_e2m3 */
1508
+ NK_PUBLIC void nk_angulars_packed_e2m3_v128relaxed(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1509
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1510
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1511
+ /** @copydoc nk_angulars_symmetric_e2m3 */
1512
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_v128relaxed(nk_e2m3_t const *vectors, nk_size_t vectors_count,
1513
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1514
+ nk_size_t result_stride, nk_size_t row_start,
1515
+ nk_size_t row_count);
1516
+ /** @copydoc nk_euclideans_packed_e2m3 */
1517
+ NK_PUBLIC void nk_euclideans_packed_e2m3_v128relaxed(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1518
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1519
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1520
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
1521
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_v128relaxed(nk_e2m3_t const *vectors, nk_size_t vectors_count,
1522
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1523
+ nk_size_t result_stride, nk_size_t row_start,
1524
+ nk_size_t row_count);
1525
+ /** @copydoc nk_angulars_packed_e4m3 */
1526
+ NK_PUBLIC void nk_angulars_packed_e4m3_v128relaxed(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1527
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1528
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1529
+ /** @copydoc nk_angulars_symmetric_e4m3 */
1530
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_v128relaxed(nk_e4m3_t const *vectors, nk_size_t vectors_count,
1531
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1532
+ nk_size_t result_stride, nk_size_t row_start,
1533
+ nk_size_t row_count);
1534
+ /** @copydoc nk_euclideans_packed_e4m3 */
1535
+ NK_PUBLIC void nk_euclideans_packed_e4m3_v128relaxed(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1536
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1537
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1538
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
1539
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_v128relaxed(nk_e4m3_t const *vectors, nk_size_t vectors_count,
1540
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1541
+ nk_size_t result_stride, nk_size_t row_start,
1542
+ nk_size_t row_count);
1543
+ /** @copydoc nk_angulars_packed_e5m2 */
1544
+ NK_PUBLIC void nk_angulars_packed_e5m2_v128relaxed(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1545
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1546
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1547
+ /** @copydoc nk_angulars_symmetric_e5m2 */
1548
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_v128relaxed(nk_e5m2_t const *vectors, nk_size_t vectors_count,
1549
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1550
+ nk_size_t result_stride, nk_size_t row_start,
1551
+ nk_size_t row_count);
1552
+ /** @copydoc nk_euclideans_packed_e5m2 */
1553
+ NK_PUBLIC void nk_euclideans_packed_e5m2_v128relaxed(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1554
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1555
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1556
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
1557
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_v128relaxed(nk_e5m2_t const *vectors, nk_size_t vectors_count,
1558
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1559
+ nk_size_t result_stride, nk_size_t row_start,
1560
+ nk_size_t row_count);
1561
+ /** @copydoc nk_angulars_packed_bf16 */
1562
+ NK_PUBLIC void nk_angulars_packed_bf16_v128relaxed(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1563
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1564
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1565
+ /** @copydoc nk_angulars_symmetric_bf16 */
1566
+ NK_PUBLIC void nk_angulars_symmetric_bf16_v128relaxed(nk_bf16_t const *vectors, nk_size_t vectors_count,
1567
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1568
+ nk_size_t result_stride, nk_size_t row_start,
1569
+ nk_size_t row_count);
1570
+ /** @copydoc nk_euclideans_packed_bf16 */
1571
+ NK_PUBLIC void nk_euclideans_packed_bf16_v128relaxed(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1572
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1573
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1574
+ /** @copydoc nk_euclideans_symmetric_bf16 */
1575
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_v128relaxed(nk_bf16_t const *vectors, nk_size_t vectors_count,
1576
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1577
+ nk_size_t result_stride, nk_size_t row_start,
1578
+ nk_size_t row_count);
1579
+ /** @copydoc nk_angulars_packed_f32 */
1580
+ NK_PUBLIC void nk_angulars_packed_f32_v128relaxed(nk_f32_t const *a, void const *b_packed, nk_f64_t *result,
1581
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1582
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1583
+ /** @copydoc nk_angulars_symmetric_f32 */
1584
+ NK_PUBLIC void nk_angulars_symmetric_f32_v128relaxed(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1585
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1586
+ nk_size_t row_start, nk_size_t row_count);
1587
+ /** @copydoc nk_euclideans_packed_f32 */
1588
+ NK_PUBLIC void nk_euclideans_packed_f32_v128relaxed(nk_f32_t const *a, void const *b_packed, nk_f64_t *result,
1589
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1590
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1591
+ /** @copydoc nk_euclideans_symmetric_f32 */
1592
+ NK_PUBLIC void nk_euclideans_symmetric_f32_v128relaxed(nk_f32_t const *vectors, nk_size_t vectors_count,
1593
+ nk_size_t depth, nk_size_t stride, nk_f64_t *result,
1594
+ nk_size_t result_stride, nk_size_t row_start,
1595
+ nk_size_t row_count);
1596
+ /** @copydoc nk_angulars_packed_f64 */
1597
+ NK_PUBLIC void nk_angulars_packed_f64_v128relaxed(nk_f64_t const *a, void const *b_packed, nk_f64_t *result,
1598
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1599
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1600
+ /** @copydoc nk_angulars_symmetric_f64 */
1601
+ NK_PUBLIC void nk_angulars_symmetric_f64_v128relaxed(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1602
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1603
+ nk_size_t row_start, nk_size_t row_count);
1604
+ /** @copydoc nk_euclideans_packed_f64 */
1605
+ NK_PUBLIC void nk_euclideans_packed_f64_v128relaxed(nk_f64_t const *a, void const *b_packed, nk_f64_t *result,
1606
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1607
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1608
+ /** @copydoc nk_euclideans_symmetric_f64 */
1609
+ NK_PUBLIC void nk_euclideans_symmetric_f64_v128relaxed(nk_f64_t const *vectors, nk_size_t vectors_count,
1610
+ nk_size_t depth, nk_size_t stride, nk_f64_t *result,
1611
+ nk_size_t result_stride, nk_size_t row_start,
1612
+ nk_size_t row_count);
1613
+ #endif // NK_TARGET_V128RELAXED
1614
+
1615
+ /* ARM NEON backends (base NEON with F32/F64 support).
1616
+ * Uses FMLA for F32 dots, FMLA (scalar) for F64.
1617
+ */
1618
+ #if NK_TARGET_NEON
1619
+ /** @copydoc nk_angulars_packed_f32 */
1620
+ NK_PUBLIC void nk_angulars_packed_f32_neon(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1621
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1622
+ nk_size_t r_stride_in_bytes);
1623
+ /** @copydoc nk_angulars_symmetric_f32 */
1624
+ NK_PUBLIC void nk_angulars_symmetric_f32_neon(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1625
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1626
+ nk_size_t row_start, nk_size_t row_count);
1627
+ /** @copydoc nk_euclideans_packed_f32 */
1628
+ NK_PUBLIC void nk_euclideans_packed_f32_neon(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1629
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1630
+ nk_size_t r_stride_in_bytes);
1631
+ /** @copydoc nk_euclideans_symmetric_f32 */
1632
+ NK_PUBLIC void nk_euclideans_symmetric_f32_neon(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1633
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1634
+ nk_size_t row_start, nk_size_t row_count);
1635
+
1636
+ /** @copydoc nk_angulars_packed_f64 */
1637
+ NK_PUBLIC void nk_angulars_packed_f64_neon(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1638
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1639
+ nk_size_t r_stride_in_bytes);
1640
+ /** @copydoc nk_angulars_symmetric_f64 */
1641
+ NK_PUBLIC void nk_angulars_symmetric_f64_neon(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1642
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1643
+ nk_size_t row_start, nk_size_t row_count);
1644
+ /** @copydoc nk_euclideans_packed_f64 */
1645
+ NK_PUBLIC void nk_euclideans_packed_f64_neon(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1646
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1647
+ nk_size_t r_stride_in_bytes);
1648
+ /** @copydoc nk_euclideans_symmetric_f64 */
1649
+ NK_PUBLIC void nk_euclideans_symmetric_f64_neon(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1650
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1651
+ nk_size_t row_start, nk_size_t row_count);
1652
+ /** @copydoc nk_angulars_packed_bf16 */
1653
+ NK_PUBLIC void nk_angulars_packed_bf16_neon(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1654
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1655
+ nk_size_t r_stride_in_bytes);
1656
+ /** @copydoc nk_angulars_symmetric_bf16 */
1657
+ NK_PUBLIC void nk_angulars_symmetric_bf16_neon(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1658
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1659
+ nk_size_t row_start, nk_size_t row_count);
1660
+ /** @copydoc nk_euclideans_packed_bf16 */
1661
+ NK_PUBLIC void nk_euclideans_packed_bf16_neon(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1662
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1663
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1664
+ /** @copydoc nk_euclideans_symmetric_bf16 */
1665
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_neon(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1666
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1667
+ nk_size_t row_start, nk_size_t row_count);
1668
+ /** @copydoc nk_angulars_packed_f16 */
1669
+ NK_PUBLIC void nk_angulars_packed_f16_neon(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1670
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1671
+ nk_size_t r_stride_in_bytes);
1672
+ /** @copydoc nk_angulars_symmetric_f16 */
1673
+ NK_PUBLIC void nk_angulars_symmetric_f16_neon(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1674
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1675
+ nk_size_t row_start, nk_size_t row_count);
1676
+ /** @copydoc nk_euclideans_packed_f16 */
1677
+ NK_PUBLIC void nk_euclideans_packed_f16_neon(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1678
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1679
+ nk_size_t r_stride_in_bytes);
1680
+ /** @copydoc nk_euclideans_symmetric_f16 */
1681
+ NK_PUBLIC void nk_euclideans_symmetric_f16_neon(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1682
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1683
+ nk_size_t row_start, nk_size_t row_count);
1684
+ #endif // NK_TARGET_NEON
1685
+
1686
+ /* ARM NEON with BF16 dot product (ARMv8.6-A BF16).
1687
+ * Uses BFDOT/BFMMLA for efficient BF16 matrix operations.
1688
+ */
1689
+ #if NK_TARGET_NEONBFDOT
1690
+ /** @copydoc nk_angulars_packed_bf16 */
1691
+ NK_PUBLIC void nk_angulars_packed_bf16_neonbfdot(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1692
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1693
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1694
+ /** @copydoc nk_angulars_symmetric_bf16 */
1695
+ NK_PUBLIC void nk_angulars_symmetric_bf16_neonbfdot(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1696
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1697
+ nk_size_t row_start, nk_size_t row_count);
1698
+ /** @copydoc nk_euclideans_packed_bf16 */
1699
+ NK_PUBLIC void nk_euclideans_packed_bf16_neonbfdot(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result,
1700
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1701
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1702
+ /** @copydoc nk_euclideans_symmetric_bf16 */
1703
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_neonbfdot(nk_bf16_t const *vectors, nk_size_t vectors_count,
1704
+ nk_size_t depth, nk_size_t stride, nk_f32_t *result,
1705
+ nk_size_t result_stride, nk_size_t row_start,
1706
+ nk_size_t row_count);
1707
+ #endif // NK_TARGET_NEONBFDOT
1708
+
1709
+ /* ARM NEON with signed/unsigned dot product (ARMv8.2-A DotProd).
1710
+ * Provides SDOT/UDOT for I8/U8 vector dot products.
1711
+ */
1712
+ #if NK_TARGET_NEONSDOT
1713
+ /** @copydoc nk_angulars_packed_i8 */
1714
+ NK_PUBLIC void nk_angulars_packed_i8_neonsdot(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1715
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1716
+ nk_size_t r_stride_in_bytes);
1717
+ /** @copydoc nk_angulars_symmetric_i8 */
1718
+ NK_PUBLIC void nk_angulars_symmetric_i8_neonsdot(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1719
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1720
+ nk_size_t row_start, nk_size_t row_count);
1721
+ /** @copydoc nk_euclideans_packed_i8 */
1722
+ NK_PUBLIC void nk_euclideans_packed_i8_neonsdot(nk_i8_t const *a, void const *b_packed, nk_f32_t *result,
1723
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1724
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1725
+ /** @copydoc nk_euclideans_symmetric_i8 */
1726
+ NK_PUBLIC void nk_euclideans_symmetric_i8_neonsdot(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1727
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1728
+ nk_size_t row_start, nk_size_t row_count);
1729
+
1730
+ /** @copydoc nk_angulars_packed_u8 */
1731
+ NK_PUBLIC void nk_angulars_packed_u8_neonsdot(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1732
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1733
+ nk_size_t r_stride_in_bytes);
1734
+ /** @copydoc nk_angulars_symmetric_u8 */
1735
+ NK_PUBLIC void nk_angulars_symmetric_u8_neonsdot(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1736
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1737
+ nk_size_t row_start, nk_size_t row_count);
1738
+ /** @copydoc nk_euclideans_packed_u8 */
1739
+ NK_PUBLIC void nk_euclideans_packed_u8_neonsdot(nk_u8_t const *a, void const *b_packed, nk_f32_t *result,
1740
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1741
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1742
+ /** @copydoc nk_euclideans_symmetric_u8 */
1743
+ NK_PUBLIC void nk_euclideans_symmetric_u8_neonsdot(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1744
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1745
+ nk_size_t row_start, nk_size_t row_count);
1746
+
1747
+ /** @copydoc nk_angulars_packed_i4 */
1748
+ NK_PUBLIC void nk_angulars_packed_i4_neonsdot(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result,
1749
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1750
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1751
+ /** @copydoc nk_angulars_symmetric_i4 */
1752
+ NK_PUBLIC void nk_angulars_symmetric_i4_neonsdot(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1753
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1754
+ nk_size_t row_start, nk_size_t row_count);
1755
+ /** @copydoc nk_euclideans_packed_i4 */
1756
+ NK_PUBLIC void nk_euclideans_packed_i4_neonsdot(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result,
1757
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1758
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1759
+ /** @copydoc nk_euclideans_symmetric_i4 */
1760
+ NK_PUBLIC void nk_euclideans_symmetric_i4_neonsdot(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1761
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1762
+ nk_size_t row_start, nk_size_t row_count);
1763
+
1764
+ /** @copydoc nk_angulars_packed_u4 */
1765
+ NK_PUBLIC void nk_angulars_packed_u4_neonsdot(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result,
1766
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1767
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1768
+ /** @copydoc nk_angulars_symmetric_u4 */
1769
+ NK_PUBLIC void nk_angulars_symmetric_u4_neonsdot(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1770
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1771
+ nk_size_t row_start, nk_size_t row_count);
1772
+ /** @copydoc nk_euclideans_packed_u4 */
1773
+ NK_PUBLIC void nk_euclideans_packed_u4_neonsdot(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result,
1774
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1775
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1776
+ /** @copydoc nk_euclideans_symmetric_u4 */
1777
+ NK_PUBLIC void nk_euclideans_symmetric_u4_neonsdot(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1778
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1779
+ nk_size_t row_start, nk_size_t row_count);
1780
+ #endif // NK_TARGET_NEONSDOT
1781
+
1782
+ /* ARM NEON with FP16 FML (fused multiply-long, ARMv8.2-A FP16FML).
1783
+ * Uses FMLAL/FMLSL for F16 and custom FP8 (E2M3/E3M2) operations.
1784
+ */
1785
+ #if NK_TARGET_NEONFHM
1786
+ /** @copydoc nk_angulars_packed_f16 */
1787
+ NK_PUBLIC void nk_angulars_packed_f16_neonfhm(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1788
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1789
+ nk_size_t r_stride_in_bytes);
1790
+ /** @copydoc nk_angulars_symmetric_f16 */
1791
+ NK_PUBLIC void nk_angulars_symmetric_f16_neonfhm(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1792
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1793
+ nk_size_t row_start, nk_size_t row_count);
1794
+ /** @copydoc nk_euclideans_packed_f16 */
1795
+ NK_PUBLIC void nk_euclideans_packed_f16_neonfhm(nk_f16_t const *a, void const *b_packed, nk_f32_t *result,
1796
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1797
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1798
+ /** @copydoc nk_euclideans_symmetric_f16 */
1799
+ NK_PUBLIC void nk_euclideans_symmetric_f16_neonfhm(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1800
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1801
+ nk_size_t row_start, nk_size_t row_count);
1802
+
1803
+ /** @copydoc nk_angulars_packed_e4m3 */
1804
+ NK_PUBLIC void nk_angulars_packed_e4m3_neonfhm(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1805
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1806
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1807
+ /** @copydoc nk_angulars_symmetric_e4m3 */
1808
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_neonfhm(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1809
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1810
+ nk_size_t row_start, nk_size_t row_count);
1811
+ /** @copydoc nk_euclideans_packed_e4m3 */
1812
+ NK_PUBLIC void nk_euclideans_packed_e4m3_neonfhm(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1813
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1814
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1815
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
1816
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_neonfhm(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1817
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1818
+ nk_size_t row_start, nk_size_t row_count);
1819
+
1820
+ /** @copydoc nk_angulars_packed_e5m2 */
1821
+ NK_PUBLIC void nk_angulars_packed_e5m2_neonfhm(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1822
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1823
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1824
+ /** @copydoc nk_angulars_symmetric_e5m2 */
1825
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_neonfhm(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1826
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1827
+ nk_size_t row_start, nk_size_t row_count);
1828
+ /** @copydoc nk_euclideans_packed_e5m2 */
1829
+ NK_PUBLIC void nk_euclideans_packed_e5m2_neonfhm(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1830
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1831
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1832
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
1833
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_neonfhm(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1834
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1835
+ nk_size_t row_start, nk_size_t row_count);
1836
+
1837
+ #endif // NK_TARGET_NEONFHM
1838
+
1839
+ /* ARM NEON with FP8 (ARMv9.2-A FP8).
1840
+ * Uses native FP8 dot-product instructions for E4M3/E5M2/E2M3/E3M2 operations.
1841
+ */
1842
+ #if NK_TARGET_NEONFP8
1843
+ /** @copydoc nk_angulars_packed_e4m3 */
1844
+ NK_PUBLIC void nk_angulars_packed_e4m3_neonfp8(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1845
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1846
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1847
+ /** @copydoc nk_angulars_symmetric_e4m3 */
1848
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_neonfp8(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1849
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1850
+ nk_size_t row_start, nk_size_t row_count);
1851
+ /** @copydoc nk_euclideans_packed_e4m3 */
1852
+ NK_PUBLIC void nk_euclideans_packed_e4m3_neonfp8(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result,
1853
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1854
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1855
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
1856
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_neonfp8(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1857
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1858
+ nk_size_t row_start, nk_size_t row_count);
1859
+
1860
+ /** @copydoc nk_angulars_packed_e5m2 */
1861
+ NK_PUBLIC void nk_angulars_packed_e5m2_neonfp8(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1862
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1863
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1864
+ /** @copydoc nk_angulars_symmetric_e5m2 */
1865
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_neonfp8(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1866
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1867
+ nk_size_t row_start, nk_size_t row_count);
1868
+ /** @copydoc nk_euclideans_packed_e5m2 */
1869
+ NK_PUBLIC void nk_euclideans_packed_e5m2_neonfp8(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result,
1870
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1871
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1872
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
1873
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_neonfp8(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1874
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1875
+ nk_size_t row_start, nk_size_t row_count);
1876
+
1877
+ /** @copydoc nk_angulars_packed_e2m3 */
1878
+ NK_PUBLIC void nk_angulars_packed_e2m3_neonfp8(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1879
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1880
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1881
+ /** @copydoc nk_angulars_symmetric_e2m3 */
1882
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_neonfp8(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1883
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1884
+ nk_size_t row_start, nk_size_t row_count);
1885
+ /** @copydoc nk_euclideans_packed_e2m3 */
1886
+ NK_PUBLIC void nk_euclideans_packed_e2m3_neonfp8(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result,
1887
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1888
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1889
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
1890
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_neonfp8(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1891
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1892
+ nk_size_t row_start, nk_size_t row_count);
1893
+
1894
+ /** @copydoc nk_angulars_packed_e3m2 */
1895
+ NK_PUBLIC void nk_angulars_packed_e3m2_neonfp8(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
1896
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1897
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1898
+ /** @copydoc nk_angulars_symmetric_e3m2 */
1899
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_neonfp8(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1900
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1901
+ nk_size_t row_start, nk_size_t row_count);
1902
+ /** @copydoc nk_euclideans_packed_e3m2 */
1903
+ NK_PUBLIC void nk_euclideans_packed_e3m2_neonfp8(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result,
1904
+ nk_size_t rows, nk_size_t cols, nk_size_t depth,
1905
+ nk_size_t a_stride_in_bytes, nk_size_t r_stride_in_bytes);
1906
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
1907
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_neonfp8(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1908
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1909
+ nk_size_t row_start, nk_size_t row_count);
1910
+
1911
+ #endif // NK_TARGET_NEONFP8
1912
+
1913
+ #if NK_TARGET_RVV
1914
+ /** @copydoc nk_angulars_packed_f32 */
1915
+ NK_PUBLIC void nk_angulars_packed_f32_rvv(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1916
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1917
+ nk_size_t r_stride_in_bytes);
1918
+ /** @copydoc nk_angulars_symmetric_f32 */
1919
+ NK_PUBLIC void nk_angulars_symmetric_f32_rvv(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1920
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1921
+ nk_size_t row_start, nk_size_t row_count);
1922
+ /** @copydoc nk_euclideans_packed_f32 */
1923
+ NK_PUBLIC void nk_euclideans_packed_f32_rvv(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1924
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1925
+ nk_size_t r_stride_in_bytes);
1926
+ /** @copydoc nk_euclideans_symmetric_f32 */
1927
+ NK_PUBLIC void nk_euclideans_symmetric_f32_rvv(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1928
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1929
+ nk_size_t row_start, nk_size_t row_count);
1930
+
1931
+ /** @copydoc nk_angulars_packed_f64 */
1932
+ NK_PUBLIC void nk_angulars_packed_f64_rvv(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1933
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1934
+ nk_size_t r_stride_in_bytes);
1935
+ /** @copydoc nk_angulars_symmetric_f64 */
1936
+ NK_PUBLIC void nk_angulars_symmetric_f64_rvv(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1937
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1938
+ nk_size_t row_start, nk_size_t row_count);
1939
+ /** @copydoc nk_euclideans_packed_f64 */
1940
+ NK_PUBLIC void nk_euclideans_packed_f64_rvv(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
1941
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1942
+ nk_size_t r_stride_in_bytes);
1943
+ /** @copydoc nk_euclideans_symmetric_f64 */
1944
+ NK_PUBLIC void nk_euclideans_symmetric_f64_rvv(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1945
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
1946
+ nk_size_t row_start, nk_size_t row_count);
1947
+
1948
+ /** @copydoc nk_angulars_packed_f16 */
1949
+ NK_PUBLIC void nk_angulars_packed_f16_rvv(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1950
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1951
+ nk_size_t r_stride_in_bytes);
1952
+ /** @copydoc nk_angulars_symmetric_f16 */
1953
+ NK_PUBLIC void nk_angulars_symmetric_f16_rvv(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1954
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1955
+ nk_size_t row_start, nk_size_t row_count);
1956
+ /** @copydoc nk_euclideans_packed_f16 */
1957
+ NK_PUBLIC void nk_euclideans_packed_f16_rvv(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1958
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1959
+ nk_size_t r_stride_in_bytes);
1960
+ /** @copydoc nk_euclideans_symmetric_f16 */
1961
+ NK_PUBLIC void nk_euclideans_symmetric_f16_rvv(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1962
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1963
+ nk_size_t row_start, nk_size_t row_count);
1964
+
1965
+ /** @copydoc nk_angulars_packed_bf16 */
1966
+ NK_PUBLIC void nk_angulars_packed_bf16_rvv(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1967
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1968
+ nk_size_t r_stride_in_bytes);
1969
+ /** @copydoc nk_angulars_symmetric_bf16 */
1970
+ NK_PUBLIC void nk_angulars_symmetric_bf16_rvv(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1971
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1972
+ nk_size_t row_start, nk_size_t row_count);
1973
+ /** @copydoc nk_euclideans_packed_bf16 */
1974
+ NK_PUBLIC void nk_euclideans_packed_bf16_rvv(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1975
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1976
+ nk_size_t r_stride_in_bytes);
1977
+ /** @copydoc nk_euclideans_symmetric_bf16 */
1978
+ NK_PUBLIC void nk_euclideans_symmetric_bf16_rvv(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1979
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1980
+ nk_size_t row_start, nk_size_t row_count);
1981
+
1982
+ /** @copydoc nk_angulars_packed_e4m3 */
1983
+ NK_PUBLIC void nk_angulars_packed_e4m3_rvv(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1984
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1985
+ nk_size_t r_stride_in_bytes);
1986
+ /** @copydoc nk_angulars_symmetric_e4m3 */
1987
+ NK_PUBLIC void nk_angulars_symmetric_e4m3_rvv(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1988
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1989
+ nk_size_t row_start, nk_size_t row_count);
1990
+ /** @copydoc nk_euclideans_packed_e4m3 */
1991
+ NK_PUBLIC void nk_euclideans_packed_e4m3_rvv(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
1992
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
1993
+ nk_size_t r_stride_in_bytes);
1994
+ /** @copydoc nk_euclideans_symmetric_e4m3 */
1995
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3_rvv(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
1996
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
1997
+ nk_size_t row_start, nk_size_t row_count);
1998
+
1999
+ /** @copydoc nk_angulars_packed_e5m2 */
2000
+ NK_PUBLIC void nk_angulars_packed_e5m2_rvv(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2001
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2002
+ nk_size_t r_stride_in_bytes);
2003
+ /** @copydoc nk_angulars_symmetric_e5m2 */
2004
+ NK_PUBLIC void nk_angulars_symmetric_e5m2_rvv(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2005
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2006
+ nk_size_t row_start, nk_size_t row_count);
2007
+ /** @copydoc nk_euclideans_packed_e5m2 */
2008
+ NK_PUBLIC void nk_euclideans_packed_e5m2_rvv(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2009
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2010
+ nk_size_t r_stride_in_bytes);
2011
+ /** @copydoc nk_euclideans_symmetric_e5m2 */
2012
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2_rvv(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2013
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2014
+ nk_size_t row_start, nk_size_t row_count);
2015
+
2016
+ /** @copydoc nk_angulars_packed_e2m3 */
2017
+ NK_PUBLIC void nk_angulars_packed_e2m3_rvv(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2018
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2019
+ nk_size_t r_stride_in_bytes);
2020
+ /** @copydoc nk_angulars_symmetric_e2m3 */
2021
+ NK_PUBLIC void nk_angulars_symmetric_e2m3_rvv(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2022
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2023
+ nk_size_t row_start, nk_size_t row_count);
2024
+ /** @copydoc nk_euclideans_packed_e2m3 */
2025
+ NK_PUBLIC void nk_euclideans_packed_e2m3_rvv(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2026
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2027
+ nk_size_t r_stride_in_bytes);
2028
+ /** @copydoc nk_euclideans_symmetric_e2m3 */
2029
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3_rvv(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2030
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2031
+ nk_size_t row_start, nk_size_t row_count);
2032
+
2033
+ /** @copydoc nk_angulars_packed_e3m2 */
2034
+ NK_PUBLIC void nk_angulars_packed_e3m2_rvv(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2035
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2036
+ nk_size_t r_stride_in_bytes);
2037
+ /** @copydoc nk_angulars_symmetric_e3m2 */
2038
+ NK_PUBLIC void nk_angulars_symmetric_e3m2_rvv(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2039
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2040
+ nk_size_t row_start, nk_size_t row_count);
2041
+ /** @copydoc nk_euclideans_packed_e3m2 */
2042
+ NK_PUBLIC void nk_euclideans_packed_e3m2_rvv(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2043
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2044
+ nk_size_t r_stride_in_bytes);
2045
+ /** @copydoc nk_euclideans_symmetric_e3m2 */
2046
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2_rvv(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2047
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2048
+ nk_size_t row_start, nk_size_t row_count);
2049
+
2050
+ /** @copydoc nk_angulars_packed_i8 */
2051
+ NK_PUBLIC void nk_angulars_packed_i8_rvv(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2052
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2053
+ nk_size_t r_stride_in_bytes);
2054
+ /** @copydoc nk_angulars_symmetric_i8 */
2055
+ NK_PUBLIC void nk_angulars_symmetric_i8_rvv(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2056
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2057
+ nk_size_t row_start, nk_size_t row_count);
2058
+ /** @copydoc nk_euclideans_packed_i8 */
2059
+ NK_PUBLIC void nk_euclideans_packed_i8_rvv(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2060
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2061
+ nk_size_t r_stride_in_bytes);
2062
+ /** @copydoc nk_euclideans_symmetric_i8 */
2063
+ NK_PUBLIC void nk_euclideans_symmetric_i8_rvv(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2064
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2065
+ nk_size_t row_start, nk_size_t row_count);
2066
+
2067
+ /** @copydoc nk_angulars_packed_u8 */
2068
+ NK_PUBLIC void nk_angulars_packed_u8_rvv(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2069
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2070
+ nk_size_t r_stride_in_bytes);
2071
+ /** @copydoc nk_angulars_symmetric_u8 */
2072
+ NK_PUBLIC void nk_angulars_symmetric_u8_rvv(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2073
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2074
+ nk_size_t row_start, nk_size_t row_count);
2075
+ /** @copydoc nk_euclideans_packed_u8 */
2076
+ NK_PUBLIC void nk_euclideans_packed_u8_rvv(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2077
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2078
+ nk_size_t r_stride_in_bytes);
2079
+ /** @copydoc nk_euclideans_symmetric_u8 */
2080
+ NK_PUBLIC void nk_euclideans_symmetric_u8_rvv(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2081
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2082
+ nk_size_t row_start, nk_size_t row_count);
2083
+ #endif // NK_TARGET_RVV
2084
+
2085
+ #if defined(__cplusplus)
2086
+ } // extern "C"
2087
+ #endif
2088
+
2089
+ #include "numkong/spatials/serial.h"
2090
+ #include "numkong/spatials/neon.h"
2091
+ #include "numkong/spatials/neonfhm.h"
2092
+ #include "numkong/spatials/neonfp8.h"
2093
+ #include "numkong/spatials/neonbfdot.h"
2094
+ #include "numkong/spatials/neonsdot.h"
2095
+ #include "numkong/spatials/haswell.h"
2096
+ #include "numkong/spatials/skylake.h"
2097
+ #include "numkong/spatials/genoa.h"
2098
+ #include "numkong/spatials/diamond.h"
2099
+ #include "numkong/spatials/icelake.h"
2100
+ #include "numkong/spatials/alder.h"
2101
+ #include "numkong/spatials/sierra.h"
2102
+ #include "numkong/spatials/sapphireamx.h"
2103
+ #include "numkong/spatials/graniteamx.h"
2104
+ #include "numkong/spatials/rvv.h"
2105
+ #include "numkong/spatials/v128relaxed.h"
2106
+ #include "numkong/spatials/sme.h"
2107
+ #include "numkong/spatials/smef64.h"
2108
+ #include "numkong/spatials/powervsx.h"
2109
+ #include "numkong/spatials/loongsonasx.h"
2110
+
2111
+ #if defined(__cplusplus)
2112
+ extern "C" {
2113
+ #endif
2114
+
2115
+ #if !NK_DYNAMIC_DISPATCH
2116
+
2117
+ NK_PUBLIC void nk_angulars_packed_f64(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
2118
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2119
+ nk_size_t r_stride_in_bytes) {
2120
+ #if NK_TARGET_SMEF64
2121
+ nk_angulars_packed_f64_smef64(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2122
+ #elif NK_TARGET_NEON
2123
+ nk_angulars_packed_f64_neon(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2124
+ #elif NK_TARGET_SKYLAKE
2125
+ nk_angulars_packed_f64_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2126
+ #elif NK_TARGET_HASWELL
2127
+ nk_angulars_packed_f64_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2128
+ #elif NK_TARGET_POWERVSX
2129
+ nk_angulars_packed_f64_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2130
+ #elif NK_TARGET_RVV
2131
+ nk_angulars_packed_f64_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2132
+ #elif NK_TARGET_V128RELAXED
2133
+ nk_angulars_packed_f64_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2134
+ #else
2135
+ nk_angulars_packed_f64_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2136
+ #endif
2137
+ }
2138
+ NK_PUBLIC void nk_angulars_symmetric_f64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2139
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
2140
+ nk_size_t row_start, nk_size_t row_count) {
2141
+ #if NK_TARGET_SMEF64
2142
+ nk_angulars_symmetric_f64_smef64(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2143
+ row_count);
2144
+ #elif NK_TARGET_NEON
2145
+ nk_angulars_symmetric_f64_neon(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2146
+ #elif NK_TARGET_SKYLAKE
2147
+ nk_angulars_symmetric_f64_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2148
+ row_count);
2149
+ #elif NK_TARGET_HASWELL
2150
+ nk_angulars_symmetric_f64_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2151
+ row_count);
2152
+ #elif NK_TARGET_POWERVSX
2153
+ nk_angulars_symmetric_f64_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2154
+ row_count);
2155
+ #elif NK_TARGET_RVV
2156
+ nk_angulars_symmetric_f64_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2157
+ #elif NK_TARGET_V128RELAXED
2158
+ nk_angulars_symmetric_f64_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2159
+ row_count);
2160
+ #else
2161
+ nk_angulars_symmetric_f64_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2162
+ row_count);
2163
+ #endif
2164
+ }
2165
+ NK_PUBLIC void nk_euclideans_packed_f64(nk_f64_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
2166
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2167
+ nk_size_t r_stride_in_bytes) {
2168
+ #if NK_TARGET_SMEF64
2169
+ nk_euclideans_packed_f64_smef64(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2170
+ #elif NK_TARGET_NEON
2171
+ nk_euclideans_packed_f64_neon(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2172
+ #elif NK_TARGET_SKYLAKE
2173
+ nk_euclideans_packed_f64_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2174
+ #elif NK_TARGET_HASWELL
2175
+ nk_euclideans_packed_f64_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2176
+ #elif NK_TARGET_POWERVSX
2177
+ nk_euclideans_packed_f64_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2178
+ #elif NK_TARGET_RVV
2179
+ nk_euclideans_packed_f64_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2180
+ #elif NK_TARGET_V128RELAXED
2181
+ nk_euclideans_packed_f64_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2182
+ #else
2183
+ nk_euclideans_packed_f64_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2184
+ #endif
2185
+ }
2186
+ NK_PUBLIC void nk_euclideans_symmetric_f64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2187
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
2188
+ nk_size_t row_start, nk_size_t row_count) {
2189
+ #if NK_TARGET_SMEF64
2190
+ nk_euclideans_symmetric_f64_smef64(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2191
+ row_count);
2192
+ #elif NK_TARGET_NEON
2193
+ nk_euclideans_symmetric_f64_neon(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2194
+ row_count);
2195
+ #elif NK_TARGET_SKYLAKE
2196
+ nk_euclideans_symmetric_f64_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2197
+ row_count);
2198
+ #elif NK_TARGET_HASWELL
2199
+ nk_euclideans_symmetric_f64_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2200
+ row_count);
2201
+ #elif NK_TARGET_POWERVSX
2202
+ nk_euclideans_symmetric_f64_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2203
+ row_count);
2204
+ #elif NK_TARGET_RVV
2205
+ nk_euclideans_symmetric_f64_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2206
+ #elif NK_TARGET_V128RELAXED
2207
+ nk_euclideans_symmetric_f64_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2208
+ row_count);
2209
+ #else
2210
+ nk_euclideans_symmetric_f64_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2211
+ row_count);
2212
+ #endif
2213
+ }
2214
+
2215
+ NK_PUBLIC void nk_angulars_packed_f32(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
2216
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2217
+ nk_size_t r_stride_in_bytes) {
2218
+ #if NK_TARGET_SMEF64
2219
+ nk_angulars_packed_f32_smef64(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2220
+ #elif NK_TARGET_NEON
2221
+ nk_angulars_packed_f32_neon(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2222
+ #elif NK_TARGET_SKYLAKE
2223
+ nk_angulars_packed_f32_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2224
+ #elif NK_TARGET_HASWELL
2225
+ nk_angulars_packed_f32_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2226
+ #elif NK_TARGET_POWERVSX
2227
+ nk_angulars_packed_f32_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2228
+ #elif NK_TARGET_RVV
2229
+ nk_angulars_packed_f32_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2230
+ #elif NK_TARGET_V128RELAXED
2231
+ nk_angulars_packed_f32_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2232
+ #else
2233
+ nk_angulars_packed_f32_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2234
+ #endif
2235
+ }
2236
+ NK_PUBLIC void nk_angulars_symmetric_f32(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2237
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
2238
+ nk_size_t row_start, nk_size_t row_count) {
2239
+ #if NK_TARGET_SMEF64
2240
+ nk_angulars_symmetric_f32_smef64(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2241
+ row_count);
2242
+ #elif NK_TARGET_NEON
2243
+ nk_angulars_symmetric_f32_neon(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2244
+ #elif NK_TARGET_SKYLAKE
2245
+ nk_angulars_symmetric_f32_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2246
+ row_count);
2247
+ #elif NK_TARGET_HASWELL
2248
+ nk_angulars_symmetric_f32_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2249
+ row_count);
2250
+ #elif NK_TARGET_POWERVSX
2251
+ nk_angulars_symmetric_f32_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2252
+ row_count);
2253
+ #elif NK_TARGET_RVV
2254
+ nk_angulars_symmetric_f32_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2255
+ #elif NK_TARGET_V128RELAXED
2256
+ nk_angulars_symmetric_f32_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2257
+ row_count);
2258
+ #else
2259
+ nk_angulars_symmetric_f32_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2260
+ row_count);
2261
+ #endif
2262
+ }
2263
+ NK_PUBLIC void nk_euclideans_packed_f32(nk_f32_t const *a, void const *b_packed, nk_f64_t *result, nk_size_t rows,
2264
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2265
+ nk_size_t r_stride_in_bytes) {
2266
+ #if NK_TARGET_SMEF64
2267
+ nk_euclideans_packed_f32_smef64(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2268
+ #elif NK_TARGET_NEON
2269
+ nk_euclideans_packed_f32_neon(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2270
+ #elif NK_TARGET_SKYLAKE
2271
+ nk_euclideans_packed_f32_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2272
+ #elif NK_TARGET_HASWELL
2273
+ nk_euclideans_packed_f32_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2274
+ #elif NK_TARGET_POWERVSX
2275
+ nk_euclideans_packed_f32_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2276
+ #elif NK_TARGET_RVV
2277
+ nk_euclideans_packed_f32_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2278
+ #elif NK_TARGET_V128RELAXED
2279
+ nk_euclideans_packed_f32_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2280
+ #else
2281
+ nk_euclideans_packed_f32_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2282
+ #endif
2283
+ }
2284
+ NK_PUBLIC void nk_euclideans_symmetric_f32(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2285
+ nk_size_t stride, nk_f64_t *result, nk_size_t result_stride,
2286
+ nk_size_t row_start, nk_size_t row_count) {
2287
+ #if NK_TARGET_SMEF64
2288
+ nk_euclideans_symmetric_f32_smef64(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2289
+ row_count);
2290
+ #elif NK_TARGET_NEON
2291
+ nk_euclideans_symmetric_f32_neon(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2292
+ row_count);
2293
+ #elif NK_TARGET_SKYLAKE
2294
+ nk_euclideans_symmetric_f32_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2295
+ row_count);
2296
+ #elif NK_TARGET_HASWELL
2297
+ nk_euclideans_symmetric_f32_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2298
+ row_count);
2299
+ #elif NK_TARGET_POWERVSX
2300
+ nk_euclideans_symmetric_f32_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2301
+ row_count);
2302
+ #elif NK_TARGET_RVV
2303
+ nk_euclideans_symmetric_f32_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2304
+ #elif NK_TARGET_V128RELAXED
2305
+ nk_euclideans_symmetric_f32_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2306
+ row_count);
2307
+ #else
2308
+ nk_euclideans_symmetric_f32_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2309
+ row_count);
2310
+ #endif
2311
+ }
2312
+
2313
+ NK_PUBLIC void nk_angulars_packed_f16(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2314
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2315
+ nk_size_t r_stride_in_bytes) {
2316
+ #if NK_TARGET_GRANITEAMX
2317
+ nk_angulars_packed_f16_graniteamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2318
+ #elif NK_TARGET_SME
2319
+ nk_angulars_packed_f16_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2320
+ #elif NK_TARGET_NEONFHM
2321
+ nk_angulars_packed_f16_neonfhm(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2322
+ #elif NK_TARGET_NEON
2323
+ nk_angulars_packed_f16_neon(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2324
+ #elif NK_TARGET_SKYLAKE
2325
+ nk_angulars_packed_f16_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2326
+ #elif NK_TARGET_HASWELL
2327
+ nk_angulars_packed_f16_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2328
+ #elif NK_TARGET_POWERVSX
2329
+ nk_angulars_packed_f16_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2330
+ #elif NK_TARGET_RVV
2331
+ nk_angulars_packed_f16_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2332
+ #else
2333
+ nk_angulars_packed_f16_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2334
+ #endif
2335
+ }
2336
+ NK_PUBLIC void nk_angulars_symmetric_f16(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2337
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2338
+ nk_size_t row_start, nk_size_t row_count) {
2339
+ #if NK_TARGET_GRANITEAMX
2340
+ nk_angulars_symmetric_f16_graniteamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2341
+ row_count);
2342
+ #elif NK_TARGET_SME
2343
+ nk_angulars_symmetric_f16_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2344
+ #elif NK_TARGET_NEONFHM
2345
+ nk_angulars_symmetric_f16_neonfhm(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2346
+ row_count);
2347
+ #elif NK_TARGET_NEON
2348
+ nk_angulars_symmetric_f16_neon(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2349
+ #elif NK_TARGET_SKYLAKE
2350
+ nk_angulars_symmetric_f16_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2351
+ row_count);
2352
+ #elif NK_TARGET_HASWELL
2353
+ nk_angulars_symmetric_f16_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2354
+ row_count);
2355
+ #elif NK_TARGET_POWERVSX
2356
+ nk_angulars_symmetric_f16_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2357
+ row_count);
2358
+ #elif NK_TARGET_RVV
2359
+ nk_angulars_symmetric_f16_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2360
+ #else
2361
+ nk_angulars_symmetric_f16_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2362
+ row_count);
2363
+ #endif
2364
+ }
2365
+ NK_PUBLIC void nk_euclideans_packed_f16(nk_f16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2366
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2367
+ nk_size_t r_stride_in_bytes) {
2368
+ #if NK_TARGET_GRANITEAMX
2369
+ nk_euclideans_packed_f16_graniteamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2370
+ #elif NK_TARGET_SME
2371
+ nk_euclideans_packed_f16_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2372
+ #elif NK_TARGET_NEONFHM
2373
+ nk_euclideans_packed_f16_neonfhm(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2374
+ #elif NK_TARGET_NEON
2375
+ nk_euclideans_packed_f16_neon(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2376
+ #elif NK_TARGET_SKYLAKE
2377
+ nk_euclideans_packed_f16_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2378
+ #elif NK_TARGET_HASWELL
2379
+ nk_euclideans_packed_f16_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2380
+ #elif NK_TARGET_POWERVSX
2381
+ nk_euclideans_packed_f16_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2382
+ #elif NK_TARGET_RVV
2383
+ nk_euclideans_packed_f16_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2384
+ #else
2385
+ nk_euclideans_packed_f16_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2386
+ #endif
2387
+ }
2388
+ NK_PUBLIC void nk_euclideans_symmetric_f16(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2389
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2390
+ nk_size_t row_start, nk_size_t row_count) {
2391
+ #if NK_TARGET_GRANITEAMX
2392
+ nk_euclideans_symmetric_f16_graniteamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2393
+ row_count);
2394
+ #elif NK_TARGET_SME
2395
+ nk_euclideans_symmetric_f16_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2396
+ #elif NK_TARGET_NEONFHM
2397
+ nk_euclideans_symmetric_f16_neonfhm(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2398
+ row_count);
2399
+ #elif NK_TARGET_NEON
2400
+ nk_euclideans_symmetric_f16_neon(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2401
+ row_count);
2402
+ #elif NK_TARGET_SKYLAKE
2403
+ nk_euclideans_symmetric_f16_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2404
+ row_count);
2405
+ #elif NK_TARGET_HASWELL
2406
+ nk_euclideans_symmetric_f16_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2407
+ row_count);
2408
+ #elif NK_TARGET_POWERVSX
2409
+ nk_euclideans_symmetric_f16_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2410
+ row_count);
2411
+ #elif NK_TARGET_RVV
2412
+ nk_euclideans_symmetric_f16_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2413
+ #else
2414
+ nk_euclideans_symmetric_f16_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2415
+ row_count);
2416
+ #endif
2417
+ }
2418
+
2419
+ NK_PUBLIC void nk_angulars_packed_bf16(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2420
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2421
+ nk_size_t r_stride_in_bytes) {
2422
+ #if NK_TARGET_SME
2423
+ nk_angulars_packed_bf16_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2424
+ #elif NK_TARGET_NEONBFDOT
2425
+ nk_angulars_packed_bf16_neonbfdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2426
+ #elif NK_TARGET_SAPPHIREAMX
2427
+ nk_angulars_packed_bf16_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2428
+ #elif NK_TARGET_GENOA
2429
+ nk_angulars_packed_bf16_genoa(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2430
+ #elif NK_TARGET_SKYLAKE
2431
+ nk_angulars_packed_bf16_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2432
+ #elif NK_TARGET_HASWELL
2433
+ nk_angulars_packed_bf16_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2434
+ #elif NK_TARGET_POWERVSX
2435
+ nk_angulars_packed_bf16_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2436
+ #elif NK_TARGET_RVV
2437
+ nk_angulars_packed_bf16_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2438
+ #elif NK_TARGET_V128RELAXED
2439
+ nk_angulars_packed_bf16_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2440
+ #else
2441
+ nk_angulars_packed_bf16_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2442
+ #endif
2443
+ }
2444
+ NK_PUBLIC void nk_angulars_symmetric_bf16(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2445
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2446
+ nk_size_t row_start, nk_size_t row_count) {
2447
+ #if NK_TARGET_SME
2448
+ nk_angulars_symmetric_bf16_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2449
+ #elif NK_TARGET_NEONBFDOT
2450
+ nk_angulars_symmetric_bf16_neonbfdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2451
+ row_count);
2452
+ #elif NK_TARGET_SAPPHIREAMX
2453
+ nk_angulars_symmetric_bf16_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2454
+ row_count);
2455
+ #elif NK_TARGET_GENOA
2456
+ nk_angulars_symmetric_bf16_genoa(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2457
+ row_count);
2458
+ #elif NK_TARGET_SKYLAKE
2459
+ nk_angulars_symmetric_bf16_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2460
+ row_count);
2461
+ #elif NK_TARGET_HASWELL
2462
+ nk_angulars_symmetric_bf16_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2463
+ row_count);
2464
+ #elif NK_TARGET_POWERVSX
2465
+ nk_angulars_symmetric_bf16_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2466
+ row_count);
2467
+ #elif NK_TARGET_RVV
2468
+ nk_angulars_symmetric_bf16_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2469
+ #elif NK_TARGET_V128RELAXED
2470
+ nk_angulars_symmetric_bf16_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2471
+ row_count);
2472
+ #else
2473
+ nk_angulars_symmetric_bf16_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2474
+ row_count);
2475
+ #endif
2476
+ }
2477
+ NK_PUBLIC void nk_euclideans_packed_bf16(nk_bf16_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2478
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2479
+ nk_size_t r_stride_in_bytes) {
2480
+ #if NK_TARGET_SME
2481
+ nk_euclideans_packed_bf16_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2482
+ #elif NK_TARGET_NEONBFDOT
2483
+ nk_euclideans_packed_bf16_neonbfdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2484
+ #elif NK_TARGET_SAPPHIREAMX
2485
+ nk_euclideans_packed_bf16_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2486
+ #elif NK_TARGET_GENOA
2487
+ nk_euclideans_packed_bf16_genoa(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2488
+ #elif NK_TARGET_SKYLAKE
2489
+ nk_euclideans_packed_bf16_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2490
+ #elif NK_TARGET_HASWELL
2491
+ nk_euclideans_packed_bf16_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2492
+ #elif NK_TARGET_POWERVSX
2493
+ nk_euclideans_packed_bf16_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2494
+ #elif NK_TARGET_RVV
2495
+ nk_euclideans_packed_bf16_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2496
+ #elif NK_TARGET_V128RELAXED
2497
+ nk_euclideans_packed_bf16_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2498
+ #else
2499
+ nk_euclideans_packed_bf16_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2500
+ #endif
2501
+ }
2502
+ NK_PUBLIC void nk_euclideans_symmetric_bf16(nk_bf16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2503
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2504
+ nk_size_t row_start, nk_size_t row_count) {
2505
+ #if NK_TARGET_SME
2506
+ nk_euclideans_symmetric_bf16_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2507
+ row_count);
2508
+ #elif NK_TARGET_NEONBFDOT
2509
+ nk_euclideans_symmetric_bf16_neonbfdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2510
+ row_count);
2511
+ #elif NK_TARGET_SAPPHIREAMX
2512
+ nk_euclideans_symmetric_bf16_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2513
+ row_count);
2514
+ #elif NK_TARGET_GENOA
2515
+ nk_euclideans_symmetric_bf16_genoa(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2516
+ row_count);
2517
+ #elif NK_TARGET_SKYLAKE
2518
+ nk_euclideans_symmetric_bf16_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2519
+ row_count);
2520
+ #elif NK_TARGET_HASWELL
2521
+ nk_euclideans_symmetric_bf16_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2522
+ row_count);
2523
+ #elif NK_TARGET_POWERVSX
2524
+ nk_euclideans_symmetric_bf16_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2525
+ row_count);
2526
+ #elif NK_TARGET_RVV
2527
+ nk_euclideans_symmetric_bf16_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2528
+ row_count);
2529
+ #elif NK_TARGET_V128RELAXED
2530
+ nk_euclideans_symmetric_bf16_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2531
+ row_count);
2532
+ #else
2533
+ nk_euclideans_symmetric_bf16_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2534
+ row_count);
2535
+ #endif
2536
+ }
2537
+
2538
+ NK_PUBLIC void nk_angulars_packed_e4m3(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2539
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2540
+ nk_size_t r_stride_in_bytes) {
2541
+ #if NK_TARGET_SME
2542
+ nk_angulars_packed_e4m3_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2543
+ #elif NK_TARGET_NEONFP8
2544
+ nk_angulars_packed_e4m3_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2545
+ #elif NK_TARGET_NEONFHM
2546
+ nk_angulars_packed_e4m3_neonfhm(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2547
+ #elif NK_TARGET_SAPPHIREAMX
2548
+ nk_angulars_packed_e4m3_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2549
+ #elif NK_TARGET_DIAMOND
2550
+ nk_angulars_packed_e4m3_diamond(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2551
+ #elif NK_TARGET_GENOA
2552
+ nk_angulars_packed_e4m3_genoa(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2553
+ #elif NK_TARGET_SKYLAKE
2554
+ nk_angulars_packed_e4m3_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2555
+ #elif NK_TARGET_HASWELL
2556
+ nk_angulars_packed_e4m3_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2557
+ #elif NK_TARGET_RVV
2558
+ nk_angulars_packed_e4m3_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2559
+ #elif NK_TARGET_V128RELAXED
2560
+ nk_angulars_packed_e4m3_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2561
+ #else
2562
+ nk_angulars_packed_e4m3_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2563
+ #endif
2564
+ }
2565
+ NK_PUBLIC void nk_angulars_symmetric_e4m3(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2566
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2567
+ nk_size_t row_start, nk_size_t row_count) {
2568
+ #if NK_TARGET_SME
2569
+ nk_angulars_symmetric_e4m3_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2570
+ #elif NK_TARGET_NEONFP8
2571
+ nk_angulars_symmetric_e4m3_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2572
+ row_count);
2573
+ #elif NK_TARGET_NEONFHM
2574
+ nk_angulars_symmetric_e4m3_neonfhm(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2575
+ row_count);
2576
+ #elif NK_TARGET_SAPPHIREAMX
2577
+ nk_angulars_symmetric_e4m3_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2578
+ row_count);
2579
+ #elif NK_TARGET_DIAMOND
2580
+ nk_angulars_symmetric_e4m3_diamond(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2581
+ row_count);
2582
+ #elif NK_TARGET_GENOA
2583
+ nk_angulars_symmetric_e4m3_genoa(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2584
+ row_count);
2585
+ #elif NK_TARGET_SKYLAKE
2586
+ nk_angulars_symmetric_e4m3_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2587
+ row_count);
2588
+ #elif NK_TARGET_HASWELL
2589
+ nk_angulars_symmetric_e4m3_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2590
+ row_count);
2591
+ #elif NK_TARGET_RVV
2592
+ nk_angulars_symmetric_e4m3_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2593
+ #elif NK_TARGET_V128RELAXED
2594
+ nk_angulars_symmetric_e4m3_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2595
+ row_count);
2596
+ #else
2597
+ nk_angulars_symmetric_e4m3_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2598
+ row_count);
2599
+ #endif
2600
+ }
2601
+ NK_PUBLIC void nk_euclideans_packed_e4m3(nk_e4m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2602
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2603
+ nk_size_t r_stride_in_bytes) {
2604
+ #if NK_TARGET_SME
2605
+ nk_euclideans_packed_e4m3_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2606
+ #elif NK_TARGET_NEONFP8
2607
+ nk_euclideans_packed_e4m3_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2608
+ #elif NK_TARGET_NEONFHM
2609
+ nk_euclideans_packed_e4m3_neonfhm(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2610
+ #elif NK_TARGET_SAPPHIREAMX
2611
+ nk_euclideans_packed_e4m3_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2612
+ #elif NK_TARGET_DIAMOND
2613
+ nk_euclideans_packed_e4m3_diamond(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2614
+ #elif NK_TARGET_GENOA
2615
+ nk_euclideans_packed_e4m3_genoa(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2616
+ #elif NK_TARGET_SKYLAKE
2617
+ nk_euclideans_packed_e4m3_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2618
+ #elif NK_TARGET_HASWELL
2619
+ nk_euclideans_packed_e4m3_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2620
+ #elif NK_TARGET_RVV
2621
+ nk_euclideans_packed_e4m3_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2622
+ #elif NK_TARGET_V128RELAXED
2623
+ nk_euclideans_packed_e4m3_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2624
+ #else
2625
+ nk_euclideans_packed_e4m3_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2626
+ #endif
2627
+ }
2628
+ NK_PUBLIC void nk_euclideans_symmetric_e4m3(nk_e4m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2629
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2630
+ nk_size_t row_start, nk_size_t row_count) {
2631
+ #if NK_TARGET_SME
2632
+ nk_euclideans_symmetric_e4m3_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2633
+ row_count);
2634
+ #elif NK_TARGET_NEONFP8
2635
+ nk_euclideans_symmetric_e4m3_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2636
+ row_count);
2637
+ #elif NK_TARGET_NEONFHM
2638
+ nk_euclideans_symmetric_e4m3_neonfhm(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2639
+ row_count);
2640
+ #elif NK_TARGET_SAPPHIREAMX
2641
+ nk_euclideans_symmetric_e4m3_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2642
+ row_count);
2643
+ #elif NK_TARGET_DIAMOND
2644
+ nk_euclideans_symmetric_e4m3_diamond(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2645
+ row_count);
2646
+ #elif NK_TARGET_GENOA
2647
+ nk_euclideans_symmetric_e4m3_genoa(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2648
+ row_count);
2649
+ #elif NK_TARGET_SKYLAKE
2650
+ nk_euclideans_symmetric_e4m3_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2651
+ row_count);
2652
+ #elif NK_TARGET_HASWELL
2653
+ nk_euclideans_symmetric_e4m3_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2654
+ row_count);
2655
+ #elif NK_TARGET_RVV
2656
+ nk_euclideans_symmetric_e4m3_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2657
+ row_count);
2658
+ #elif NK_TARGET_V128RELAXED
2659
+ nk_euclideans_symmetric_e4m3_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2660
+ row_count);
2661
+ #else
2662
+ nk_euclideans_symmetric_e4m3_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2663
+ row_count);
2664
+ #endif
2665
+ }
2666
+
2667
+ NK_PUBLIC void nk_angulars_packed_e5m2(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2668
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2669
+ nk_size_t r_stride_in_bytes) {
2670
+ #if NK_TARGET_SME
2671
+ nk_angulars_packed_e5m2_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2672
+ #elif NK_TARGET_NEONFP8
2673
+ nk_angulars_packed_e5m2_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2674
+ #elif NK_TARGET_NEONFHM
2675
+ nk_angulars_packed_e5m2_neonfhm(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2676
+ #elif NK_TARGET_SAPPHIREAMX
2677
+ nk_angulars_packed_e5m2_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2678
+ #elif NK_TARGET_DIAMOND
2679
+ nk_angulars_packed_e5m2_diamond(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2680
+ #elif NK_TARGET_GENOA
2681
+ nk_angulars_packed_e5m2_genoa(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2682
+ #elif NK_TARGET_SKYLAKE
2683
+ nk_angulars_packed_e5m2_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2684
+ #elif NK_TARGET_HASWELL
2685
+ nk_angulars_packed_e5m2_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2686
+ #elif NK_TARGET_RVV
2687
+ nk_angulars_packed_e5m2_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2688
+ #elif NK_TARGET_V128RELAXED
2689
+ nk_angulars_packed_e5m2_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2690
+ #else
2691
+ nk_angulars_packed_e5m2_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2692
+ #endif
2693
+ }
2694
+ NK_PUBLIC void nk_angulars_symmetric_e5m2(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2695
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2696
+ nk_size_t row_start, nk_size_t row_count) {
2697
+ #if NK_TARGET_SME
2698
+ nk_angulars_symmetric_e5m2_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2699
+ #elif NK_TARGET_NEONFP8
2700
+ nk_angulars_symmetric_e5m2_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2701
+ row_count);
2702
+ #elif NK_TARGET_NEONFHM
2703
+ nk_angulars_symmetric_e5m2_neonfhm(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2704
+ row_count);
2705
+ #elif NK_TARGET_SAPPHIREAMX
2706
+ nk_angulars_symmetric_e5m2_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2707
+ row_count);
2708
+ #elif NK_TARGET_DIAMOND
2709
+ nk_angulars_symmetric_e5m2_diamond(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2710
+ row_count);
2711
+ #elif NK_TARGET_GENOA
2712
+ nk_angulars_symmetric_e5m2_genoa(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2713
+ row_count);
2714
+ #elif NK_TARGET_SKYLAKE
2715
+ nk_angulars_symmetric_e5m2_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2716
+ row_count);
2717
+ #elif NK_TARGET_HASWELL
2718
+ nk_angulars_symmetric_e5m2_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2719
+ row_count);
2720
+ #elif NK_TARGET_RVV
2721
+ nk_angulars_symmetric_e5m2_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2722
+ #elif NK_TARGET_V128RELAXED
2723
+ nk_angulars_symmetric_e5m2_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2724
+ row_count);
2725
+ #else
2726
+ nk_angulars_symmetric_e5m2_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2727
+ row_count);
2728
+ #endif
2729
+ }
2730
+ NK_PUBLIC void nk_euclideans_packed_e5m2(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2731
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2732
+ nk_size_t r_stride_in_bytes) {
2733
+ #if NK_TARGET_SME
2734
+ nk_euclideans_packed_e5m2_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2735
+ #elif NK_TARGET_NEONFP8
2736
+ nk_euclideans_packed_e5m2_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2737
+ #elif NK_TARGET_NEONFHM
2738
+ nk_euclideans_packed_e5m2_neonfhm(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2739
+ #elif NK_TARGET_SAPPHIREAMX
2740
+ nk_euclideans_packed_e5m2_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2741
+ #elif NK_TARGET_DIAMOND
2742
+ nk_euclideans_packed_e5m2_diamond(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2743
+ #elif NK_TARGET_GENOA
2744
+ nk_euclideans_packed_e5m2_genoa(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2745
+ #elif NK_TARGET_SKYLAKE
2746
+ nk_euclideans_packed_e5m2_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2747
+ #elif NK_TARGET_HASWELL
2748
+ nk_euclideans_packed_e5m2_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2749
+ #elif NK_TARGET_RVV
2750
+ nk_euclideans_packed_e5m2_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2751
+ #elif NK_TARGET_V128RELAXED
2752
+ nk_euclideans_packed_e5m2_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2753
+ #else
2754
+ nk_euclideans_packed_e5m2_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2755
+ #endif
2756
+ }
2757
+ NK_PUBLIC void nk_euclideans_symmetric_e5m2(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2758
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2759
+ nk_size_t row_start, nk_size_t row_count) {
2760
+ #if NK_TARGET_SME
2761
+ nk_euclideans_symmetric_e5m2_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2762
+ row_count);
2763
+ #elif NK_TARGET_NEONFP8
2764
+ nk_euclideans_symmetric_e5m2_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2765
+ row_count);
2766
+ #elif NK_TARGET_NEONFHM
2767
+ nk_euclideans_symmetric_e5m2_neonfhm(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2768
+ row_count);
2769
+ #elif NK_TARGET_SAPPHIREAMX
2770
+ nk_euclideans_symmetric_e5m2_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2771
+ row_count);
2772
+ #elif NK_TARGET_DIAMOND
2773
+ nk_euclideans_symmetric_e5m2_diamond(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2774
+ row_count);
2775
+ #elif NK_TARGET_GENOA
2776
+ nk_euclideans_symmetric_e5m2_genoa(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2777
+ row_count);
2778
+ #elif NK_TARGET_SKYLAKE
2779
+ nk_euclideans_symmetric_e5m2_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2780
+ row_count);
2781
+ #elif NK_TARGET_HASWELL
2782
+ nk_euclideans_symmetric_e5m2_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2783
+ row_count);
2784
+ #elif NK_TARGET_RVV
2785
+ nk_euclideans_symmetric_e5m2_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2786
+ row_count);
2787
+ #elif NK_TARGET_V128RELAXED
2788
+ nk_euclideans_symmetric_e5m2_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2789
+ row_count);
2790
+ #else
2791
+ nk_euclideans_symmetric_e5m2_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2792
+ row_count);
2793
+ #endif
2794
+ }
2795
+
2796
+ NK_PUBLIC void nk_angulars_packed_e2m3(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2797
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2798
+ nk_size_t r_stride_in_bytes) {
2799
+ #if NK_TARGET_SME
2800
+ nk_angulars_packed_e2m3_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2801
+ #elif NK_TARGET_SAPPHIREAMX
2802
+ nk_angulars_packed_e2m3_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2803
+ #elif NK_TARGET_NEONFP8
2804
+ nk_angulars_packed_e2m3_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2805
+ #elif NK_TARGET_SKYLAKE
2806
+ nk_angulars_packed_e2m3_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2807
+ #elif NK_TARGET_SIERRA
2808
+ nk_angulars_packed_e2m3_sierra(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2809
+ #elif NK_TARGET_ALDER
2810
+ nk_angulars_packed_e2m3_alder(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2811
+ #elif NK_TARGET_HASWELL
2812
+ nk_angulars_packed_e2m3_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2813
+ #elif NK_TARGET_RVV
2814
+ nk_angulars_packed_e2m3_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2815
+ #elif NK_TARGET_V128RELAXED
2816
+ nk_angulars_packed_e2m3_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2817
+ #else
2818
+ nk_angulars_packed_e2m3_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2819
+ #endif
2820
+ }
2821
+ NK_PUBLIC void nk_angulars_symmetric_e2m3(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2822
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2823
+ nk_size_t row_start, nk_size_t row_count) {
2824
+ #if NK_TARGET_SME
2825
+ nk_angulars_symmetric_e2m3_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2826
+ #elif NK_TARGET_SAPPHIREAMX
2827
+ nk_angulars_symmetric_e2m3_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2828
+ row_count);
2829
+ #elif NK_TARGET_NEONFP8
2830
+ nk_angulars_symmetric_e2m3_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2831
+ row_count);
2832
+ #elif NK_TARGET_SKYLAKE
2833
+ nk_angulars_symmetric_e2m3_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2834
+ row_count);
2835
+ #elif NK_TARGET_SIERRA
2836
+ nk_angulars_symmetric_e2m3_sierra(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2837
+ row_count);
2838
+ #elif NK_TARGET_ALDER
2839
+ nk_angulars_symmetric_e2m3_alder(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2840
+ row_count);
2841
+ #elif NK_TARGET_HASWELL
2842
+ nk_angulars_symmetric_e2m3_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2843
+ row_count);
2844
+ #elif NK_TARGET_RVV
2845
+ nk_angulars_symmetric_e2m3_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2846
+ #elif NK_TARGET_V128RELAXED
2847
+ nk_angulars_symmetric_e2m3_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2848
+ row_count);
2849
+ #else
2850
+ nk_angulars_symmetric_e2m3_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2851
+ row_count);
2852
+ #endif
2853
+ }
2854
+ NK_PUBLIC void nk_euclideans_packed_e2m3(nk_e2m3_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2855
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2856
+ nk_size_t r_stride_in_bytes) {
2857
+ #if NK_TARGET_SME
2858
+ nk_euclideans_packed_e2m3_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2859
+ #elif NK_TARGET_SAPPHIREAMX
2860
+ nk_euclideans_packed_e2m3_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2861
+ #elif NK_TARGET_NEONFP8
2862
+ nk_euclideans_packed_e2m3_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2863
+ #elif NK_TARGET_SKYLAKE
2864
+ nk_euclideans_packed_e2m3_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2865
+ #elif NK_TARGET_SIERRA
2866
+ nk_euclideans_packed_e2m3_sierra(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2867
+ #elif NK_TARGET_ALDER
2868
+ nk_euclideans_packed_e2m3_alder(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2869
+ #elif NK_TARGET_HASWELL
2870
+ nk_euclideans_packed_e2m3_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2871
+ #elif NK_TARGET_RVV
2872
+ nk_euclideans_packed_e2m3_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2873
+ #elif NK_TARGET_V128RELAXED
2874
+ nk_euclideans_packed_e2m3_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2875
+ #else
2876
+ nk_euclideans_packed_e2m3_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2877
+ #endif
2878
+ }
2879
+ NK_PUBLIC void nk_euclideans_symmetric_e2m3(nk_e2m3_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2880
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2881
+ nk_size_t row_start, nk_size_t row_count) {
2882
+ #if NK_TARGET_SME
2883
+ nk_euclideans_symmetric_e2m3_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2884
+ row_count);
2885
+ #elif NK_TARGET_SAPPHIREAMX
2886
+ nk_euclideans_symmetric_e2m3_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2887
+ row_count);
2888
+ #elif NK_TARGET_NEONFP8
2889
+ nk_euclideans_symmetric_e2m3_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2890
+ row_count);
2891
+ #elif NK_TARGET_SKYLAKE
2892
+ nk_euclideans_symmetric_e2m3_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2893
+ row_count);
2894
+ #elif NK_TARGET_SIERRA
2895
+ nk_euclideans_symmetric_e2m3_sierra(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2896
+ row_count);
2897
+ #elif NK_TARGET_ALDER
2898
+ nk_euclideans_symmetric_e2m3_alder(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2899
+ row_count);
2900
+ #elif NK_TARGET_HASWELL
2901
+ nk_euclideans_symmetric_e2m3_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2902
+ row_count);
2903
+ #elif NK_TARGET_RVV
2904
+ nk_euclideans_symmetric_e2m3_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2905
+ row_count);
2906
+ #elif NK_TARGET_V128RELAXED
2907
+ nk_euclideans_symmetric_e2m3_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2908
+ row_count);
2909
+ #else
2910
+ nk_euclideans_symmetric_e2m3_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2911
+ row_count);
2912
+ #endif
2913
+ }
2914
+
2915
+ NK_PUBLIC void nk_angulars_packed_e3m2(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2916
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2917
+ nk_size_t r_stride_in_bytes) {
2918
+ #if NK_TARGET_SME
2919
+ nk_angulars_packed_e3m2_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2920
+ #elif NK_TARGET_SAPPHIREAMX
2921
+ nk_angulars_packed_e3m2_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2922
+ #elif NK_TARGET_NEONFP8
2923
+ nk_angulars_packed_e3m2_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2924
+ #elif NK_TARGET_SKYLAKE
2925
+ nk_angulars_packed_e3m2_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2926
+ #elif NK_TARGET_HASWELL
2927
+ nk_angulars_packed_e3m2_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2928
+ #elif NK_TARGET_RVV
2929
+ nk_angulars_packed_e3m2_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2930
+ #else
2931
+ nk_angulars_packed_e3m2_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2932
+ #endif
2933
+ }
2934
+ NK_PUBLIC void nk_angulars_symmetric_e3m2(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2935
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2936
+ nk_size_t row_start, nk_size_t row_count) {
2937
+ #if NK_TARGET_SME
2938
+ nk_angulars_symmetric_e3m2_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2939
+ #elif NK_TARGET_SAPPHIREAMX
2940
+ nk_angulars_symmetric_e3m2_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2941
+ row_count);
2942
+ #elif NK_TARGET_NEONFP8
2943
+ nk_angulars_symmetric_e3m2_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2944
+ row_count);
2945
+ #elif NK_TARGET_SKYLAKE
2946
+ nk_angulars_symmetric_e3m2_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2947
+ row_count);
2948
+ #elif NK_TARGET_HASWELL
2949
+ nk_angulars_symmetric_e3m2_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2950
+ row_count);
2951
+ #elif NK_TARGET_RVV
2952
+ nk_angulars_symmetric_e3m2_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
2953
+ #else
2954
+ nk_angulars_symmetric_e3m2_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2955
+ row_count);
2956
+ #endif
2957
+ }
2958
+ NK_PUBLIC void nk_euclideans_packed_e3m2(nk_e3m2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
2959
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
2960
+ nk_size_t r_stride_in_bytes) {
2961
+ #if NK_TARGET_SME
2962
+ nk_euclideans_packed_e3m2_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2963
+ #elif NK_TARGET_SAPPHIREAMX
2964
+ nk_euclideans_packed_e3m2_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2965
+ #elif NK_TARGET_NEONFP8
2966
+ nk_euclideans_packed_e3m2_neonfp8(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2967
+ #elif NK_TARGET_SKYLAKE
2968
+ nk_euclideans_packed_e3m2_skylake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2969
+ #elif NK_TARGET_HASWELL
2970
+ nk_euclideans_packed_e3m2_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2971
+ #elif NK_TARGET_RVV
2972
+ nk_euclideans_packed_e3m2_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2973
+ #else
2974
+ nk_euclideans_packed_e3m2_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
2975
+ #endif
2976
+ }
2977
+ NK_PUBLIC void nk_euclideans_symmetric_e3m2(nk_e3m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
2978
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
2979
+ nk_size_t row_start, nk_size_t row_count) {
2980
+ #if NK_TARGET_SME
2981
+ nk_euclideans_symmetric_e3m2_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2982
+ row_count);
2983
+ #elif NK_TARGET_SAPPHIREAMX
2984
+ nk_euclideans_symmetric_e3m2_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2985
+ row_count);
2986
+ #elif NK_TARGET_NEONFP8
2987
+ nk_euclideans_symmetric_e3m2_neonfp8(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2988
+ row_count);
2989
+ #elif NK_TARGET_SKYLAKE
2990
+ nk_euclideans_symmetric_e3m2_skylake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2991
+ row_count);
2992
+ #elif NK_TARGET_HASWELL
2993
+ nk_euclideans_symmetric_e3m2_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2994
+ row_count);
2995
+ #elif NK_TARGET_RVV
2996
+ nk_euclideans_symmetric_e3m2_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start,
2997
+ row_count);
2998
+ #else
2999
+ nk_euclideans_symmetric_e3m2_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3000
+ row_count);
3001
+ #endif
3002
+ }
3003
+
3004
+ NK_PUBLIC void nk_angulars_packed_i8(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3005
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3006
+ nk_size_t r_stride_in_bytes) {
3007
+ #if NK_TARGET_SME
3008
+ nk_angulars_packed_i8_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3009
+ #elif NK_TARGET_NEONSDOT
3010
+ nk_angulars_packed_i8_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3011
+ #elif NK_TARGET_SAPPHIREAMX
3012
+ nk_angulars_packed_i8_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3013
+ #elif NK_TARGET_ICELAKE
3014
+ nk_angulars_packed_i8_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3015
+ #elif NK_TARGET_SIERRA
3016
+ nk_angulars_packed_i8_sierra(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3017
+ #elif NK_TARGET_ALDER
3018
+ nk_angulars_packed_i8_alder(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3019
+ #elif NK_TARGET_HASWELL
3020
+ nk_angulars_packed_i8_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3021
+ #elif NK_TARGET_POWERVSX
3022
+ nk_angulars_packed_i8_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3023
+ #elif NK_TARGET_RVV
3024
+ nk_angulars_packed_i8_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3025
+ #elif NK_TARGET_V128RELAXED
3026
+ nk_angulars_packed_i8_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3027
+ #else
3028
+ nk_angulars_packed_i8_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3029
+ #endif
3030
+ }
3031
+ NK_PUBLIC void nk_angulars_symmetric_i8(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3032
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3033
+ nk_size_t row_start, nk_size_t row_count) {
3034
+ #if NK_TARGET_SME
3035
+ nk_angulars_symmetric_i8_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3036
+ #elif NK_TARGET_NEONSDOT
3037
+ nk_angulars_symmetric_i8_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3038
+ row_count);
3039
+ #elif NK_TARGET_SAPPHIREAMX
3040
+ nk_angulars_symmetric_i8_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3041
+ row_count);
3042
+ #elif NK_TARGET_ICELAKE
3043
+ nk_angulars_symmetric_i8_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3044
+ row_count);
3045
+ #elif NK_TARGET_SIERRA
3046
+ nk_angulars_symmetric_i8_sierra(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3047
+ #elif NK_TARGET_ALDER
3048
+ nk_angulars_symmetric_i8_alder(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3049
+ #elif NK_TARGET_HASWELL
3050
+ nk_angulars_symmetric_i8_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3051
+ row_count);
3052
+ #elif NK_TARGET_POWERVSX
3053
+ nk_angulars_symmetric_i8_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3054
+ row_count);
3055
+ #elif NK_TARGET_RVV
3056
+ nk_angulars_symmetric_i8_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3057
+ #elif NK_TARGET_V128RELAXED
3058
+ nk_angulars_symmetric_i8_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3059
+ row_count);
3060
+ #else
3061
+ nk_angulars_symmetric_i8_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3062
+ #endif
3063
+ }
3064
+ NK_PUBLIC void nk_euclideans_packed_i8(nk_i8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3065
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3066
+ nk_size_t r_stride_in_bytes) {
3067
+ #if NK_TARGET_SME
3068
+ nk_euclideans_packed_i8_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3069
+ #elif NK_TARGET_NEONSDOT
3070
+ nk_euclideans_packed_i8_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3071
+ #elif NK_TARGET_SAPPHIREAMX
3072
+ nk_euclideans_packed_i8_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3073
+ #elif NK_TARGET_ICELAKE
3074
+ nk_euclideans_packed_i8_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3075
+ #elif NK_TARGET_SIERRA
3076
+ nk_euclideans_packed_i8_sierra(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3077
+ #elif NK_TARGET_ALDER
3078
+ nk_euclideans_packed_i8_alder(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3079
+ #elif NK_TARGET_HASWELL
3080
+ nk_euclideans_packed_i8_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3081
+ #elif NK_TARGET_POWERVSX
3082
+ nk_euclideans_packed_i8_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3083
+ #elif NK_TARGET_RVV
3084
+ nk_euclideans_packed_i8_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3085
+ #elif NK_TARGET_V128RELAXED
3086
+ nk_euclideans_packed_i8_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3087
+ #else
3088
+ nk_euclideans_packed_i8_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3089
+ #endif
3090
+ }
3091
+ NK_PUBLIC void nk_euclideans_symmetric_i8(nk_i8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3092
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3093
+ nk_size_t row_start, nk_size_t row_count) {
3094
+ #if NK_TARGET_SME
3095
+ nk_euclideans_symmetric_i8_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3096
+ #elif NK_TARGET_NEONSDOT
3097
+ nk_euclideans_symmetric_i8_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3098
+ row_count);
3099
+ #elif NK_TARGET_SAPPHIREAMX
3100
+ nk_euclideans_symmetric_i8_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3101
+ row_count);
3102
+ #elif NK_TARGET_ICELAKE
3103
+ nk_euclideans_symmetric_i8_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3104
+ row_count);
3105
+ #elif NK_TARGET_SIERRA
3106
+ nk_euclideans_symmetric_i8_sierra(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3107
+ row_count);
3108
+ #elif NK_TARGET_ALDER
3109
+ nk_euclideans_symmetric_i8_alder(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3110
+ row_count);
3111
+ #elif NK_TARGET_HASWELL
3112
+ nk_euclideans_symmetric_i8_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3113
+ row_count);
3114
+ #elif NK_TARGET_POWERVSX
3115
+ nk_euclideans_symmetric_i8_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3116
+ row_count);
3117
+ #elif NK_TARGET_RVV
3118
+ nk_euclideans_symmetric_i8_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3119
+ #elif NK_TARGET_V128RELAXED
3120
+ nk_euclideans_symmetric_i8_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3121
+ row_count);
3122
+ #else
3123
+ nk_euclideans_symmetric_i8_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3124
+ row_count);
3125
+ #endif
3126
+ }
3127
+
3128
+ NK_PUBLIC void nk_angulars_packed_u8(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3129
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3130
+ nk_size_t r_stride_in_bytes) {
3131
+ #if NK_TARGET_SME
3132
+ nk_angulars_packed_u8_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3133
+ #elif NK_TARGET_NEONSDOT
3134
+ nk_angulars_packed_u8_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3135
+ #elif NK_TARGET_SAPPHIREAMX
3136
+ nk_angulars_packed_u8_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3137
+ #elif NK_TARGET_ICELAKE
3138
+ nk_angulars_packed_u8_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3139
+ #elif NK_TARGET_SIERRA
3140
+ nk_angulars_packed_u8_sierra(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3141
+ #elif NK_TARGET_ALDER
3142
+ nk_angulars_packed_u8_alder(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3143
+ #elif NK_TARGET_HASWELL
3144
+ nk_angulars_packed_u8_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3145
+ #elif NK_TARGET_POWERVSX
3146
+ nk_angulars_packed_u8_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3147
+ #elif NK_TARGET_RVV
3148
+ nk_angulars_packed_u8_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3149
+ #elif NK_TARGET_V128RELAXED
3150
+ nk_angulars_packed_u8_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3151
+ #else
3152
+ nk_angulars_packed_u8_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3153
+ #endif
3154
+ }
3155
+ NK_PUBLIC void nk_angulars_symmetric_u8(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3156
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3157
+ nk_size_t row_start, nk_size_t row_count) {
3158
+ #if NK_TARGET_SME
3159
+ nk_angulars_symmetric_u8_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3160
+ #elif NK_TARGET_NEONSDOT
3161
+ nk_angulars_symmetric_u8_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3162
+ row_count);
3163
+ #elif NK_TARGET_SAPPHIREAMX
3164
+ nk_angulars_symmetric_u8_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3165
+ row_count);
3166
+ #elif NK_TARGET_ICELAKE
3167
+ nk_angulars_symmetric_u8_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3168
+ row_count);
3169
+ #elif NK_TARGET_SIERRA
3170
+ nk_angulars_symmetric_u8_sierra(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3171
+ #elif NK_TARGET_ALDER
3172
+ nk_angulars_symmetric_u8_alder(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3173
+ #elif NK_TARGET_HASWELL
3174
+ nk_angulars_symmetric_u8_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3175
+ row_count);
3176
+ #elif NK_TARGET_POWERVSX
3177
+ nk_angulars_symmetric_u8_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3178
+ row_count);
3179
+ #elif NK_TARGET_RVV
3180
+ nk_angulars_symmetric_u8_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3181
+ #elif NK_TARGET_V128RELAXED
3182
+ nk_angulars_symmetric_u8_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3183
+ row_count);
3184
+ #else
3185
+ nk_angulars_symmetric_u8_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3186
+ #endif
3187
+ }
3188
+ NK_PUBLIC void nk_euclideans_packed_u8(nk_u8_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3189
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3190
+ nk_size_t r_stride_in_bytes) {
3191
+ #if NK_TARGET_SME
3192
+ nk_euclideans_packed_u8_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3193
+ #elif NK_TARGET_NEONSDOT
3194
+ nk_euclideans_packed_u8_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3195
+ #elif NK_TARGET_SAPPHIREAMX
3196
+ nk_euclideans_packed_u8_sapphireamx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3197
+ #elif NK_TARGET_ICELAKE
3198
+ nk_euclideans_packed_u8_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3199
+ #elif NK_TARGET_SIERRA
3200
+ nk_euclideans_packed_u8_sierra(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3201
+ #elif NK_TARGET_ALDER
3202
+ nk_euclideans_packed_u8_alder(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3203
+ #elif NK_TARGET_HASWELL
3204
+ nk_euclideans_packed_u8_haswell(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3205
+ #elif NK_TARGET_POWERVSX
3206
+ nk_euclideans_packed_u8_powervsx(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3207
+ #elif NK_TARGET_RVV
3208
+ nk_euclideans_packed_u8_rvv(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3209
+ #elif NK_TARGET_V128RELAXED
3210
+ nk_euclideans_packed_u8_v128relaxed(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3211
+ #else
3212
+ nk_euclideans_packed_u8_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3213
+ #endif
3214
+ }
3215
+ NK_PUBLIC void nk_euclideans_symmetric_u8(nk_u8_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3216
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3217
+ nk_size_t row_start, nk_size_t row_count) {
3218
+ #if NK_TARGET_SME
3219
+ nk_euclideans_symmetric_u8_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3220
+ #elif NK_TARGET_NEONSDOT
3221
+ nk_euclideans_symmetric_u8_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3222
+ row_count);
3223
+ #elif NK_TARGET_SAPPHIREAMX
3224
+ nk_euclideans_symmetric_u8_sapphireamx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3225
+ row_count);
3226
+ #elif NK_TARGET_ICELAKE
3227
+ nk_euclideans_symmetric_u8_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3228
+ row_count);
3229
+ #elif NK_TARGET_SIERRA
3230
+ nk_euclideans_symmetric_u8_sierra(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3231
+ row_count);
3232
+ #elif NK_TARGET_ALDER
3233
+ nk_euclideans_symmetric_u8_alder(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3234
+ row_count);
3235
+ #elif NK_TARGET_HASWELL
3236
+ nk_euclideans_symmetric_u8_haswell(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3237
+ row_count);
3238
+ #elif NK_TARGET_POWERVSX
3239
+ nk_euclideans_symmetric_u8_powervsx(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3240
+ row_count);
3241
+ #elif NK_TARGET_RVV
3242
+ nk_euclideans_symmetric_u8_rvv(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3243
+ #elif NK_TARGET_V128RELAXED
3244
+ nk_euclideans_symmetric_u8_v128relaxed(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3245
+ row_count);
3246
+ #else
3247
+ nk_euclideans_symmetric_u8_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3248
+ row_count);
3249
+ #endif
3250
+ }
3251
+
3252
+ NK_PUBLIC void nk_angulars_packed_i4(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3253
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3254
+ nk_size_t r_stride_in_bytes) {
3255
+ #if NK_TARGET_SME
3256
+ nk_angulars_packed_i4_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3257
+ #elif NK_TARGET_NEONSDOT
3258
+ nk_angulars_packed_i4_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3259
+ #elif NK_TARGET_ICELAKE
3260
+ nk_angulars_packed_i4_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3261
+ #else
3262
+ nk_angulars_packed_i4_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3263
+ #endif
3264
+ }
3265
+ NK_PUBLIC void nk_angulars_symmetric_i4(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3266
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3267
+ nk_size_t row_start, nk_size_t row_count) {
3268
+ #if NK_TARGET_SME
3269
+ nk_angulars_symmetric_i4_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3270
+ #elif NK_TARGET_NEONSDOT
3271
+ nk_angulars_symmetric_i4_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3272
+ row_count);
3273
+ #elif NK_TARGET_ICELAKE
3274
+ nk_angulars_symmetric_i4_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3275
+ row_count);
3276
+ #else
3277
+ nk_angulars_symmetric_i4_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3278
+ #endif
3279
+ }
3280
+ NK_PUBLIC void nk_euclideans_packed_i4(nk_i4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3281
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3282
+ nk_size_t r_stride_in_bytes) {
3283
+ #if NK_TARGET_SME
3284
+ nk_euclideans_packed_i4_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3285
+ #elif NK_TARGET_NEONSDOT
3286
+ nk_euclideans_packed_i4_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3287
+ #elif NK_TARGET_ICELAKE
3288
+ nk_euclideans_packed_i4_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3289
+ #else
3290
+ nk_euclideans_packed_i4_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3291
+ #endif
3292
+ }
3293
+ NK_PUBLIC void nk_euclideans_symmetric_i4(nk_i4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3294
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3295
+ nk_size_t row_start, nk_size_t row_count) {
3296
+ #if NK_TARGET_SME
3297
+ nk_euclideans_symmetric_i4_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3298
+ #elif NK_TARGET_NEONSDOT
3299
+ nk_euclideans_symmetric_i4_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3300
+ row_count);
3301
+ #elif NK_TARGET_ICELAKE
3302
+ nk_euclideans_symmetric_i4_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3303
+ row_count);
3304
+ #else
3305
+ nk_euclideans_symmetric_i4_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3306
+ row_count);
3307
+ #endif
3308
+ }
3309
+
3310
+ NK_PUBLIC void nk_angulars_packed_u4(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3311
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3312
+ nk_size_t r_stride_in_bytes) {
3313
+ #if NK_TARGET_SME
3314
+ nk_angulars_packed_u4_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3315
+ #elif NK_TARGET_NEONSDOT
3316
+ nk_angulars_packed_u4_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3317
+ #elif NK_TARGET_ICELAKE
3318
+ nk_angulars_packed_u4_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3319
+ #else
3320
+ nk_angulars_packed_u4_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3321
+ #endif
3322
+ }
3323
+ NK_PUBLIC void nk_angulars_symmetric_u4(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3324
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3325
+ nk_size_t row_start, nk_size_t row_count) {
3326
+ #if NK_TARGET_SME
3327
+ nk_angulars_symmetric_u4_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3328
+ #elif NK_TARGET_NEONSDOT
3329
+ nk_angulars_symmetric_u4_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3330
+ row_count);
3331
+ #elif NK_TARGET_ICELAKE
3332
+ nk_angulars_symmetric_u4_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3333
+ row_count);
3334
+ #else
3335
+ nk_angulars_symmetric_u4_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3336
+ #endif
3337
+ }
3338
+ NK_PUBLIC void nk_euclideans_packed_u4(nk_u4x2_t const *a, void const *b_packed, nk_f32_t *result, nk_size_t rows,
3339
+ nk_size_t cols, nk_size_t depth, nk_size_t a_stride_in_bytes,
3340
+ nk_size_t r_stride_in_bytes) {
3341
+ #if NK_TARGET_SME
3342
+ nk_euclideans_packed_u4_sme(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3343
+ #elif NK_TARGET_NEONSDOT
3344
+ nk_euclideans_packed_u4_neonsdot(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3345
+ #elif NK_TARGET_ICELAKE
3346
+ nk_euclideans_packed_u4_icelake(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3347
+ #else
3348
+ nk_euclideans_packed_u4_serial(a, b_packed, result, rows, cols, depth, a_stride_in_bytes, r_stride_in_bytes);
3349
+ #endif
3350
+ }
3351
+ NK_PUBLIC void nk_euclideans_symmetric_u4(nk_u4x2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
3352
+ nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
3353
+ nk_size_t row_start, nk_size_t row_count) {
3354
+ #if NK_TARGET_SME
3355
+ nk_euclideans_symmetric_u4_sme(vectors, vectors_count, depth, stride, result, result_stride, row_start, row_count);
3356
+ #elif NK_TARGET_NEONSDOT
3357
+ nk_euclideans_symmetric_u4_neonsdot(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3358
+ row_count);
3359
+ #elif NK_TARGET_ICELAKE
3360
+ nk_euclideans_symmetric_u4_icelake(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3361
+ row_count);
3362
+ #else
3363
+ nk_euclideans_symmetric_u4_serial(vectors, vectors_count, depth, stride, result, result_stride, row_start,
3364
+ row_count);
3365
+ #endif
3366
+ }
3367
+
3368
+ #endif // !NK_DYNAMIC_DISPATCH
3369
+
3370
+ #if defined(__cplusplus)
3371
+ } // extern "C"
3372
+ #endif
3373
+
3374
+ #endif // NK_SPATIALS_H