usearch 2.21.3 → 2.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (602) hide show
  1. package/README.md +24 -24
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +7 -7
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +83 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +235 -52
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/dist/cjs/usearch.d.ts +8 -3
  16. package/javascript/dist/cjs/usearch.js +12 -4
  17. package/javascript/dist/esm/usearch.d.ts +8 -3
  18. package/javascript/dist/esm/usearch.js +12 -4
  19. package/javascript/lib.cpp +17 -0
  20. package/javascript/usearch.ts +14 -6
  21. package/numkong/.clang-tidy +30 -0
  22. package/{simsimd → numkong}/.cmake-format.py +18 -0
  23. package/numkong/.editorconfig +12 -0
  24. package/numkong/.git-blame-ignore-revs +123 -0
  25. package/numkong/.mailmap +15 -0
  26. package/numkong/.spi.yml +11 -0
  27. package/numkong/.swift-format +14 -0
  28. package/numkong/CITATION.cff +81 -0
  29. package/numkong/CONTRIBUTING.md +500 -0
  30. package/numkong/Cargo.toml +54 -0
  31. package/numkong/MANIFEST.in +8 -0
  32. package/numkong/Package.swift +84 -0
  33. package/numkong/README.md +569 -0
  34. package/numkong/bench/README.md +217 -0
  35. package/numkong/bench/bench-browser.html +323 -0
  36. package/numkong/bench/bench.cpp +344 -0
  37. package/numkong/bench/bench.hpp +816 -0
  38. package/numkong/bench/bench.js +148 -0
  39. package/numkong/bench/bench.mjs +616 -0
  40. package/numkong/bench/bench_attention.cpp +113 -0
  41. package/numkong/bench/bench_cast.cpp +264 -0
  42. package/numkong/bench/bench_cross_amx.cpp +62 -0
  43. package/numkong/bench/bench_cross_arm.cpp +245 -0
  44. package/numkong/bench/bench_cross_blas.cpp +309 -0
  45. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  46. package/numkong/bench/bench_cross_power.cpp +84 -0
  47. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  48. package/numkong/bench/bench_cross_serial.cpp +151 -0
  49. package/numkong/bench/bench_cross_sme.cpp +163 -0
  50. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  51. package/numkong/bench/bench_cross_x86.cpp +455 -0
  52. package/numkong/bench/bench_curved.cpp +210 -0
  53. package/numkong/bench/bench_dot.cpp +315 -0
  54. package/numkong/bench/bench_each.cpp +503 -0
  55. package/numkong/bench/bench_geospatial.cpp +112 -0
  56. package/numkong/bench/bench_maxsim.cpp +154 -0
  57. package/numkong/bench/bench_mesh.cpp +148 -0
  58. package/numkong/bench/bench_probability.cpp +60 -0
  59. package/numkong/bench/bench_reduce.cpp +371 -0
  60. package/numkong/bench/bench_set.cpp +73 -0
  61. package/numkong/bench/bench_sparse.cpp +198 -0
  62. package/numkong/bench/bench_spatial.cpp +347 -0
  63. package/numkong/bench/bench_trigonometry.cpp +137 -0
  64. package/numkong/bench/swift/Bench.swift +506 -0
  65. package/numkong/binding.gyp +169 -0
  66. package/numkong/build.rs +484 -0
  67. package/numkong/c/dispatch.h +512 -0
  68. package/numkong/c/dispatch_bf16.c +447 -0
  69. package/numkong/c/dispatch_bf16c.c +52 -0
  70. package/numkong/c/dispatch_e2m3.c +296 -0
  71. package/numkong/c/dispatch_e3m2.c +284 -0
  72. package/numkong/c/dispatch_e4m3.c +318 -0
  73. package/numkong/c/dispatch_e5m2.c +316 -0
  74. package/numkong/c/dispatch_f16.c +452 -0
  75. package/numkong/c/dispatch_f16c.c +58 -0
  76. package/numkong/c/dispatch_f32.c +434 -0
  77. package/numkong/c/dispatch_f32c.c +99 -0
  78. package/numkong/c/dispatch_f64.c +348 -0
  79. package/numkong/c/dispatch_f64c.c +98 -0
  80. package/numkong/c/dispatch_i16.c +96 -0
  81. package/numkong/c/dispatch_i32.c +89 -0
  82. package/numkong/c/dispatch_i4.c +153 -0
  83. package/numkong/c/dispatch_i64.c +86 -0
  84. package/numkong/c/dispatch_i8.c +348 -0
  85. package/numkong/c/dispatch_other.c +348 -0
  86. package/numkong/c/dispatch_u1.c +193 -0
  87. package/numkong/c/dispatch_u16.c +124 -0
  88. package/numkong/c/dispatch_u32.c +118 -0
  89. package/numkong/c/dispatch_u4.c +153 -0
  90. package/numkong/c/dispatch_u64.c +102 -0
  91. package/numkong/c/dispatch_u8.c +364 -0
  92. package/numkong/c/numkong.c +940 -0
  93. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  94. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  95. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  96. package/numkong/cmake/nk_math_lib.cmake +46 -0
  97. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  99. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  100. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  101. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  102. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  103. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  104. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  105. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  106. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  107. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  108. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  109. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  110. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  111. package/numkong/include/README.md +643 -0
  112. package/numkong/include/module.modulemap +129 -0
  113. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  114. package/numkong/include/numkong/attention/sme.h +2075 -0
  115. package/numkong/include/numkong/attention.h +49 -0
  116. package/numkong/include/numkong/capabilities.h +930 -0
  117. package/numkong/include/numkong/cast/README.md +262 -0
  118. package/numkong/include/numkong/cast/diamond.h +64 -0
  119. package/numkong/include/numkong/cast/haswell.h +823 -0
  120. package/numkong/include/numkong/cast/icelake.h +475 -0
  121. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  122. package/numkong/include/numkong/cast/neon.h +1174 -0
  123. package/numkong/include/numkong/cast/powervsx.h +449 -0
  124. package/numkong/include/numkong/cast/rvv.h +970 -0
  125. package/numkong/include/numkong/cast/sapphire.h +262 -0
  126. package/numkong/include/numkong/cast/serial.h +2336 -0
  127. package/numkong/include/numkong/cast/skylake.h +915 -0
  128. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  129. package/numkong/include/numkong/cast.h +254 -0
  130. package/numkong/include/numkong/cast.hpp +44 -0
  131. package/numkong/include/numkong/curved/README.md +223 -0
  132. package/numkong/include/numkong/curved/genoa.h +182 -0
  133. package/numkong/include/numkong/curved/haswell.h +276 -0
  134. package/numkong/include/numkong/curved/neon.h +329 -0
  135. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  136. package/numkong/include/numkong/curved/rvv.h +305 -0
  137. package/numkong/include/numkong/curved/serial.h +207 -0
  138. package/numkong/include/numkong/curved/skylake.h +457 -0
  139. package/numkong/include/numkong/curved/smef64.h +530 -0
  140. package/numkong/include/numkong/curved.h +513 -0
  141. package/numkong/include/numkong/curved.hpp +144 -0
  142. package/numkong/include/numkong/dot/README.md +442 -0
  143. package/numkong/include/numkong/dot/alder.h +563 -0
  144. package/numkong/include/numkong/dot/diamond.h +158 -0
  145. package/numkong/include/numkong/dot/genoa.h +289 -0
  146. package/numkong/include/numkong/dot/haswell.h +1723 -0
  147. package/numkong/include/numkong/dot/icelake.h +997 -0
  148. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  149. package/numkong/include/numkong/dot/neon.h +869 -0
  150. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  151. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  152. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  153. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  154. package/numkong/include/numkong/dot/powervsx.h +752 -0
  155. package/numkong/include/numkong/dot/rvv.h +722 -0
  156. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  157. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  158. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  159. package/numkong/include/numkong/dot/sapphire.h +141 -0
  160. package/numkong/include/numkong/dot/serial.h +874 -0
  161. package/numkong/include/numkong/dot/sierra.h +405 -0
  162. package/numkong/include/numkong/dot/skylake.h +1116 -0
  163. package/numkong/include/numkong/dot/sve.h +420 -0
  164. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  165. package/numkong/include/numkong/dot/svehalf.h +168 -0
  166. package/numkong/include/numkong/dot/svesdot.h +90 -0
  167. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  168. package/numkong/include/numkong/dot.h +1136 -0
  169. package/numkong/include/numkong/dot.hpp +94 -0
  170. package/numkong/include/numkong/dots/README.md +496 -0
  171. package/numkong/include/numkong/dots/alder.h +117 -0
  172. package/numkong/include/numkong/dots/diamond.h +86 -0
  173. package/numkong/include/numkong/dots/genoa.h +100 -0
  174. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  175. package/numkong/include/numkong/dots/haswell.h +310 -0
  176. package/numkong/include/numkong/dots/icelake.h +180 -0
  177. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  178. package/numkong/include/numkong/dots/neon.h +123 -0
  179. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  180. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  181. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  182. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  183. package/numkong/include/numkong/dots/powervsx.h +194 -0
  184. package/numkong/include/numkong/dots/rvv.h +2593 -0
  185. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  186. package/numkong/include/numkong/dots/serial.h +2867 -0
  187. package/numkong/include/numkong/dots/sierra.h +100 -0
  188. package/numkong/include/numkong/dots/skylake.h +207 -0
  189. package/numkong/include/numkong/dots/sme.h +5041 -0
  190. package/numkong/include/numkong/dots/smebi32.h +477 -0
  191. package/numkong/include/numkong/dots/smef64.h +1328 -0
  192. package/numkong/include/numkong/dots/smehalf.h +47 -0
  193. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  194. package/numkong/include/numkong/dots.h +3110 -0
  195. package/numkong/include/numkong/dots.hpp +641 -0
  196. package/numkong/include/numkong/each/README.md +465 -0
  197. package/numkong/include/numkong/each/haswell.h +1658 -0
  198. package/numkong/include/numkong/each/icelake.h +272 -0
  199. package/numkong/include/numkong/each/neon.h +1138 -0
  200. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  201. package/numkong/include/numkong/each/neonhalf.h +318 -0
  202. package/numkong/include/numkong/each/rvv.h +1121 -0
  203. package/numkong/include/numkong/each/sapphire.h +343 -0
  204. package/numkong/include/numkong/each/serial.h +282 -0
  205. package/numkong/include/numkong/each/skylake.h +1562 -0
  206. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  207. package/numkong/include/numkong/each.h +2232 -0
  208. package/numkong/include/numkong/each.hpp +434 -0
  209. package/numkong/include/numkong/geospatial/README.md +147 -0
  210. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  211. package/numkong/include/numkong/geospatial/neon.h +615 -0
  212. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  213. package/numkong/include/numkong/geospatial/serial.h +309 -0
  214. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  215. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  216. package/numkong/include/numkong/geospatial.h +453 -0
  217. package/numkong/include/numkong/geospatial.hpp +235 -0
  218. package/numkong/include/numkong/matrix.hpp +336 -0
  219. package/numkong/include/numkong/maxsim/README.md +187 -0
  220. package/numkong/include/numkong/maxsim/alder.h +511 -0
  221. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  222. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  223. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  224. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  225. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  226. package/numkong/include/numkong/maxsim/serial.h +490 -0
  227. package/numkong/include/numkong/maxsim/sme.h +993 -0
  228. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  229. package/numkong/include/numkong/maxsim.h +571 -0
  230. package/numkong/include/numkong/maxsim.hpp +133 -0
  231. package/numkong/include/numkong/mesh/README.md +219 -0
  232. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  233. package/numkong/include/numkong/mesh/neon.h +1821 -0
  234. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  235. package/numkong/include/numkong/mesh/rvv.h +936 -0
  236. package/numkong/include/numkong/mesh/serial.h +721 -0
  237. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  238. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  239. package/numkong/include/numkong/mesh.h +666 -0
  240. package/numkong/include/numkong/mesh.hpp +701 -0
  241. package/numkong/include/numkong/numkong.h +78 -0
  242. package/numkong/include/numkong/numkong.hpp +58 -0
  243. package/numkong/include/numkong/probability/README.md +177 -0
  244. package/numkong/include/numkong/probability/haswell.h +267 -0
  245. package/numkong/include/numkong/probability/neon.h +255 -0
  246. package/numkong/include/numkong/probability/rvv.h +414 -0
  247. package/numkong/include/numkong/probability/serial.h +181 -0
  248. package/numkong/include/numkong/probability/skylake.h +324 -0
  249. package/numkong/include/numkong/probability.h +380 -0
  250. package/numkong/include/numkong/probability.hpp +120 -0
  251. package/numkong/include/numkong/random.h +50 -0
  252. package/numkong/include/numkong/random.hpp +285 -0
  253. package/numkong/include/numkong/reduce/README.md +552 -0
  254. package/numkong/include/numkong/reduce/alder.h +636 -0
  255. package/numkong/include/numkong/reduce/genoa.h +201 -0
  256. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  257. package/numkong/include/numkong/reduce/icelake.h +549 -0
  258. package/numkong/include/numkong/reduce/neon.h +3980 -0
  259. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  260. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  261. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  262. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  263. package/numkong/include/numkong/reduce/serial.h +753 -0
  264. package/numkong/include/numkong/reduce/sierra.h +338 -0
  265. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  266. package/numkong/include/numkong/reduce/sve.h +52 -0
  267. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  268. package/numkong/include/numkong/reduce.h +1582 -0
  269. package/numkong/include/numkong/reduce.hpp +778 -0
  270. package/numkong/include/numkong/scalar/README.md +89 -0
  271. package/numkong/include/numkong/scalar/haswell.h +122 -0
  272. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  273. package/numkong/include/numkong/scalar/neon.h +122 -0
  274. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  275. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  276. package/numkong/include/numkong/scalar/rvv.h +211 -0
  277. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  278. package/numkong/include/numkong/scalar/serial.h +332 -0
  279. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  280. package/numkong/include/numkong/scalar.h +696 -0
  281. package/numkong/include/numkong/set/README.md +179 -0
  282. package/numkong/include/numkong/set/haswell.h +334 -0
  283. package/numkong/include/numkong/set/icelake.h +485 -0
  284. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  285. package/numkong/include/numkong/set/neon.h +363 -0
  286. package/numkong/include/numkong/set/powervsx.h +326 -0
  287. package/numkong/include/numkong/set/rvv.h +226 -0
  288. package/numkong/include/numkong/set/rvvbb.h +117 -0
  289. package/numkong/include/numkong/set/serial.h +174 -0
  290. package/numkong/include/numkong/set/sve.h +187 -0
  291. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  292. package/numkong/include/numkong/set.h +471 -0
  293. package/numkong/include/numkong/set.hpp +114 -0
  294. package/numkong/include/numkong/sets/README.md +149 -0
  295. package/numkong/include/numkong/sets/haswell.h +63 -0
  296. package/numkong/include/numkong/sets/icelake.h +66 -0
  297. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  298. package/numkong/include/numkong/sets/neon.h +61 -0
  299. package/numkong/include/numkong/sets/powervsx.h +65 -0
  300. package/numkong/include/numkong/sets/serial.h +43 -0
  301. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  302. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  303. package/numkong/include/numkong/sets.h +382 -0
  304. package/numkong/include/numkong/sparse/README.md +156 -0
  305. package/numkong/include/numkong/sparse/icelake.h +463 -0
  306. package/numkong/include/numkong/sparse/neon.h +288 -0
  307. package/numkong/include/numkong/sparse/serial.h +117 -0
  308. package/numkong/include/numkong/sparse/sve2.h +504 -0
  309. package/numkong/include/numkong/sparse/turin.h +322 -0
  310. package/numkong/include/numkong/sparse.h +363 -0
  311. package/numkong/include/numkong/sparse.hpp +113 -0
  312. package/numkong/include/numkong/spatial/README.md +439 -0
  313. package/numkong/include/numkong/spatial/alder.h +604 -0
  314. package/numkong/include/numkong/spatial/diamond.h +240 -0
  315. package/numkong/include/numkong/spatial/genoa.h +222 -0
  316. package/numkong/include/numkong/spatial/haswell.h +979 -0
  317. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  318. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  319. package/numkong/include/numkong/spatial/neon.h +846 -0
  320. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  321. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  322. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  323. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  324. package/numkong/include/numkong/spatial/rvv.h +991 -0
  325. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  326. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  327. package/numkong/include/numkong/spatial/serial.h +347 -0
  328. package/numkong/include/numkong/spatial/sierra.h +516 -0
  329. package/numkong/include/numkong/spatial/skylake.h +605 -0
  330. package/numkong/include/numkong/spatial/sve.h +246 -0
  331. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  332. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  333. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  334. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  335. package/numkong/include/numkong/spatial.h +1681 -0
  336. package/numkong/include/numkong/spatial.hpp +183 -0
  337. package/numkong/include/numkong/spatials/README.md +580 -0
  338. package/numkong/include/numkong/spatials/alder.h +94 -0
  339. package/numkong/include/numkong/spatials/diamond.h +82 -0
  340. package/numkong/include/numkong/spatials/genoa.h +94 -0
  341. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  342. package/numkong/include/numkong/spatials/haswell.h +219 -0
  343. package/numkong/include/numkong/spatials/icelake.h +113 -0
  344. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  345. package/numkong/include/numkong/spatials/neon.h +109 -0
  346. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  347. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  348. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  349. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  350. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  351. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  352. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  353. package/numkong/include/numkong/spatials/serial.h +248 -0
  354. package/numkong/include/numkong/spatials/sierra.h +96 -0
  355. package/numkong/include/numkong/spatials/skylake.h +184 -0
  356. package/numkong/include/numkong/spatials/sme.h +1927 -0
  357. package/numkong/include/numkong/spatials/smef64.h +483 -0
  358. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  359. package/numkong/include/numkong/spatials.h +3374 -0
  360. package/numkong/include/numkong/spatials.hpp +509 -0
  361. package/numkong/include/numkong/tensor.hpp +1981 -0
  362. package/numkong/include/numkong/trigonometry/README.md +184 -0
  363. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  364. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  365. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  366. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  367. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  368. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  369. package/numkong/include/numkong/trigonometry.h +467 -0
  370. package/numkong/include/numkong/trigonometry.hpp +166 -0
  371. package/numkong/include/numkong/types.h +1695 -0
  372. package/numkong/include/numkong/types.hpp +6027 -0
  373. package/numkong/include/numkong/vector.hpp +766 -0
  374. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  375. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  376. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  377. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  378. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  380. package/numkong/javascript/README.md +254 -0
  381. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  382. package/numkong/javascript/numkong-browser.ts +40 -0
  383. package/numkong/javascript/numkong-wasm.ts +750 -0
  384. package/numkong/javascript/numkong.c +712 -0
  385. package/numkong/javascript/numkong.ts +613 -0
  386. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  387. package/numkong/javascript/tsconfig-cjs.json +13 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  389. package/numkong/javascript/types.ts +784 -0
  390. package/numkong/numkong.gypi +104 -0
  391. package/numkong/package.json +108 -0
  392. package/numkong/probes/arm_neon.c +8 -0
  393. package/numkong/probes/arm_neon_bfdot.c +9 -0
  394. package/numkong/probes/arm_neon_fhm.c +9 -0
  395. package/numkong/probes/arm_neon_half.c +8 -0
  396. package/numkong/probes/arm_neon_sdot.c +9 -0
  397. package/numkong/probes/arm_neonfp8.c +9 -0
  398. package/numkong/probes/arm_sme.c +16 -0
  399. package/numkong/probes/arm_sme2.c +16 -0
  400. package/numkong/probes/arm_sme2p1.c +16 -0
  401. package/numkong/probes/arm_sme_bf16.c +16 -0
  402. package/numkong/probes/arm_sme_bi32.c +16 -0
  403. package/numkong/probes/arm_sme_f64.c +16 -0
  404. package/numkong/probes/arm_sme_fa64.c +14 -0
  405. package/numkong/probes/arm_sme_half.c +16 -0
  406. package/numkong/probes/arm_sme_lut2.c +15 -0
  407. package/numkong/probes/arm_sve.c +18 -0
  408. package/numkong/probes/arm_sve2.c +20 -0
  409. package/numkong/probes/arm_sve2p1.c +18 -0
  410. package/numkong/probes/arm_sve_bfdot.c +20 -0
  411. package/numkong/probes/arm_sve_half.c +18 -0
  412. package/numkong/probes/arm_sve_sdot.c +21 -0
  413. package/numkong/probes/loongarch_lasx.c +12 -0
  414. package/numkong/probes/power_vsx.c +12 -0
  415. package/numkong/probes/probe.js +127 -0
  416. package/numkong/probes/riscv_rvv.c +14 -0
  417. package/numkong/probes/riscv_rvv_bb.c +15 -0
  418. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  419. package/numkong/probes/riscv_rvv_half.c +14 -0
  420. package/numkong/probes/wasm_v128relaxed.c +11 -0
  421. package/numkong/probes/x86_alder.c +17 -0
  422. package/numkong/probes/x86_diamond.c +17 -0
  423. package/numkong/probes/x86_genoa.c +17 -0
  424. package/numkong/probes/x86_graniteamx.c +19 -0
  425. package/numkong/probes/x86_haswell.c +11 -0
  426. package/numkong/probes/x86_icelake.c +17 -0
  427. package/numkong/probes/x86_sapphire.c +16 -0
  428. package/numkong/probes/x86_sapphireamx.c +18 -0
  429. package/numkong/probes/x86_sierra.c +17 -0
  430. package/numkong/probes/x86_skylake.c +15 -0
  431. package/numkong/probes/x86_turin.c +17 -0
  432. package/numkong/rust/README.md +698 -0
  433. package/numkong/rust/capabilities.rs +96 -0
  434. package/numkong/rust/cast.rs +314 -0
  435. package/numkong/rust/curved.rs +391 -0
  436. package/numkong/rust/each.rs +2721 -0
  437. package/numkong/rust/geospatial.rs +343 -0
  438. package/numkong/rust/matrix.rs +5424 -0
  439. package/numkong/rust/maxsim.rs +540 -0
  440. package/numkong/rust/mesh.rs +837 -0
  441. package/numkong/rust/numkong.rs +409 -0
  442. package/numkong/rust/probability.rs +272 -0
  443. package/numkong/rust/reduce.rs +1249 -0
  444. package/numkong/rust/set.rs +194 -0
  445. package/numkong/rust/sparse.rs +513 -0
  446. package/numkong/rust/spatial.rs +1815 -0
  447. package/numkong/rust/tensor.rs +7565 -0
  448. package/numkong/rust/types.rs +3666 -0
  449. package/numkong/rust/vector.rs +1899 -0
  450. package/numkong/rustfmt.toml +2 -0
  451. package/numkong/swift/Matrix.swift +1388 -0
  452. package/numkong/swift/MaxSim.swift +151 -0
  453. package/numkong/swift/NumKong.swift +283 -0
  454. package/numkong/swift/Numerics.swift +329 -0
  455. package/numkong/swift/README.md +433 -0
  456. package/numkong/swift/Scalar.swift +741 -0
  457. package/numkong/swift/Tensor.swift +209 -0
  458. package/numkong/test/README.md +315 -0
  459. package/numkong/test/playwright.config.ts +16 -0
  460. package/numkong/test/swift/Test.swift +534 -0
  461. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  462. package/numkong/test/test-browser-bundle.html +188 -0
  463. package/numkong/test/test-browser-runner.mjs +43 -0
  464. package/numkong/test/test-browser.html +305 -0
  465. package/numkong/test/test-dtypes.mjs +453 -0
  466. package/numkong/test/test-wasm.mjs +267 -0
  467. package/numkong/test/test.cpp +442 -0
  468. package/numkong/test/test.hpp +580 -0
  469. package/numkong/test/test.mjs +363 -0
  470. package/numkong/test/test_base.py +1041 -0
  471. package/numkong/test/test_cast.cpp +242 -0
  472. package/numkong/test/test_cross.hpp +551 -0
  473. package/numkong/test/test_cross.py +467 -0
  474. package/numkong/test/test_cross_amx.cpp +50 -0
  475. package/numkong/test/test_cross_arm.cpp +225 -0
  476. package/numkong/test/test_cross_blas.cpp +354 -0
  477. package/numkong/test/test_cross_loongarch.cpp +90 -0
  478. package/numkong/test/test_cross_power.cpp +73 -0
  479. package/numkong/test/test_cross_rvv.cpp +109 -0
  480. package/numkong/test/test_cross_serial.cpp +181 -0
  481. package/numkong/test/test_cross_sme.cpp +148 -0
  482. package/numkong/test/test_cross_wasm.cpp +141 -0
  483. package/numkong/test/test_cross_x86.cpp +442 -0
  484. package/numkong/test/test_curved.cpp +182 -0
  485. package/numkong/test/test_curved.py +207 -0
  486. package/numkong/test/test_dot.cpp +353 -0
  487. package/numkong/test/test_dot.py +296 -0
  488. package/numkong/test/test_each.cpp +437 -0
  489. package/numkong/test/test_each.py +756 -0
  490. package/numkong/test/test_geospatial.cpp +155 -0
  491. package/numkong/test/test_geospatial.py +241 -0
  492. package/numkong/test/test_maxsim.cpp +138 -0
  493. package/numkong/test/test_maxsim.py +209 -0
  494. package/numkong/test/test_mesh.cpp +203 -0
  495. package/numkong/test/test_mesh.py +394 -0
  496. package/numkong/test/test_probability.cpp +136 -0
  497. package/numkong/test/test_probability.py +166 -0
  498. package/numkong/test/test_reduce.cpp +563 -0
  499. package/numkong/test/test_reduce.py +781 -0
  500. package/numkong/test/test_set.cpp +151 -0
  501. package/numkong/test/test_set.py +120 -0
  502. package/numkong/test/test_similarities.py +579 -0
  503. package/numkong/test/test_sparse.cpp +121 -0
  504. package/numkong/test/test_sparse.py +118 -0
  505. package/numkong/test/test_spatial.cpp +518 -0
  506. package/numkong/test/test_spatial.py +269 -0
  507. package/numkong/test/test_tensor.cpp +884 -0
  508. package/numkong/test/test_tensor.py +1417 -0
  509. package/numkong/test/test_trigonometry.cpp +156 -0
  510. package/numkong/test/test_trigonometry.py +180 -0
  511. package/package.json +7 -8
  512. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  513. package/prebuilds/linux-arm64/usearch.node +0 -0
  514. package/prebuilds/linux-x64/usearch.node +0 -0
  515. package/fp16/CMakeLists.txt +0 -176
  516. package/fp16/LICENSE +0 -11
  517. package/fp16/README.md +0 -20
  518. package/fp16/bench/alt-element.cc +0 -89
  519. package/fp16/bench/from-alt-array.cc +0 -175
  520. package/fp16/bench/from-ieee-array.cc +0 -399
  521. package/fp16/bench/ieee-element.cc +0 -272
  522. package/fp16/bench/to-alt-array.cc +0 -91
  523. package/fp16/bench/to-ieee-array.cc +0 -303
  524. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  525. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  526. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  527. package/fp16/configure.py +0 -65
  528. package/fp16/confu.yaml +0 -8
  529. package/fp16/include/fp16/__init__.py +0 -0
  530. package/fp16/include/fp16/avx.py +0 -50
  531. package/fp16/include/fp16/avx2.py +0 -53
  532. package/fp16/include/fp16/bitcasts.h +0 -92
  533. package/fp16/include/fp16/fp16.h +0 -451
  534. package/fp16/include/fp16/psimd.h +0 -131
  535. package/fp16/include/fp16.h +0 -11
  536. package/fp16/jni/Android.mk +0 -6
  537. package/fp16/jni/Application.mk +0 -4
  538. package/fp16/test/alt-from-fp32-value.cc +0 -498
  539. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  540. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  541. package/fp16/test/alt-to-fp32-value.cc +0 -348
  542. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  543. package/fp16/test/bitcasts.cc +0 -56
  544. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  545. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  546. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  547. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  548. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  549. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  550. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  551. package/fp16/test/peachpy/stubs.py +0 -38
  552. package/fp16/test/tables.cc +0 -530
  553. package/fp16/test/tables.h +0 -13
  554. package/fp16/third-party/THHalf.h +0 -92
  555. package/fp16/third-party/eigen-half.h +0 -170
  556. package/fp16/third-party/float16-compressor.h +0 -80
  557. package/fp16/third-party/half.hpp +0 -2916
  558. package/fp16/third-party/npy-halffloat.h +0 -328
  559. package/simsimd/.git-blame-ignore-revs +0 -2
  560. package/simsimd/CONTRIBUTING.md +0 -321
  561. package/simsimd/Cargo.toml +0 -49
  562. package/simsimd/MANIFEST.in +0 -4
  563. package/simsimd/Package.swift +0 -28
  564. package/simsimd/README.md +0 -1493
  565. package/simsimd/binding.gyp +0 -35
  566. package/simsimd/build.rs +0 -70
  567. package/simsimd/c/lib.c +0 -374
  568. package/simsimd/include/module.modulemap +0 -4
  569. package/simsimd/include/simsimd/binary.h +0 -486
  570. package/simsimd/include/simsimd/curved.h +0 -1541
  571. package/simsimd/include/simsimd/dot.h +0 -1853
  572. package/simsimd/include/simsimd/elementwise.h +0 -2465
  573. package/simsimd/include/simsimd/geospatial.h +0 -43
  574. package/simsimd/include/simsimd/mesh.h +0 -69
  575. package/simsimd/include/simsimd/probability.h +0 -606
  576. package/simsimd/include/simsimd/simsimd.h +0 -2567
  577. package/simsimd/include/simsimd/sparse.h +0 -1384
  578. package/simsimd/include/simsimd/spatial.h +0 -2335
  579. package/simsimd/include/simsimd/types.h +0 -666
  580. package/simsimd/javascript/fallback.ts +0 -233
  581. package/simsimd/javascript/lib.c +0 -125
  582. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  583. package/simsimd/javascript/simsimd.ts +0 -179
  584. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  585. package/simsimd/package-lock.json +0 -1761
  586. package/simsimd/package.json +0 -61
  587. package/simsimd/rust/lib.rs +0 -1821
  588. package/simsimd/scripts/bench.cxx +0 -1170
  589. package/simsimd/scripts/bench.ipynb +0 -187
  590. package/simsimd/scripts/bench.js +0 -127
  591. package/simsimd/scripts/bench.rs +0 -876
  592. package/simsimd/scripts/bench_vectors.py +0 -734
  593. package/simsimd/scripts/bench_vectors_live.py +0 -213
  594. package/simsimd/scripts/test.c +0 -187
  595. package/simsimd/scripts/test.mjs +0 -220
  596. package/simsimd/scripts/test.py +0 -1505
  597. package/simsimd/swift/SimSIMD.swift +0 -149
  598. package/simsimd/swift/Test.swift +0 -128
  599. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  600. /package/{simsimd → numkong}/LICENSE +0 -0
  601. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  602. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -0,0 +1,1121 @@
1
+ /**
2
+ * @brief SIMD-accelerated Elementwise Arithmetic for RISC-V.
3
+ * @file include/numkong/each/rvv.h
4
+ * @author Ash Vardanian
5
+ * @date February 6, 2026
6
+ *
7
+ * @sa include/numkong/each.h
8
+ */
9
+ #ifndef NK_EACH_RVV_H
10
+ #define NK_EACH_RVV_H
11
+
12
+ #if NK_TARGET_RISCV64_
13
+ #if NK_TARGET_RVV
14
+
15
+ #include "numkong/types.h"
16
+ #include "numkong/cast/rvv.h"
17
+
18
+ #if defined(__clang__)
19
+ #pragma clang attribute push(__attribute__((target("arch=+v"))), apply_to = function)
20
+ #elif defined(__GNUC__)
21
+ #pragma GCC push_options
22
+ #pragma GCC target("arch=+v")
23
+ #endif
24
+
25
+ #if defined(__cplusplus)
26
+ extern "C" {
27
+ #endif
28
+
29
+ NK_PUBLIC void nk_each_sum_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result) {
30
+ for (nk_size_t vector_length; n > 0;
31
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
32
+ vector_length = __riscv_vsetvl_e64m4(n);
33
+ vfloat64m4_t a_f64m4 = __riscv_vle64_v_f64m4(a, vector_length);
34
+ vfloat64m4_t b_f64m4 = __riscv_vle64_v_f64m4(b, vector_length);
35
+ __riscv_vse64_v_f64m4(result, __riscv_vfadd_vv_f64m4(a_f64m4, b_f64m4, vector_length), vector_length);
36
+ }
37
+ }
38
+
39
+ NK_PUBLIC void nk_each_sum_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *result) {
40
+ for (nk_size_t vector_length; n > 0;
41
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
42
+ vector_length = __riscv_vsetvl_e32m4(n);
43
+ vfloat32m4_t a_f32m4 = __riscv_vle32_v_f32m4(a, vector_length);
44
+ vfloat32m4_t b_f32m4 = __riscv_vle32_v_f32m4(b, vector_length);
45
+ __riscv_vse32_v_f32m4(result, __riscv_vfadd_vv_f32m4(a_f32m4, b_f32m4, vector_length), vector_length);
46
+ }
47
+ }
48
+
49
+ NK_PUBLIC void nk_each_sum_f16_rvv(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f16_t *result) {
50
+ for (nk_size_t vector_length; n > 0;
51
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
52
+ vector_length = __riscv_vsetvl_e16m1(n);
53
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
54
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)b, vector_length);
55
+ vfloat32m2_t a_f32m2 = nk_f16m1_to_f32m2_rvv_(a_u16m1, vector_length);
56
+ vfloat32m2_t b_f32m2 = nk_f16m1_to_f32m2_rvv_(b_u16m1, vector_length);
57
+ vfloat32m2_t result_f32m2 = __riscv_vfadd_vv_f32m2(a_f32m2, b_f32m2, vector_length);
58
+ vuint16m1_t result_u16m1 = nk_f32m2_to_f16m1_rvv_(result_f32m2, vector_length);
59
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
60
+ }
61
+ }
62
+
63
+ NK_PUBLIC void nk_each_sum_bf16_rvv(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_bf16_t *result) {
64
+ for (nk_size_t vector_length; n > 0;
65
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
66
+ vector_length = __riscv_vsetvl_e16m1(n);
67
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
68
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)b, vector_length);
69
+ vfloat32m2_t a_f32m2 = nk_bf16m1_to_f32m2_rvv_(a_u16m1, vector_length);
70
+ vfloat32m2_t b_f32m2 = nk_bf16m1_to_f32m2_rvv_(b_u16m1, vector_length);
71
+ vfloat32m2_t result_f32m2 = __riscv_vfadd_vv_f32m2(a_f32m2, b_f32m2, vector_length);
72
+ vuint16m1_t result_u16m1 = nk_f32m2_to_bf16m1_rvv_(result_f32m2, vector_length);
73
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
74
+ }
75
+ }
76
+
77
+ NK_PUBLIC void nk_each_sum_i8_rvv(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i8_t *result) {
78
+ for (nk_size_t vector_length; n > 0;
79
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
80
+ vector_length = __riscv_vsetvl_e8m4(n);
81
+ vint8m4_t a_i8m4 = __riscv_vle8_v_i8m4(a, vector_length);
82
+ vint8m4_t b_i8m4 = __riscv_vle8_v_i8m4(b, vector_length);
83
+ __riscv_vse8_v_i8m4(result, __riscv_vsadd_vv_i8m4(a_i8m4, b_i8m4, vector_length), vector_length);
84
+ }
85
+ }
86
+
87
+ NK_PUBLIC void nk_each_sum_u8_rvv(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u8_t *result) {
88
+ for (nk_size_t vector_length; n > 0;
89
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
90
+ vector_length = __riscv_vsetvl_e8m4(n);
91
+ vuint8m4_t a_u8m4 = __riscv_vle8_v_u8m4(a, vector_length);
92
+ vuint8m4_t b_u8m4 = __riscv_vle8_v_u8m4(b, vector_length);
93
+ __riscv_vse8_v_u8m4(result, __riscv_vsaddu_vv_u8m4(a_u8m4, b_u8m4, vector_length), vector_length);
94
+ }
95
+ }
96
+
97
+ NK_PUBLIC void nk_each_sum_i16_rvv(nk_i16_t const *a, nk_i16_t const *b, nk_size_t n, nk_i16_t *result) {
98
+ for (nk_size_t vector_length; n > 0;
99
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
100
+ vector_length = __riscv_vsetvl_e16m4(n);
101
+ vint16m4_t a_i16m4 = __riscv_vle16_v_i16m4(a, vector_length);
102
+ vint16m4_t b_i16m4 = __riscv_vle16_v_i16m4(b, vector_length);
103
+ __riscv_vse16_v_i16m4(result, __riscv_vsadd_vv_i16m4(a_i16m4, b_i16m4, vector_length), vector_length);
104
+ }
105
+ }
106
+
107
+ NK_PUBLIC void nk_each_sum_u16_rvv(nk_u16_t const *a, nk_u16_t const *b, nk_size_t n, nk_u16_t *result) {
108
+ for (nk_size_t vector_length; n > 0;
109
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
110
+ vector_length = __riscv_vsetvl_e16m4(n);
111
+ vuint16m4_t a_u16m4 = __riscv_vle16_v_u16m4(a, vector_length);
112
+ vuint16m4_t b_u16m4 = __riscv_vle16_v_u16m4(b, vector_length);
113
+ __riscv_vse16_v_u16m4(result, __riscv_vsaddu_vv_u16m4(a_u16m4, b_u16m4, vector_length), vector_length);
114
+ }
115
+ }
116
+
117
+ NK_PUBLIC void nk_each_sum_i32_rvv(nk_i32_t const *a, nk_i32_t const *b, nk_size_t n, nk_i32_t *result) {
118
+ for (nk_size_t vector_length; n > 0;
119
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
120
+ vector_length = __riscv_vsetvl_e32m4(n);
121
+ vint32m4_t a_i32m4 = __riscv_vle32_v_i32m4(a, vector_length);
122
+ vint32m4_t b_i32m4 = __riscv_vle32_v_i32m4(b, vector_length);
123
+ __riscv_vse32_v_i32m4(result, __riscv_vsadd_vv_i32m4(a_i32m4, b_i32m4, vector_length), vector_length);
124
+ }
125
+ }
126
+
127
+ NK_PUBLIC void nk_each_sum_u32_rvv(nk_u32_t const *a, nk_u32_t const *b, nk_size_t n, nk_u32_t *result) {
128
+ for (nk_size_t vector_length; n > 0;
129
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
130
+ vector_length = __riscv_vsetvl_e32m4(n);
131
+ vuint32m4_t a_u32m4 = __riscv_vle32_v_u32m4(a, vector_length);
132
+ vuint32m4_t b_u32m4 = __riscv_vle32_v_u32m4(b, vector_length);
133
+ __riscv_vse32_v_u32m4(result, __riscv_vsaddu_vv_u32m4(a_u32m4, b_u32m4, vector_length), vector_length);
134
+ }
135
+ }
136
+
137
+ NK_PUBLIC void nk_each_sum_i64_rvv(nk_i64_t const *a, nk_i64_t const *b, nk_size_t n, nk_i64_t *result) {
138
+ for (nk_size_t vector_length; n > 0;
139
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
140
+ vector_length = __riscv_vsetvl_e64m4(n);
141
+ vint64m4_t a_i64m4 = __riscv_vle64_v_i64m4(a, vector_length);
142
+ vint64m4_t b_i64m4 = __riscv_vle64_v_i64m4(b, vector_length);
143
+ __riscv_vse64_v_i64m4(result, __riscv_vsadd_vv_i64m4(a_i64m4, b_i64m4, vector_length), vector_length);
144
+ }
145
+ }
146
+
147
+ NK_PUBLIC void nk_each_sum_u64_rvv(nk_u64_t const *a, nk_u64_t const *b, nk_size_t n, nk_u64_t *result) {
148
+ for (nk_size_t vector_length; n > 0;
149
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
150
+ vector_length = __riscv_vsetvl_e64m4(n);
151
+ vuint64m4_t a_u64m4 = __riscv_vle64_v_u64m4(a, vector_length);
152
+ vuint64m4_t b_u64m4 = __riscv_vle64_v_u64m4(b, vector_length);
153
+ __riscv_vse64_v_u64m4(result, __riscv_vsaddu_vv_u64m4(a_u64m4, b_u64m4, vector_length), vector_length);
154
+ }
155
+ }
156
+
157
+ NK_PUBLIC void nk_each_sum_e4m3_rvv(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_e4m3_t *result) {
158
+ for (nk_size_t vector_length; n > 0;
159
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
160
+ vector_length = __riscv_vsetvl_e8m1(n);
161
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
162
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b, vector_length);
163
+ vfloat32m4_t a_f32m4 = nk_e4m3m1_to_f32m4_rvv_(a_u8m1, vector_length);
164
+ vfloat32m4_t b_f32m4 = nk_e4m3m1_to_f32m4_rvv_(b_u8m1, vector_length);
165
+ vfloat32m4_t result_f32m4 = __riscv_vfadd_vv_f32m4(a_f32m4, b_f32m4, vector_length);
166
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e4m3m1_rvv_(result_f32m4, vector_length);
167
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
168
+ }
169
+ }
170
+
171
+ NK_PUBLIC void nk_each_sum_e5m2_rvv(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_e5m2_t *result) {
172
+ for (nk_size_t vector_length; n > 0;
173
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
174
+ vector_length = __riscv_vsetvl_e8m1(n);
175
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
176
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b, vector_length);
177
+ vfloat32m4_t a_f32m4 = nk_e5m2m1_to_f32m4_rvv_(a_u8m1, vector_length);
178
+ vfloat32m4_t b_f32m4 = nk_e5m2m1_to_f32m4_rvv_(b_u8m1, vector_length);
179
+ vfloat32m4_t result_f32m4 = __riscv_vfadd_vv_f32m4(a_f32m4, b_f32m4, vector_length);
180
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e5m2m1_rvv_(result_f32m4, vector_length);
181
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
182
+ }
183
+ }
184
+
185
+ NK_PUBLIC void nk_each_scale_f64_rvv(nk_f64_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
186
+ nk_f64_t *result) {
187
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
188
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
189
+ vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, max_vector_length);
190
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
191
+ vector_length = __riscv_vsetvl_e64m4(n);
192
+ vfloat64m4_t a_f64m4 = __riscv_vle64_v_f64m4(a, vector_length);
193
+ a_f64m4 = __riscv_vfmadd_vf_f64m4(a_f64m4, alpha_val, beta_f64m4, vector_length);
194
+ __riscv_vse64_v_f64m4(result, a_f64m4, vector_length);
195
+ }
196
+ }
197
+
198
+ NK_PUBLIC void nk_each_scale_f32_rvv(nk_f32_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
199
+ nk_f32_t *result) {
200
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
201
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
202
+ vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
203
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
204
+ vector_length = __riscv_vsetvl_e32m4(n);
205
+ vfloat32m4_t a_f32m4 = __riscv_vle32_v_f32m4(a, vector_length);
206
+ a_f32m4 = __riscv_vfmadd_vf_f32m4(a_f32m4, alpha_val, beta_f32m4, vector_length);
207
+ __riscv_vse32_v_f32m4(result, a_f32m4, vector_length);
208
+ }
209
+ }
210
+
211
+ NK_PUBLIC void nk_each_scale_f16_rvv(nk_f16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
212
+ nk_f16_t *result) {
213
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
214
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
215
+ vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
216
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
217
+ vector_length = __riscv_vsetvl_e16m1(n);
218
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
219
+ vfloat32m2_t a_f32m2 = nk_f16m1_to_f32m2_rvv_(a_u16m1, vector_length);
220
+ a_f32m2 = __riscv_vfmadd_vf_f32m2(a_f32m2, alpha_val, beta_f32m2, vector_length);
221
+ vuint16m1_t result_u16m1 = nk_f32m2_to_f16m1_rvv_(a_f32m2, vector_length);
222
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
223
+ }
224
+ }
225
+
226
+ NK_PUBLIC void nk_each_scale_bf16_rvv(nk_bf16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
227
+ nk_bf16_t *result) {
228
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
229
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
230
+ vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
231
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
232
+ vector_length = __riscv_vsetvl_e16m1(n);
233
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
234
+ vfloat32m2_t a_f32m2 = nk_bf16m1_to_f32m2_rvv_(a_u16m1, vector_length);
235
+ a_f32m2 = __riscv_vfmadd_vf_f32m2(a_f32m2, alpha_val, beta_f32m2, vector_length);
236
+ vuint16m1_t result_u16m1 = nk_f32m2_to_bf16m1_rvv_(a_f32m2, vector_length);
237
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
238
+ }
239
+ }
240
+
241
+ NK_PUBLIC void nk_each_scale_i8_rvv(nk_i8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
242
+ nk_i8_t *result) {
243
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
244
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
245
+ vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
246
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
247
+ vector_length = __riscv_vsetvl_e8m1(n);
248
+ vint8m1_t a_i8m1 = __riscv_vle8_v_i8m1(a, vector_length);
249
+ vint16m2_t a_i16m2 = __riscv_vwadd_vx_i16m2(a_i8m1, 0, vector_length);
250
+ vint32m4_t a_i32m4 = __riscv_vwadd_vx_i32m4(a_i16m2, 0, vector_length);
251
+ vfloat32m4_t a_f32m4 = __riscv_vfcvt_f_x_v_f32m4(a_i32m4, vector_length);
252
+ a_f32m4 = __riscv_vfmadd_vf_f32m4(a_f32m4, alpha_val, beta_f32m4, vector_length);
253
+ vint32m4_t result_i32m4 = __riscv_vfcvt_x_f_v_i32m4(a_f32m4, vector_length);
254
+ result_i32m4 = __riscv_vmax_vx_i32m4(result_i32m4, -128, vector_length);
255
+ result_i32m4 = __riscv_vmin_vx_i32m4(result_i32m4, 127, vector_length);
256
+ vint16m2_t result_i16m2 = __riscv_vncvt_x_x_w_i16m2(result_i32m4, vector_length);
257
+ vint8m1_t result_i8m1 = __riscv_vncvt_x_x_w_i8m1(result_i16m2, vector_length);
258
+ __riscv_vse8_v_i8m1(result, result_i8m1, vector_length);
259
+ }
260
+ }
261
+
262
+ NK_PUBLIC void nk_each_scale_u8_rvv(nk_u8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
263
+ nk_u8_t *result) {
264
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
265
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
266
+ vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
267
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
268
+ vector_length = __riscv_vsetvl_e8m1(n);
269
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1(a, vector_length);
270
+ vuint16m2_t a_u16m2 = __riscv_vwaddu_vx_u16m2(a_u8m1, 0, vector_length);
271
+ vuint32m4_t a_u32m4 = __riscv_vwaddu_vx_u32m4(a_u16m2, 0, vector_length);
272
+ vfloat32m4_t a_f32m4 = __riscv_vfcvt_f_xu_v_f32m4(a_u32m4, vector_length);
273
+ a_f32m4 = __riscv_vfmadd_vf_f32m4(a_f32m4, alpha_val, beta_f32m4, vector_length);
274
+ vuint32m4_t result_u32m4 = __riscv_vfcvt_xu_f_v_u32m4(a_f32m4, vector_length);
275
+ result_u32m4 = __riscv_vmaxu_vx_u32m4(result_u32m4, 0, vector_length);
276
+ result_u32m4 = __riscv_vminu_vx_u32m4(result_u32m4, 255, vector_length);
277
+ vuint16m2_t result_u16m2 = __riscv_vncvt_x_x_w_u16m2(result_u32m4, vector_length);
278
+ vuint8m1_t result_u8m1 = __riscv_vncvt_x_x_w_u8m1(result_u16m2, vector_length);
279
+ __riscv_vse8_v_u8m1(result, result_u8m1, vector_length);
280
+ }
281
+ }
282
+
283
+ NK_PUBLIC void nk_each_scale_i16_rvv(nk_i16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
284
+ nk_i16_t *result) {
285
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
286
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
287
+ vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
288
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
289
+ vector_length = __riscv_vsetvl_e16m1(n);
290
+ vint16m1_t a_i16m1 = __riscv_vle16_v_i16m1(a, vector_length);
291
+ vint32m2_t a_i32m2 = __riscv_vwadd_vx_i32m2(a_i16m1, 0, vector_length);
292
+ vfloat32m2_t a_f32m2 = __riscv_vfcvt_f_x_v_f32m2(a_i32m2, vector_length);
293
+ a_f32m2 = __riscv_vfmadd_vf_f32m2(a_f32m2, alpha_val, beta_f32m2, vector_length);
294
+ vint32m2_t result_i32m2 = __riscv_vfcvt_x_f_v_i32m2(a_f32m2, vector_length);
295
+ result_i32m2 = __riscv_vmax_vx_i32m2(result_i32m2, -32768, vector_length);
296
+ result_i32m2 = __riscv_vmin_vx_i32m2(result_i32m2, 32767, vector_length);
297
+ vint16m1_t result_i16m1 = __riscv_vncvt_x_x_w_i16m1(result_i32m2, vector_length);
298
+ __riscv_vse16_v_i16m1(result, result_i16m1, vector_length);
299
+ }
300
+ }
301
+
302
+ NK_PUBLIC void nk_each_scale_u16_rvv(nk_u16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
303
+ nk_u16_t *result) {
304
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
305
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
306
+ vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
307
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
308
+ vector_length = __riscv_vsetvl_e16m1(n);
309
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1(a, vector_length);
310
+ vuint32m2_t a_u32m2 = __riscv_vwaddu_vx_u32m2(a_u16m1, 0, vector_length);
311
+ vfloat32m2_t a_f32m2 = __riscv_vfcvt_f_xu_v_f32m2(a_u32m2, vector_length);
312
+ a_f32m2 = __riscv_vfmadd_vf_f32m2(a_f32m2, alpha_val, beta_f32m2, vector_length);
313
+ vuint32m2_t result_u32m2 = __riscv_vfcvt_xu_f_v_u32m2(a_f32m2, vector_length);
314
+ result_u32m2 = __riscv_vmaxu_vx_u32m2(result_u32m2, 0, vector_length);
315
+ result_u32m2 = __riscv_vminu_vx_u32m2(result_u32m2, 65535, vector_length);
316
+ vuint16m1_t result_u16m1 = __riscv_vncvt_x_x_w_u16m1(result_u32m2, vector_length);
317
+ __riscv_vse16_v_u16m1(result, result_u16m1, vector_length);
318
+ }
319
+ }
320
+
321
+ NK_PUBLIC void nk_each_scale_i32_rvv(nk_i32_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
322
+ nk_i32_t *result) {
323
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
324
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
325
+ vfloat64m2_t beta_f64m2 = __riscv_vfmv_v_f_f64m2(beta_val, max_vector_length);
326
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
327
+ vector_length = __riscv_vsetvl_e32m1(n);
328
+ vint32m1_t a_i32m1 = __riscv_vle32_v_i32m1(a, vector_length);
329
+ vfloat64m2_t a_f64m2 = __riscv_vfwcvt_f_x_v_f64m2(a_i32m1, vector_length);
330
+ a_f64m2 = __riscv_vfmadd_vf_f64m2(a_f64m2, alpha_val, beta_f64m2, vector_length);
331
+ a_f64m2 = __riscv_vfmax_vf_f64m2(a_f64m2, -2147483648.0, vector_length);
332
+ a_f64m2 = __riscv_vfmin_vf_f64m2(a_f64m2, 2147483647.0, vector_length);
333
+ vint32m1_t result_i32m1 = __riscv_vfncvt_x_f_w_i32m1(a_f64m2, vector_length);
334
+ __riscv_vse32_v_i32m1(result, result_i32m1, vector_length);
335
+ }
336
+ }
337
+
338
+ NK_PUBLIC void nk_each_scale_u32_rvv(nk_u32_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
339
+ nk_u32_t *result) {
340
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
341
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
342
+ vfloat64m2_t beta_f64m2 = __riscv_vfmv_v_f_f64m2(beta_val, max_vector_length);
343
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
344
+ vector_length = __riscv_vsetvl_e32m1(n);
345
+ vuint32m1_t a_u32m1 = __riscv_vle32_v_u32m1(a, vector_length);
346
+ vfloat64m2_t a_f64m2 = __riscv_vfwcvt_f_xu_v_f64m2(a_u32m1, vector_length);
347
+ a_f64m2 = __riscv_vfmadd_vf_f64m2(a_f64m2, alpha_val, beta_f64m2, vector_length);
348
+ a_f64m2 = __riscv_vfmax_vf_f64m2(a_f64m2, 0.0, vector_length);
349
+ a_f64m2 = __riscv_vfmin_vf_f64m2(a_f64m2, 4294967295.0, vector_length);
350
+ vuint32m1_t result_u32m1 = __riscv_vfncvt_xu_f_w_u32m1(a_f64m2, vector_length);
351
+ __riscv_vse32_v_u32m1(result, result_u32m1, vector_length);
352
+ }
353
+ }
354
+
355
+ NK_PUBLIC void nk_each_scale_i64_rvv(nk_i64_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
356
+ nk_i64_t *result) {
357
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
358
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
359
+ vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, max_vector_length);
360
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
361
+ vector_length = __riscv_vsetvl_e64m4(n);
362
+ vint64m4_t a_i64m4 = __riscv_vle64_v_i64m4(a, vector_length);
363
+ vfloat64m4_t a_f64m4 = __riscv_vfcvt_f_x_v_f64m4(a_i64m4, vector_length);
364
+ a_f64m4 = __riscv_vfmadd_vf_f64m4(a_f64m4, alpha_val, beta_f64m4, vector_length);
365
+ vint64m4_t result_i64m4 = __riscv_vfcvt_x_f_v_i64m4(a_f64m4, vector_length);
366
+ __riscv_vse64_v_i64m4(result, result_i64m4, vector_length);
367
+ }
368
+ }
369
+
370
+ NK_PUBLIC void nk_each_scale_u64_rvv(nk_u64_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
371
+ nk_u64_t *result) {
372
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
373
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
374
+ vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, max_vector_length);
375
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
376
+ vector_length = __riscv_vsetvl_e64m4(n);
377
+ vuint64m4_t a_u64m4 = __riscv_vle64_v_u64m4(a, vector_length);
378
+ vfloat64m4_t a_f64m4 = __riscv_vfcvt_f_xu_v_f64m4(a_u64m4, vector_length);
379
+ a_f64m4 = __riscv_vfmadd_vf_f64m4(a_f64m4, alpha_val, beta_f64m4, vector_length);
380
+ a_f64m4 = __riscv_vfmax_vf_f64m4(a_f64m4, 0.0, vector_length);
381
+ vuint64m4_t result_u64m4 = __riscv_vfcvt_xu_f_v_u64m4(a_f64m4, vector_length);
382
+ __riscv_vse64_v_u64m4(result, result_u64m4, vector_length);
383
+ }
384
+ }
385
+
386
+ NK_PUBLIC void nk_each_scale_e4m3_rvv(nk_e4m3_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
387
+ nk_e4m3_t *result) {
388
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
389
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
390
+ vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
391
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
392
+ vector_length = __riscv_vsetvl_e8m1(n);
393
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
394
+ vfloat32m4_t a_f32m4 = nk_e4m3m1_to_f32m4_rvv_(a_u8m1, vector_length);
395
+ a_f32m4 = __riscv_vfmadd_vf_f32m4(a_f32m4, alpha_val, beta_f32m4, vector_length);
396
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e4m3m1_rvv_(a_f32m4, vector_length);
397
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
398
+ }
399
+ }
400
+
401
+ NK_PUBLIC void nk_each_scale_e5m2_rvv(nk_e5m2_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
402
+ nk_e5m2_t *result) {
403
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
404
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
405
+ vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
406
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
407
+ vector_length = __riscv_vsetvl_e8m1(n);
408
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
409
+ vfloat32m4_t a_f32m4 = nk_e5m2m1_to_f32m4_rvv_(a_u8m1, vector_length);
410
+ a_f32m4 = __riscv_vfmadd_vf_f32m4(a_f32m4, alpha_val, beta_f32m4, vector_length);
411
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e5m2m1_rvv_(a_f32m4, vector_length);
412
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
413
+ }
414
+ }
415
+
416
+ NK_PUBLIC void nk_each_blend_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t const *alpha,
417
+ nk_f64_t const *beta, nk_f64_t *result) {
418
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
419
+ if (alpha_val == 1 && beta_val == 1) {
420
+ nk_each_sum_f64_rvv(a, b, n, result);
421
+ return;
422
+ }
423
+ else if (alpha_val == 0 || beta_val == 0) {
424
+ nk_f64_t zero = 0;
425
+ if (beta_val == 0) { nk_each_scale_f64_rvv(a, n, alpha, &zero, result); }
426
+ else { nk_each_scale_f64_rvv(b, n, beta, &zero, result); }
427
+ return;
428
+ }
429
+ for (nk_size_t vector_length; n > 0;
430
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
431
+ vector_length = __riscv_vsetvl_e64m4(n);
432
+ vfloat64m4_t a_f64m4 = __riscv_vle64_v_f64m4(a, vector_length);
433
+ vfloat64m4_t b_f64m4 = __riscv_vle64_v_f64m4(b, vector_length);
434
+ vfloat64m4_t result_f64m4 = __riscv_vfmul_vf_f64m4(a_f64m4, alpha_val, vector_length);
435
+ result_f64m4 = __riscv_vfmacc_vf_f64m4(result_f64m4, beta_val, b_f64m4, vector_length);
436
+ __riscv_vse64_v_f64m4(result, result_f64m4, vector_length);
437
+ }
438
+ }
439
+
440
+ NK_PUBLIC void nk_each_blend_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t const *alpha,
441
+ nk_f32_t const *beta, nk_f32_t *result) {
442
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
443
+ if (alpha_val == 1 && beta_val == 1) {
444
+ nk_each_sum_f32_rvv(a, b, n, result);
445
+ return;
446
+ }
447
+ else if (alpha_val == 0 || beta_val == 0) {
448
+ nk_f32_t zero = 0;
449
+ if (beta_val == 0) { nk_each_scale_f32_rvv(a, n, alpha, &zero, result); }
450
+ else { nk_each_scale_f32_rvv(b, n, beta, &zero, result); }
451
+ return;
452
+ }
453
+ for (nk_size_t vector_length; n > 0;
454
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
455
+ vector_length = __riscv_vsetvl_e32m4(n);
456
+ vfloat32m4_t a_f32m4 = __riscv_vle32_v_f32m4(a, vector_length);
457
+ vfloat32m4_t b_f32m4 = __riscv_vle32_v_f32m4(b, vector_length);
458
+ vfloat32m4_t a_scaled_f32m4 = __riscv_vfmul_vf_f32m4(a_f32m4, alpha_val, vector_length);
459
+ vfloat32m4_t result_f32m4 = __riscv_vfmacc_vf_f32m4(a_scaled_f32m4, beta_val, b_f32m4, vector_length);
460
+ __riscv_vse32_v_f32m4(result, result_f32m4, vector_length);
461
+ }
462
+ }
463
+
464
+ NK_PUBLIC void nk_each_blend_f16_rvv(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t const *alpha,
465
+ nk_f32_t const *beta, nk_f16_t *result) {
466
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
467
+ if (alpha_val == 1 && beta_val == 1) {
468
+ nk_each_sum_f16_rvv(a, b, n, result);
469
+ return;
470
+ }
471
+ else if (alpha_val == 0 || beta_val == 0) {
472
+ nk_f32_t zero = 0;
473
+ if (beta_val == 0) { nk_each_scale_f16_rvv(a, n, alpha, &zero, result); }
474
+ else { nk_each_scale_f16_rvv(b, n, beta, &zero, result); }
475
+ return;
476
+ }
477
+ for (nk_size_t vector_length; n > 0;
478
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
479
+ vector_length = __riscv_vsetvl_e16m1(n);
480
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
481
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)b, vector_length);
482
+ vfloat32m2_t a_f32m2 = nk_f16m1_to_f32m2_rvv_(a_u16m1, vector_length);
483
+ vfloat32m2_t b_f32m2 = nk_f16m1_to_f32m2_rvv_(b_u16m1, vector_length);
484
+ vfloat32m2_t result_f32m2 = __riscv_vfmul_vf_f32m2(a_f32m2, alpha_val, vector_length);
485
+ result_f32m2 = __riscv_vfmacc_vf_f32m2(result_f32m2, beta_val, b_f32m2, vector_length);
486
+ vuint16m1_t result_u16m1 = nk_f32m2_to_f16m1_rvv_(result_f32m2, vector_length);
487
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
488
+ }
489
+ }
490
+
491
+ NK_PUBLIC void nk_each_blend_bf16_rvv(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t const *alpha,
492
+ nk_f32_t const *beta, nk_bf16_t *result) {
493
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
494
+ if (alpha_val == 1 && beta_val == 1) {
495
+ nk_each_sum_bf16_rvv(a, b, n, result);
496
+ return;
497
+ }
498
+ else if (alpha_val == 0 || beta_val == 0) {
499
+ nk_f32_t zero = 0;
500
+ if (beta_val == 0) { nk_each_scale_bf16_rvv(a, n, alpha, &zero, result); }
501
+ else { nk_each_scale_bf16_rvv(b, n, beta, &zero, result); }
502
+ return;
503
+ }
504
+ for (nk_size_t vector_length; n > 0;
505
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
506
+ vector_length = __riscv_vsetvl_e16m1(n);
507
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
508
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)b, vector_length);
509
+ vfloat32m2_t a_f32m2 = nk_bf16m1_to_f32m2_rvv_(a_u16m1, vector_length);
510
+ vfloat32m2_t b_f32m2 = nk_bf16m1_to_f32m2_rvv_(b_u16m1, vector_length);
511
+ vfloat32m2_t result_f32m2 = __riscv_vfmul_vf_f32m2(a_f32m2, alpha_val, vector_length);
512
+ result_f32m2 = __riscv_vfmacc_vf_f32m2(result_f32m2, beta_val, b_f32m2, vector_length);
513
+ vuint16m1_t result_u16m1 = nk_f32m2_to_bf16m1_rvv_(result_f32m2, vector_length);
514
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
515
+ }
516
+ }
517
+
518
+ NK_PUBLIC void nk_each_blend_i8_rvv(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t const *alpha,
519
+ nk_f32_t const *beta, nk_i8_t *result) {
520
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
521
+ if (alpha_val == 1 && beta_val == 1) {
522
+ nk_each_sum_i8_rvv(a, b, n, result);
523
+ return;
524
+ }
525
+ else if (alpha_val == 0 || beta_val == 0) {
526
+ nk_f32_t zero = 0;
527
+ if (beta_val == 0) { nk_each_scale_i8_rvv(a, n, alpha, &zero, result); }
528
+ else { nk_each_scale_i8_rvv(b, n, beta, &zero, result); }
529
+ return;
530
+ }
531
+ for (nk_size_t vector_length; n > 0;
532
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
533
+ vector_length = __riscv_vsetvl_e8m1(n);
534
+ vint8m1_t a_i8m1 = __riscv_vle8_v_i8m1(a, vector_length);
535
+ vint8m1_t b_i8m1 = __riscv_vle8_v_i8m1(b, vector_length);
536
+ vint16m2_t a_i16m2 = __riscv_vwadd_vx_i16m2(a_i8m1, 0, vector_length);
537
+ vint32m4_t a_i32m4 = __riscv_vwadd_vx_i32m4(a_i16m2, 0, vector_length);
538
+ vfloat32m4_t a_f32m4 = __riscv_vfcvt_f_x_v_f32m4(a_i32m4, vector_length);
539
+ vint16m2_t b_i16m2 = __riscv_vwadd_vx_i16m2(b_i8m1, 0, vector_length);
540
+ vint32m4_t b_i32m4 = __riscv_vwadd_vx_i32m4(b_i16m2, 0, vector_length);
541
+ vfloat32m4_t b_f32m4 = __riscv_vfcvt_f_x_v_f32m4(b_i32m4, vector_length);
542
+ vfloat32m4_t result_f32m4 = __riscv_vfmul_vf_f32m4(a_f32m4, alpha_val, vector_length);
543
+ result_f32m4 = __riscv_vfmacc_vf_f32m4(result_f32m4, beta_val, b_f32m4, vector_length);
544
+ vint32m4_t result_i32m4 = __riscv_vfcvt_x_f_v_i32m4(result_f32m4, vector_length);
545
+ result_i32m4 = __riscv_vmax_vx_i32m4(result_i32m4, -128, vector_length);
546
+ result_i32m4 = __riscv_vmin_vx_i32m4(result_i32m4, 127, vector_length);
547
+ vint16m2_t result_i16m2 = __riscv_vncvt_x_x_w_i16m2(result_i32m4, vector_length);
548
+ vint8m1_t result_i8m1 = __riscv_vncvt_x_x_w_i8m1(result_i16m2, vector_length);
549
+ __riscv_vse8_v_i8m1(result, result_i8m1, vector_length);
550
+ }
551
+ }
552
+
553
+ NK_PUBLIC void nk_each_blend_u8_rvv(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t const *alpha,
554
+ nk_f32_t const *beta, nk_u8_t *result) {
555
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
556
+ if (alpha_val == 1 && beta_val == 1) {
557
+ nk_each_sum_u8_rvv(a, b, n, result);
558
+ return;
559
+ }
560
+ else if (alpha_val == 0 || beta_val == 0) {
561
+ nk_f32_t zero = 0;
562
+ if (beta_val == 0) { nk_each_scale_u8_rvv(a, n, alpha, &zero, result); }
563
+ else { nk_each_scale_u8_rvv(b, n, beta, &zero, result); }
564
+ return;
565
+ }
566
+ for (nk_size_t vector_length; n > 0;
567
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
568
+ vector_length = __riscv_vsetvl_e8m1(n);
569
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1(a, vector_length);
570
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1(b, vector_length);
571
+ vuint16m2_t a_u16m2 = __riscv_vwaddu_vx_u16m2(a_u8m1, 0, vector_length);
572
+ vuint32m4_t a_u32m4 = __riscv_vwaddu_vx_u32m4(a_u16m2, 0, vector_length);
573
+ vfloat32m4_t a_f32m4 = __riscv_vfcvt_f_xu_v_f32m4(a_u32m4, vector_length);
574
+ vuint16m2_t b_u16m2 = __riscv_vwaddu_vx_u16m2(b_u8m1, 0, vector_length);
575
+ vuint32m4_t b_u32m4 = __riscv_vwaddu_vx_u32m4(b_u16m2, 0, vector_length);
576
+ vfloat32m4_t b_f32m4 = __riscv_vfcvt_f_xu_v_f32m4(b_u32m4, vector_length);
577
+ vfloat32m4_t result_f32m4 = __riscv_vfmul_vf_f32m4(a_f32m4, alpha_val, vector_length);
578
+ result_f32m4 = __riscv_vfmacc_vf_f32m4(result_f32m4, beta_val, b_f32m4, vector_length);
579
+ vuint32m4_t result_u32m4 = __riscv_vfcvt_xu_f_v_u32m4(result_f32m4, vector_length);
580
+ result_u32m4 = __riscv_vmaxu_vx_u32m4(result_u32m4, 0, vector_length);
581
+ result_u32m4 = __riscv_vminu_vx_u32m4(result_u32m4, 255, vector_length);
582
+ vuint16m2_t result_u16m2 = __riscv_vncvt_x_x_w_u16m2(result_u32m4, vector_length);
583
+ vuint8m1_t result_u8m1 = __riscv_vncvt_x_x_w_u8m1(result_u16m2, vector_length);
584
+ __riscv_vse8_v_u8m1(result, result_u8m1, vector_length);
585
+ }
586
+ }
587
+
588
+ NK_PUBLIC void nk_each_blend_e4m3_rvv(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t const *alpha,
589
+ nk_f32_t const *beta, nk_e4m3_t *result) {
590
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
591
+ for (nk_size_t vector_length; n > 0;
592
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
593
+ vector_length = __riscv_vsetvl_e8m1(n);
594
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
595
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b, vector_length);
596
+ vfloat32m4_t a_f32m4 = nk_e4m3m1_to_f32m4_rvv_(a_u8m1, vector_length);
597
+ vfloat32m4_t b_f32m4 = nk_e4m3m1_to_f32m4_rvv_(b_u8m1, vector_length);
598
+ vfloat32m4_t result_f32m4 = __riscv_vfmul_vf_f32m4(a_f32m4, alpha_val, vector_length);
599
+ result_f32m4 = __riscv_vfmacc_vf_f32m4(result_f32m4, beta_val, b_f32m4, vector_length);
600
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e4m3m1_rvv_(result_f32m4, vector_length);
601
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
602
+ }
603
+ }
604
+
605
+ NK_PUBLIC void nk_each_blend_e5m2_rvv(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t const *alpha,
606
+ nk_f32_t const *beta, nk_e5m2_t *result) {
607
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
608
+ for (nk_size_t vector_length; n > 0;
609
+ n -= vector_length, a += vector_length, b += vector_length, result += vector_length) {
610
+ vector_length = __riscv_vsetvl_e8m1(n);
611
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
612
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b, vector_length);
613
+ vfloat32m4_t a_f32m4 = nk_e5m2m1_to_f32m4_rvv_(a_u8m1, vector_length);
614
+ vfloat32m4_t b_f32m4 = nk_e5m2m1_to_f32m4_rvv_(b_u8m1, vector_length);
615
+ vfloat32m4_t result_f32m4 = __riscv_vfmul_vf_f32m4(a_f32m4, alpha_val, vector_length);
616
+ result_f32m4 = __riscv_vfmacc_vf_f32m4(result_f32m4, beta_val, b_f32m4, vector_length);
617
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e5m2m1_rvv_(result_f32m4, vector_length);
618
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
619
+ }
620
+ }
621
+
622
+ NK_PUBLIC void nk_each_fma_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
623
+ nk_f64_t const *alpha, nk_f64_t const *beta, nk_f64_t *result) {
624
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
625
+ for (nk_size_t vector_length; n > 0;
626
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
627
+ vector_length = __riscv_vsetvl_e64m4(n);
628
+ vfloat64m4_t a_f64m4 = __riscv_vle64_v_f64m4(a, vector_length);
629
+ vfloat64m4_t b_f64m4 = __riscv_vle64_v_f64m4(b, vector_length);
630
+ vfloat64m4_t c_f64m4 = __riscv_vle64_v_f64m4(c, vector_length);
631
+ vfloat64m4_t product_f64m4 = __riscv_vfmul_vv_f64m4(a_f64m4, b_f64m4, vector_length);
632
+ vfloat64m4_t scaled_product_f64m4 = __riscv_vfmul_vf_f64m4(product_f64m4, alpha_val, vector_length);
633
+ vfloat64m4_t result_f64m4 = __riscv_vfmacc_vf_f64m4(scaled_product_f64m4, beta_val, c_f64m4, vector_length);
634
+ __riscv_vse64_v_f64m4(result, result_f64m4, vector_length);
635
+ }
636
+ }
637
+
638
+ NK_PUBLIC void nk_each_fma_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n,
639
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_f32_t *result) {
640
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
641
+ for (nk_size_t vector_length; n > 0;
642
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
643
+ vector_length = __riscv_vsetvl_e32m4(n);
644
+ vfloat32m4_t a_f32m4 = __riscv_vle32_v_f32m4(a, vector_length);
645
+ vfloat32m4_t b_f32m4 = __riscv_vle32_v_f32m4(b, vector_length);
646
+ vfloat32m4_t c_f32m4 = __riscv_vle32_v_f32m4(c, vector_length);
647
+ vfloat32m4_t product_f32m4 = __riscv_vfmul_vv_f32m4(a_f32m4, b_f32m4, vector_length);
648
+ vfloat32m4_t scaled_product_f32m4 = __riscv_vfmul_vf_f32m4(product_f32m4, alpha_val, vector_length);
649
+ vfloat32m4_t result_f32m4 = __riscv_vfmacc_vf_f32m4(scaled_product_f32m4, beta_val, c_f32m4, vector_length);
650
+ __riscv_vse32_v_f32m4(result, result_f32m4, vector_length);
651
+ }
652
+ }
653
+
654
+ NK_PUBLIC void nk_each_fma_f16_rvv(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
655
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_f16_t *result) {
656
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
657
+ for (nk_size_t vector_length; n > 0;
658
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
659
+ vector_length = __riscv_vsetvl_e16m1(n);
660
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
661
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)b, vector_length);
662
+ vuint16m1_t c_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)c, vector_length);
663
+ vfloat32m2_t a_f32m2 = nk_f16m1_to_f32m2_rvv_(a_u16m1, vector_length);
664
+ vfloat32m2_t b_f32m2 = nk_f16m1_to_f32m2_rvv_(b_u16m1, vector_length);
665
+ vfloat32m2_t c_f32m2 = nk_f16m1_to_f32m2_rvv_(c_u16m1, vector_length);
666
+ vfloat32m2_t product_f32m2 = __riscv_vfmul_vv_f32m2(a_f32m2, b_f32m2, vector_length);
667
+ vfloat32m2_t scaled_product_f32m2 = __riscv_vfmul_vf_f32m2(product_f32m2, alpha_val, vector_length);
668
+ vfloat32m2_t result_f32m2 = __riscv_vfmacc_vf_f32m2(scaled_product_f32m2, beta_val, c_f32m2, vector_length);
669
+ vuint16m1_t result_u16m1 = nk_f32m2_to_f16m1_rvv_(result_f32m2, vector_length);
670
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
671
+ }
672
+ }
673
+
674
+ NK_PUBLIC void nk_each_fma_bf16_rvv(nk_bf16_t const *a, nk_bf16_t const *b, nk_bf16_t const *c, nk_size_t n,
675
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_bf16_t *result) {
676
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
677
+ for (nk_size_t vector_length; n > 0;
678
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
679
+ vector_length = __riscv_vsetvl_e16m1(n);
680
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
681
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)b, vector_length);
682
+ vuint16m1_t c_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)c, vector_length);
683
+ vfloat32m2_t a_f32m2 = nk_bf16m1_to_f32m2_rvv_(a_u16m1, vector_length);
684
+ vfloat32m2_t b_f32m2 = nk_bf16m1_to_f32m2_rvv_(b_u16m1, vector_length);
685
+ vfloat32m2_t c_f32m2 = nk_bf16m1_to_f32m2_rvv_(c_u16m1, vector_length);
686
+ vfloat32m2_t product_f32m2 = __riscv_vfmul_vv_f32m2(a_f32m2, b_f32m2, vector_length);
687
+ vfloat32m2_t scaled_product_f32m2 = __riscv_vfmul_vf_f32m2(product_f32m2, alpha_val, vector_length);
688
+ vfloat32m2_t result_f32m2 = __riscv_vfmacc_vf_f32m2(scaled_product_f32m2, beta_val, c_f32m2, vector_length);
689
+ vuint16m1_t result_u16m1 = nk_f32m2_to_bf16m1_rvv_(result_f32m2, vector_length);
690
+ __riscv_vse16_v_u16m1((nk_u16_t *)result, result_u16m1, vector_length);
691
+ }
692
+ }
693
+
694
+ NK_PUBLIC void nk_each_fma_i8_rvv(nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const *c, nk_size_t n,
695
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_i8_t *result) {
696
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
697
+ for (nk_size_t vector_length; n > 0;
698
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
699
+ vector_length = __riscv_vsetvl_e8m1(n);
700
+ vint8m1_t a_i8m1 = __riscv_vle8_v_i8m1(a, vector_length);
701
+ vint8m1_t b_i8m1 = __riscv_vle8_v_i8m1(b, vector_length);
702
+ vint8m1_t c_i8m1 = __riscv_vle8_v_i8m1(c, vector_length);
703
+ vint16m2_t a_i16m2 = __riscv_vwadd_vx_i16m2(a_i8m1, 0, vector_length);
704
+ vint32m4_t a_i32m4 = __riscv_vwadd_vx_i32m4(a_i16m2, 0, vector_length);
705
+ vfloat32m4_t a_f32m4 = __riscv_vfcvt_f_x_v_f32m4(a_i32m4, vector_length);
706
+ vint16m2_t b_i16m2 = __riscv_vwadd_vx_i16m2(b_i8m1, 0, vector_length);
707
+ vint32m4_t b_i32m4 = __riscv_vwadd_vx_i32m4(b_i16m2, 0, vector_length);
708
+ vfloat32m4_t b_f32m4 = __riscv_vfcvt_f_x_v_f32m4(b_i32m4, vector_length);
709
+ vint16m2_t c_i16m2 = __riscv_vwadd_vx_i16m2(c_i8m1, 0, vector_length);
710
+ vint32m4_t c_i32m4 = __riscv_vwadd_vx_i32m4(c_i16m2, 0, vector_length);
711
+ vfloat32m4_t c_f32m4 = __riscv_vfcvt_f_x_v_f32m4(c_i32m4, vector_length);
712
+ vfloat32m4_t product_f32m4 = __riscv_vfmul_vv_f32m4(a_f32m4, b_f32m4, vector_length);
713
+ vfloat32m4_t scaled_product_f32m4 = __riscv_vfmul_vf_f32m4(product_f32m4, alpha_val, vector_length);
714
+ vfloat32m4_t result_f32m4 = __riscv_vfmacc_vf_f32m4(scaled_product_f32m4, beta_val, c_f32m4, vector_length);
715
+ vint32m4_t result_i32m4 = __riscv_vfcvt_x_f_v_i32m4(result_f32m4, vector_length);
716
+ result_i32m4 = __riscv_vmax_vx_i32m4(result_i32m4, -128, vector_length);
717
+ result_i32m4 = __riscv_vmin_vx_i32m4(result_i32m4, 127, vector_length);
718
+ vint16m2_t result_i16m2 = __riscv_vncvt_x_x_w_i16m2(result_i32m4, vector_length);
719
+ vint8m1_t result_i8m1 = __riscv_vncvt_x_x_w_i8m1(result_i16m2, vector_length);
720
+ __riscv_vse8_v_i8m1(result, result_i8m1, vector_length);
721
+ }
722
+ }
723
+
724
+ NK_PUBLIC void nk_each_fma_u8_rvv(nk_u8_t const *a, nk_u8_t const *b, nk_u8_t const *c, nk_size_t n,
725
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_u8_t *result) {
726
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
727
+ for (nk_size_t vector_length; n > 0;
728
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
729
+ vector_length = __riscv_vsetvl_e8m1(n);
730
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1(a, vector_length);
731
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1(b, vector_length);
732
+ vuint8m1_t c_u8m1 = __riscv_vle8_v_u8m1(c, vector_length);
733
+ vuint16m2_t a_u16m2 = __riscv_vwaddu_vx_u16m2(a_u8m1, 0, vector_length);
734
+ vuint32m4_t a_u32m4 = __riscv_vwaddu_vx_u32m4(a_u16m2, 0, vector_length);
735
+ vfloat32m4_t a_f32m4 = __riscv_vfcvt_f_xu_v_f32m4(a_u32m4, vector_length);
736
+ vuint16m2_t b_u16m2 = __riscv_vwaddu_vx_u16m2(b_u8m1, 0, vector_length);
737
+ vuint32m4_t b_u32m4 = __riscv_vwaddu_vx_u32m4(b_u16m2, 0, vector_length);
738
+ vfloat32m4_t b_f32m4 = __riscv_vfcvt_f_xu_v_f32m4(b_u32m4, vector_length);
739
+ vuint16m2_t c_u16m2 = __riscv_vwaddu_vx_u16m2(c_u8m1, 0, vector_length);
740
+ vuint32m4_t c_u32m4 = __riscv_vwaddu_vx_u32m4(c_u16m2, 0, vector_length);
741
+ vfloat32m4_t c_f32m4 = __riscv_vfcvt_f_xu_v_f32m4(c_u32m4, vector_length);
742
+ vfloat32m4_t product_f32m4 = __riscv_vfmul_vv_f32m4(a_f32m4, b_f32m4, vector_length);
743
+ vfloat32m4_t scaled_product_f32m4 = __riscv_vfmul_vf_f32m4(product_f32m4, alpha_val, vector_length);
744
+ vfloat32m4_t result_f32m4 = __riscv_vfmacc_vf_f32m4(scaled_product_f32m4, beta_val, c_f32m4, vector_length);
745
+ vuint32m4_t result_u32m4 = __riscv_vfcvt_xu_f_v_u32m4(result_f32m4, vector_length);
746
+ result_u32m4 = __riscv_vmaxu_vx_u32m4(result_u32m4, 0, vector_length);
747
+ result_u32m4 = __riscv_vminu_vx_u32m4(result_u32m4, 255, vector_length);
748
+ vuint16m2_t result_u16m2 = __riscv_vncvt_x_x_w_u16m2(result_u32m4, vector_length);
749
+ vuint8m1_t result_u8m1 = __riscv_vncvt_x_x_w_u8m1(result_u16m2, vector_length);
750
+ __riscv_vse8_v_u8m1(result, result_u8m1, vector_length);
751
+ }
752
+ }
753
+
754
+ NK_PUBLIC void nk_each_fma_i16_rvv(nk_i16_t const *a, nk_i16_t const *b, nk_i16_t const *c, nk_size_t n,
755
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_i16_t *result) {
756
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
757
+ for (nk_size_t vector_length; n > 0;
758
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
759
+ vector_length = __riscv_vsetvl_e16m1(n);
760
+ vint16m1_t a_i16m1 = __riscv_vle16_v_i16m1(a, vector_length);
761
+ vint16m1_t b_i16m1 = __riscv_vle16_v_i16m1(b, vector_length);
762
+ vint16m1_t c_i16m1 = __riscv_vle16_v_i16m1(c, vector_length);
763
+ vint32m2_t a_i32m2 = __riscv_vwadd_vx_i32m2(a_i16m1, 0, vector_length);
764
+ vfloat32m2_t a_f32m2 = __riscv_vfcvt_f_x_v_f32m2(a_i32m2, vector_length);
765
+ vint32m2_t b_i32m2 = __riscv_vwadd_vx_i32m2(b_i16m1, 0, vector_length);
766
+ vfloat32m2_t b_f32m2 = __riscv_vfcvt_f_x_v_f32m2(b_i32m2, vector_length);
767
+ vint32m2_t c_i32m2 = __riscv_vwadd_vx_i32m2(c_i16m1, 0, vector_length);
768
+ vfloat32m2_t c_f32m2 = __riscv_vfcvt_f_x_v_f32m2(c_i32m2, vector_length);
769
+ vfloat32m2_t product_f32m2 = __riscv_vfmul_vv_f32m2(a_f32m2, b_f32m2, vector_length);
770
+ vfloat32m2_t scaled_product_f32m2 = __riscv_vfmul_vf_f32m2(product_f32m2, alpha_val, vector_length);
771
+ vfloat32m2_t result_f32m2 = __riscv_vfmacc_vf_f32m2(scaled_product_f32m2, beta_val, c_f32m2, vector_length);
772
+ vint32m2_t result_i32m2 = __riscv_vfcvt_x_f_v_i32m2(result_f32m2, vector_length);
773
+ result_i32m2 = __riscv_vmax_vx_i32m2(result_i32m2, -32768, vector_length);
774
+ result_i32m2 = __riscv_vmin_vx_i32m2(result_i32m2, 32767, vector_length);
775
+ vint16m1_t result_i16m1 = __riscv_vncvt_x_x_w_i16m1(result_i32m2, vector_length);
776
+ __riscv_vse16_v_i16m1(result, result_i16m1, vector_length);
777
+ }
778
+ }
779
+
780
+ NK_PUBLIC void nk_each_fma_u16_rvv(nk_u16_t const *a, nk_u16_t const *b, nk_u16_t const *c, nk_size_t n,
781
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_u16_t *result) {
782
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
783
+ for (nk_size_t vector_length; n > 0;
784
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
785
+ vector_length = __riscv_vsetvl_e16m1(n);
786
+ vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1(a, vector_length);
787
+ vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1(b, vector_length);
788
+ vuint16m1_t c_u16m1 = __riscv_vle16_v_u16m1(c, vector_length);
789
+ vuint32m2_t a_u32m2 = __riscv_vwaddu_vx_u32m2(a_u16m1, 0, vector_length);
790
+ vfloat32m2_t a_f32m2 = __riscv_vfcvt_f_xu_v_f32m2(a_u32m2, vector_length);
791
+ vuint32m2_t b_u32m2 = __riscv_vwaddu_vx_u32m2(b_u16m1, 0, vector_length);
792
+ vfloat32m2_t b_f32m2 = __riscv_vfcvt_f_xu_v_f32m2(b_u32m2, vector_length);
793
+ vuint32m2_t c_u32m2 = __riscv_vwaddu_vx_u32m2(c_u16m1, 0, vector_length);
794
+ vfloat32m2_t c_f32m2 = __riscv_vfcvt_f_xu_v_f32m2(c_u32m2, vector_length);
795
+ vfloat32m2_t product_f32m2 = __riscv_vfmul_vv_f32m2(a_f32m2, b_f32m2, vector_length);
796
+ vfloat32m2_t scaled_product_f32m2 = __riscv_vfmul_vf_f32m2(product_f32m2, alpha_val, vector_length);
797
+ vfloat32m2_t result_f32m2 = __riscv_vfmacc_vf_f32m2(scaled_product_f32m2, beta_val, c_f32m2, vector_length);
798
+ vuint32m2_t result_u32m2 = __riscv_vfcvt_xu_f_v_u32m2(result_f32m2, vector_length);
799
+ result_u32m2 = __riscv_vmaxu_vx_u32m2(result_u32m2, 0, vector_length);
800
+ result_u32m2 = __riscv_vminu_vx_u32m2(result_u32m2, 65535, vector_length);
801
+ vuint16m1_t result_u16m1 = __riscv_vncvt_x_x_w_u16m1(result_u32m2, vector_length);
802
+ __riscv_vse16_v_u16m1(result, result_u16m1, vector_length);
803
+ }
804
+ }
805
+
806
+ NK_PUBLIC void nk_each_fma_i32_rvv(nk_i32_t const *a, nk_i32_t const *b, nk_i32_t const *c, nk_size_t n,
807
+ nk_f64_t const *alpha, nk_f64_t const *beta, nk_i32_t *result) {
808
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
809
+ for (nk_size_t vector_length; n > 0;
810
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
811
+ vector_length = __riscv_vsetvl_e32m1(n);
812
+ vint32m1_t a_i32m1 = __riscv_vle32_v_i32m1(a, vector_length);
813
+ vint32m1_t b_i32m1 = __riscv_vle32_v_i32m1(b, vector_length);
814
+ vint32m1_t c_i32m1 = __riscv_vle32_v_i32m1(c, vector_length);
815
+ vfloat64m2_t a_f64m2 = __riscv_vfwcvt_f_x_v_f64m2(a_i32m1, vector_length);
816
+ vfloat64m2_t b_f64m2 = __riscv_vfwcvt_f_x_v_f64m2(b_i32m1, vector_length);
817
+ vfloat64m2_t c_f64m2 = __riscv_vfwcvt_f_x_v_f64m2(c_i32m1, vector_length);
818
+ vfloat64m2_t product_f64m2 = __riscv_vfmul_vv_f64m2(a_f64m2, b_f64m2, vector_length);
819
+ vfloat64m2_t scaled_product_f64m2 = __riscv_vfmul_vf_f64m2(product_f64m2, alpha_val, vector_length);
820
+ vfloat64m2_t result_f64m2 = __riscv_vfmacc_vf_f64m2(scaled_product_f64m2, beta_val, c_f64m2, vector_length);
821
+ result_f64m2 = __riscv_vfmax_vf_f64m2(result_f64m2, -2147483648.0, vector_length);
822
+ result_f64m2 = __riscv_vfmin_vf_f64m2(result_f64m2, 2147483647.0, vector_length);
823
+ vint32m1_t result_i32m1 = __riscv_vfncvt_x_f_w_i32m1(result_f64m2, vector_length);
824
+ __riscv_vse32_v_i32m1(result, result_i32m1, vector_length);
825
+ }
826
+ }
827
+
828
+ NK_PUBLIC void nk_each_fma_u32_rvv(nk_u32_t const *a, nk_u32_t const *b, nk_u32_t const *c, nk_size_t n,
829
+ nk_f64_t const *alpha, nk_f64_t const *beta, nk_u32_t *result) {
830
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
831
+ for (nk_size_t vector_length; n > 0;
832
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
833
+ vector_length = __riscv_vsetvl_e32m1(n);
834
+ vuint32m1_t a_u32m1 = __riscv_vle32_v_u32m1(a, vector_length);
835
+ vuint32m1_t b_u32m1 = __riscv_vle32_v_u32m1(b, vector_length);
836
+ vuint32m1_t c_u32m1 = __riscv_vle32_v_u32m1(c, vector_length);
837
+ vfloat64m2_t a_f64m2 = __riscv_vfwcvt_f_xu_v_f64m2(a_u32m1, vector_length);
838
+ vfloat64m2_t b_f64m2 = __riscv_vfwcvt_f_xu_v_f64m2(b_u32m1, vector_length);
839
+ vfloat64m2_t c_f64m2 = __riscv_vfwcvt_f_xu_v_f64m2(c_u32m1, vector_length);
840
+ vfloat64m2_t product_f64m2 = __riscv_vfmul_vv_f64m2(a_f64m2, b_f64m2, vector_length);
841
+ vfloat64m2_t scaled_product_f64m2 = __riscv_vfmul_vf_f64m2(product_f64m2, alpha_val, vector_length);
842
+ vfloat64m2_t result_f64m2 = __riscv_vfmacc_vf_f64m2(scaled_product_f64m2, beta_val, c_f64m2, vector_length);
843
+ result_f64m2 = __riscv_vfmax_vf_f64m2(result_f64m2, 0.0, vector_length);
844
+ result_f64m2 = __riscv_vfmin_vf_f64m2(result_f64m2, 4294967295.0, vector_length);
845
+ vuint32m1_t result_u32m1 = __riscv_vfncvt_xu_f_w_u32m1(result_f64m2, vector_length);
846
+ __riscv_vse32_v_u32m1(result, result_u32m1, vector_length);
847
+ }
848
+ }
849
+
850
+ NK_PUBLIC void nk_each_fma_i64_rvv(nk_i64_t const *a, nk_i64_t const *b, nk_i64_t const *c, nk_size_t n,
851
+ nk_f64_t const *alpha, nk_f64_t const *beta, nk_i64_t *result) {
852
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
853
+ for (nk_size_t vector_length; n > 0;
854
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
855
+ vector_length = __riscv_vsetvl_e64m4(n);
856
+ vint64m4_t a_i64m4 = __riscv_vle64_v_i64m4(a, vector_length);
857
+ vint64m4_t b_i64m4 = __riscv_vle64_v_i64m4(b, vector_length);
858
+ vint64m4_t c_i64m4 = __riscv_vle64_v_i64m4(c, vector_length);
859
+ vfloat64m4_t a_f64m4 = __riscv_vfcvt_f_x_v_f64m4(a_i64m4, vector_length);
860
+ vfloat64m4_t b_f64m4 = __riscv_vfcvt_f_x_v_f64m4(b_i64m4, vector_length);
861
+ vfloat64m4_t c_f64m4 = __riscv_vfcvt_f_x_v_f64m4(c_i64m4, vector_length);
862
+ vfloat64m4_t product_f64m4 = __riscv_vfmul_vv_f64m4(a_f64m4, b_f64m4, vector_length);
863
+ vfloat64m4_t scaled_product_f64m4 = __riscv_vfmul_vf_f64m4(product_f64m4, alpha_val, vector_length);
864
+ vfloat64m4_t result_f64m4 = __riscv_vfmacc_vf_f64m4(scaled_product_f64m4, beta_val, c_f64m4, vector_length);
865
+ vint64m4_t result_i64m4 = __riscv_vfcvt_x_f_v_i64m4(result_f64m4, vector_length);
866
+ __riscv_vse64_v_i64m4(result, result_i64m4, vector_length);
867
+ }
868
+ }
869
+
870
+ NK_PUBLIC void nk_each_fma_u64_rvv(nk_u64_t const *a, nk_u64_t const *b, nk_u64_t const *c, nk_size_t n,
871
+ nk_f64_t const *alpha, nk_f64_t const *beta, nk_u64_t *result) {
872
+ nk_f64_t alpha_val = *alpha, beta_val = *beta;
873
+ for (nk_size_t vector_length; n > 0;
874
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
875
+ vector_length = __riscv_vsetvl_e64m4(n);
876
+ vuint64m4_t a_u64m4 = __riscv_vle64_v_u64m4(a, vector_length);
877
+ vuint64m4_t b_u64m4 = __riscv_vle64_v_u64m4(b, vector_length);
878
+ vuint64m4_t c_u64m4 = __riscv_vle64_v_u64m4(c, vector_length);
879
+ vfloat64m4_t a_f64m4 = __riscv_vfcvt_f_xu_v_f64m4(a_u64m4, vector_length);
880
+ vfloat64m4_t b_f64m4 = __riscv_vfcvt_f_xu_v_f64m4(b_u64m4, vector_length);
881
+ vfloat64m4_t c_f64m4 = __riscv_vfcvt_f_xu_v_f64m4(c_u64m4, vector_length);
882
+ vfloat64m4_t product_f64m4 = __riscv_vfmul_vv_f64m4(a_f64m4, b_f64m4, vector_length);
883
+ vfloat64m4_t scaled_product_f64m4 = __riscv_vfmul_vf_f64m4(product_f64m4, alpha_val, vector_length);
884
+ vfloat64m4_t result_f64m4 = __riscv_vfmacc_vf_f64m4(scaled_product_f64m4, beta_val, c_f64m4, vector_length);
885
+ result_f64m4 = __riscv_vfmax_vf_f64m4(result_f64m4, 0.0, vector_length);
886
+ vuint64m4_t result_u64m4 = __riscv_vfcvt_xu_f_v_u64m4(result_f64m4, vector_length);
887
+ __riscv_vse64_v_u64m4(result, result_u64m4, vector_length);
888
+ }
889
+ }
890
+
891
+ NK_PUBLIC void nk_each_fma_e4m3_rvv(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_e4m3_t const *c, nk_size_t n,
892
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_e4m3_t *result) {
893
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
894
+ for (nk_size_t vector_length; n > 0;
895
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
896
+ vector_length = __riscv_vsetvl_e8m1(n);
897
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
898
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b, vector_length);
899
+ vuint8m1_t c_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)c, vector_length);
900
+ vfloat32m4_t a_f32m4 = nk_e4m3m1_to_f32m4_rvv_(a_u8m1, vector_length);
901
+ vfloat32m4_t b_f32m4 = nk_e4m3m1_to_f32m4_rvv_(b_u8m1, vector_length);
902
+ vfloat32m4_t c_f32m4 = nk_e4m3m1_to_f32m4_rvv_(c_u8m1, vector_length);
903
+ vfloat32m4_t product_f32m4 = __riscv_vfmul_vv_f32m4(a_f32m4, b_f32m4, vector_length);
904
+ vfloat32m4_t scaled_product_f32m4 = __riscv_vfmul_vf_f32m4(product_f32m4, alpha_val, vector_length);
905
+ vfloat32m4_t result_f32m4 = __riscv_vfmacc_vf_f32m4(scaled_product_f32m4, beta_val, c_f32m4, vector_length);
906
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e4m3m1_rvv_(result_f32m4, vector_length);
907
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
908
+ }
909
+ }
910
+
911
+ NK_PUBLIC void nk_each_fma_e5m2_rvv(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_e5m2_t const *c, nk_size_t n,
912
+ nk_f32_t const *alpha, nk_f32_t const *beta, nk_e5m2_t *result) {
913
+ nk_f32_t alpha_val = *alpha, beta_val = *beta;
914
+ for (nk_size_t vector_length; n > 0;
915
+ n -= vector_length, a += vector_length, b += vector_length, c += vector_length, result += vector_length) {
916
+ vector_length = __riscv_vsetvl_e8m1(n);
917
+ vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
918
+ vuint8m1_t b_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)b, vector_length);
919
+ vuint8m1_t c_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)c, vector_length);
920
+ vfloat32m4_t a_f32m4 = nk_e5m2m1_to_f32m4_rvv_(a_u8m1, vector_length);
921
+ vfloat32m4_t b_f32m4 = nk_e5m2m1_to_f32m4_rvv_(b_u8m1, vector_length);
922
+ vfloat32m4_t c_f32m4 = nk_e5m2m1_to_f32m4_rvv_(c_u8m1, vector_length);
923
+ vfloat32m4_t product_f32m4 = __riscv_vfmul_vv_f32m4(a_f32m4, b_f32m4, vector_length);
924
+ vfloat32m4_t scaled_product_f32m4 = __riscv_vfmul_vf_f32m4(product_f32m4, alpha_val, vector_length);
925
+ vfloat32m4_t result_f32m4 = __riscv_vfmacc_vf_f32m4(scaled_product_f32m4, beta_val, c_f32m4, vector_length);
926
+ vuint8m1_t result_u8m1 = nk_f32m4_to_e5m2m1_rvv_(result_f32m4, vector_length);
927
+ __riscv_vse8_v_u8m1((nk_u8_t *)result, result_u8m1, vector_length);
928
+ }
929
+ }
930
+
931
+ NK_PUBLIC void nk_each_scale_f32c_rvv(nk_f32c_t const *a, nk_size_t n, nk_f32c_t const *alpha, nk_f32c_t const *beta,
932
+ nk_f32c_t *result) {
933
+ nk_f32_t alpha_real = alpha->real, alpha_imag = alpha->imag;
934
+ nk_f32_t beta_real = beta->real, beta_imag = beta->imag;
935
+ nk_f32_t const *a_f32 = (nk_f32_t const *)a;
936
+ nk_f32_t *result_f32 = (nk_f32_t *)result;
937
+ for (nk_size_t vector_length; n > 0;
938
+ n -= vector_length, a_f32 += vector_length * 2, result_f32 += vector_length * 2) {
939
+ vector_length = __riscv_vsetvl_e32m2(n);
940
+ vfloat32m2x2_t a_f32m2x2 = __riscv_vlseg2e32_v_f32m2x2(a_f32, vector_length);
941
+ vfloat32m2_t a_real_f32m2 = __riscv_vget_v_f32m2x2_f32m2(a_f32m2x2, 0);
942
+ vfloat32m2_t a_imag_f32m2 = __riscv_vget_v_f32m2x2_f32m2(a_f32m2x2, 1);
943
+ vfloat32m2_t y_real_f32m2 = __riscv_vfmv_v_f_f32m2(beta_real, vector_length);
944
+ y_real_f32m2 = __riscv_vfmacc_vf_f32m2(y_real_f32m2, alpha_real, a_real_f32m2, vector_length);
945
+ y_real_f32m2 = __riscv_vfnmsac_vf_f32m2(y_real_f32m2, alpha_imag, a_imag_f32m2, vector_length);
946
+ vfloat32m2_t y_imag_f32m2 = __riscv_vfmv_v_f_f32m2(beta_imag, vector_length);
947
+ y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(y_imag_f32m2, alpha_real, a_imag_f32m2, vector_length);
948
+ y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(y_imag_f32m2, alpha_imag, a_real_f32m2, vector_length);
949
+ vfloat32m2x2_t out_f32m2x2 = __riscv_vcreate_v_f32m2x2(y_real_f32m2, y_imag_f32m2);
950
+ __riscv_vsseg2e32_v_f32m2x2(result_f32, out_f32m2x2, vector_length);
951
+ }
952
+ }
953
+
954
+ NK_PUBLIC void nk_each_scale_f64c_rvv(nk_f64c_t const *a, nk_size_t n, nk_f64c_t const *alpha, nk_f64c_t const *beta,
955
+ nk_f64c_t *result) {
956
+ nk_f64_t alpha_real = alpha->real, alpha_imag = alpha->imag;
957
+ nk_f64_t beta_real = beta->real, beta_imag = beta->imag;
958
+ nk_f64_t const *a_f64 = (nk_f64_t const *)a;
959
+ nk_f64_t *result_f64 = (nk_f64_t *)result;
960
+ for (nk_size_t vector_length; n > 0;
961
+ n -= vector_length, a_f64 += vector_length * 2, result_f64 += vector_length * 2) {
962
+ vector_length = __riscv_vsetvl_e64m2(n);
963
+ vfloat64m2x2_t a_f64m2x2 = __riscv_vlseg2e64_v_f64m2x2(a_f64, vector_length);
964
+ vfloat64m2_t a_real_f64m2 = __riscv_vget_v_f64m2x2_f64m2(a_f64m2x2, 0);
965
+ vfloat64m2_t a_imag_f64m2 = __riscv_vget_v_f64m2x2_f64m2(a_f64m2x2, 1);
966
+ vfloat64m2_t y_real_f64m2 = __riscv_vfmv_v_f_f64m2(beta_real, vector_length);
967
+ y_real_f64m2 = __riscv_vfmacc_vf_f64m2(y_real_f64m2, alpha_real, a_real_f64m2, vector_length);
968
+ y_real_f64m2 = __riscv_vfnmsac_vf_f64m2(y_real_f64m2, alpha_imag, a_imag_f64m2, vector_length);
969
+ vfloat64m2_t y_imag_f64m2 = __riscv_vfmv_v_f_f64m2(beta_imag, vector_length);
970
+ y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(y_imag_f64m2, alpha_real, a_imag_f64m2, vector_length);
971
+ y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(y_imag_f64m2, alpha_imag, a_real_f64m2, vector_length);
972
+ vfloat64m2x2_t out_f64m2x2 = __riscv_vcreate_v_f64m2x2(y_real_f64m2, y_imag_f64m2);
973
+ __riscv_vsseg2e64_v_f64m2x2(result_f64, out_f64m2x2, vector_length);
974
+ }
975
+ }
976
+
977
+ NK_PUBLIC void nk_each_blend_f32c_rvv(nk_f32c_t const *a, nk_f32c_t const *b, nk_size_t n, nk_f32c_t const *alpha,
978
+ nk_f32c_t const *beta, nk_f32c_t *result) {
979
+ nk_f32_t alpha_real = alpha->real, alpha_imag = alpha->imag;
980
+ nk_f32_t beta_real = beta->real, beta_imag = beta->imag;
981
+ nk_f32_t const *a_f32 = (nk_f32_t const *)a;
982
+ nk_f32_t const *b_f32 = (nk_f32_t const *)b;
983
+ nk_f32_t *result_f32 = (nk_f32_t *)result;
984
+ for (nk_size_t vector_length; n > 0;
985
+ n -= vector_length, a_f32 += vector_length * 2, b_f32 += vector_length * 2, result_f32 += vector_length * 2) {
986
+ vector_length = __riscv_vsetvl_e32m2(n);
987
+ vfloat32m2x2_t a_f32m2x2 = __riscv_vlseg2e32_v_f32m2x2(a_f32, vector_length);
988
+ vfloat32m2x2_t b_f32m2x2 = __riscv_vlseg2e32_v_f32m2x2(b_f32, vector_length);
989
+ vfloat32m2_t a_real_f32m2 = __riscv_vget_v_f32m2x2_f32m2(a_f32m2x2, 0);
990
+ vfloat32m2_t a_imag_f32m2 = __riscv_vget_v_f32m2x2_f32m2(a_f32m2x2, 1);
991
+ vfloat32m2_t b_real_f32m2 = __riscv_vget_v_f32m2x2_f32m2(b_f32m2x2, 0);
992
+ vfloat32m2_t b_imag_f32m2 = __riscv_vget_v_f32m2x2_f32m2(b_f32m2x2, 1);
993
+ vfloat32m2_t ya_real_f32m2 = __riscv_vfmul_vf_f32m2(a_real_f32m2, alpha_real, vector_length);
994
+ ya_real_f32m2 = __riscv_vfnmsac_vf_f32m2(ya_real_f32m2, alpha_imag, a_imag_f32m2, vector_length);
995
+ vfloat32m2_t ya_imag_f32m2 = __riscv_vfmul_vf_f32m2(a_imag_f32m2, alpha_real, vector_length);
996
+ ya_imag_f32m2 = __riscv_vfmacc_vf_f32m2(ya_imag_f32m2, alpha_imag, a_real_f32m2, vector_length);
997
+ vfloat32m2_t y_real_f32m2 = __riscv_vfmacc_vf_f32m2(ya_real_f32m2, beta_real, b_real_f32m2, vector_length);
998
+ y_real_f32m2 = __riscv_vfnmsac_vf_f32m2(y_real_f32m2, beta_imag, b_imag_f32m2, vector_length);
999
+ vfloat32m2_t y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(ya_imag_f32m2, beta_real, b_imag_f32m2, vector_length);
1000
+ y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(y_imag_f32m2, beta_imag, b_real_f32m2, vector_length);
1001
+ vfloat32m2x2_t out_f32m2x2 = __riscv_vcreate_v_f32m2x2(y_real_f32m2, y_imag_f32m2);
1002
+ __riscv_vsseg2e32_v_f32m2x2(result_f32, out_f32m2x2, vector_length);
1003
+ }
1004
+ }
1005
+
1006
+ NK_PUBLIC void nk_each_blend_f64c_rvv(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n, nk_f64c_t const *alpha,
1007
+ nk_f64c_t const *beta, nk_f64c_t *result) {
1008
+ nk_f64_t alpha_real = alpha->real, alpha_imag = alpha->imag;
1009
+ nk_f64_t beta_real = beta->real, beta_imag = beta->imag;
1010
+ nk_f64_t const *a_f64 = (nk_f64_t const *)a;
1011
+ nk_f64_t const *b_f64 = (nk_f64_t const *)b;
1012
+ nk_f64_t *result_f64 = (nk_f64_t *)result;
1013
+ for (nk_size_t vector_length; n > 0;
1014
+ n -= vector_length, a_f64 += vector_length * 2, b_f64 += vector_length * 2, result_f64 += vector_length * 2) {
1015
+ vector_length = __riscv_vsetvl_e64m2(n);
1016
+ vfloat64m2x2_t a_f64m2x2 = __riscv_vlseg2e64_v_f64m2x2(a_f64, vector_length);
1017
+ vfloat64m2x2_t b_f64m2x2 = __riscv_vlseg2e64_v_f64m2x2(b_f64, vector_length);
1018
+ vfloat64m2_t a_real_f64m2 = __riscv_vget_v_f64m2x2_f64m2(a_f64m2x2, 0);
1019
+ vfloat64m2_t a_imag_f64m2 = __riscv_vget_v_f64m2x2_f64m2(a_f64m2x2, 1);
1020
+ vfloat64m2_t b_real_f64m2 = __riscv_vget_v_f64m2x2_f64m2(b_f64m2x2, 0);
1021
+ vfloat64m2_t b_imag_f64m2 = __riscv_vget_v_f64m2x2_f64m2(b_f64m2x2, 1);
1022
+ vfloat64m2_t ya_real_f64m2 = __riscv_vfmul_vf_f64m2(a_real_f64m2, alpha_real, vector_length);
1023
+ ya_real_f64m2 = __riscv_vfnmsac_vf_f64m2(ya_real_f64m2, alpha_imag, a_imag_f64m2, vector_length);
1024
+ vfloat64m2_t ya_imag_f64m2 = __riscv_vfmul_vf_f64m2(a_imag_f64m2, alpha_real, vector_length);
1025
+ ya_imag_f64m2 = __riscv_vfmacc_vf_f64m2(ya_imag_f64m2, alpha_imag, a_real_f64m2, vector_length);
1026
+ vfloat64m2_t y_real_f64m2 = __riscv_vfmacc_vf_f64m2(ya_real_f64m2, beta_real, b_real_f64m2, vector_length);
1027
+ y_real_f64m2 = __riscv_vfnmsac_vf_f64m2(y_real_f64m2, beta_imag, b_imag_f64m2, vector_length);
1028
+ vfloat64m2_t y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(ya_imag_f64m2, beta_real, b_imag_f64m2, vector_length);
1029
+ y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(y_imag_f64m2, beta_imag, b_real_f64m2, vector_length);
1030
+ vfloat64m2x2_t out_f64m2x2 = __riscv_vcreate_v_f64m2x2(y_real_f64m2, y_imag_f64m2);
1031
+ __riscv_vsseg2e64_v_f64m2x2(result_f64, out_f64m2x2, vector_length);
1032
+ }
1033
+ }
1034
+
1035
+ NK_PUBLIC void nk_each_fma_f32c_rvv(nk_f32c_t const *a, nk_f32c_t const *b, nk_f32c_t const *c, nk_size_t n,
1036
+ nk_f32c_t const *alpha, nk_f32c_t const *beta, nk_f32c_t *result) {
1037
+ nk_f32_t alpha_real = alpha->real, alpha_imag = alpha->imag;
1038
+ nk_f32_t beta_real = beta->real, beta_imag = beta->imag;
1039
+ nk_f32_t const *a_f32 = (nk_f32_t const *)a;
1040
+ nk_f32_t const *b_f32 = (nk_f32_t const *)b;
1041
+ nk_f32_t const *c_f32 = (nk_f32_t const *)c;
1042
+ nk_f32_t *result_f32 = (nk_f32_t *)result;
1043
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a_f32 += vector_length * 2, b_f32 += vector_length * 2,
1044
+ c_f32 += vector_length * 2, result_f32 += vector_length * 2) {
1045
+ vector_length = __riscv_vsetvl_e32m2(n);
1046
+ vfloat32m2x2_t a_f32m2x2 = __riscv_vlseg2e32_v_f32m2x2(a_f32, vector_length);
1047
+ vfloat32m2x2_t b_f32m2x2 = __riscv_vlseg2e32_v_f32m2x2(b_f32, vector_length);
1048
+ vfloat32m2x2_t c_f32m2x2 = __riscv_vlseg2e32_v_f32m2x2(c_f32, vector_length);
1049
+ vfloat32m2_t a_real_f32m2 = __riscv_vget_v_f32m2x2_f32m2(a_f32m2x2, 0);
1050
+ vfloat32m2_t a_imag_f32m2 = __riscv_vget_v_f32m2x2_f32m2(a_f32m2x2, 1);
1051
+ vfloat32m2_t b_real_f32m2 = __riscv_vget_v_f32m2x2_f32m2(b_f32m2x2, 0);
1052
+ vfloat32m2_t b_imag_f32m2 = __riscv_vget_v_f32m2x2_f32m2(b_f32m2x2, 1);
1053
+ vfloat32m2_t c_real_f32m2 = __riscv_vget_v_f32m2x2_f32m2(c_f32m2x2, 0);
1054
+ vfloat32m2_t c_imag_f32m2 = __riscv_vget_v_f32m2x2_f32m2(c_f32m2x2, 1);
1055
+ vfloat32m2_t ab_real_f32m2 = __riscv_vfmul_vv_f32m2(a_real_f32m2, b_real_f32m2, vector_length);
1056
+ ab_real_f32m2 = __riscv_vfnmsac_vv_f32m2(ab_real_f32m2, a_imag_f32m2, b_imag_f32m2, vector_length);
1057
+ vfloat32m2_t ab_imag_f32m2 = __riscv_vfmul_vv_f32m2(a_real_f32m2, b_imag_f32m2, vector_length);
1058
+ ab_imag_f32m2 = __riscv_vfmacc_vv_f32m2(ab_imag_f32m2, a_imag_f32m2, b_real_f32m2, vector_length);
1059
+ vfloat32m2_t y_real_f32m2 = __riscv_vfmul_vf_f32m2(ab_real_f32m2, alpha_real, vector_length);
1060
+ y_real_f32m2 = __riscv_vfnmsac_vf_f32m2(y_real_f32m2, alpha_imag, ab_imag_f32m2, vector_length);
1061
+ vfloat32m2_t y_imag_f32m2 = __riscv_vfmul_vf_f32m2(ab_imag_f32m2, alpha_real, vector_length);
1062
+ y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(y_imag_f32m2, alpha_imag, ab_real_f32m2, vector_length);
1063
+ y_real_f32m2 = __riscv_vfmacc_vf_f32m2(y_real_f32m2, beta_real, c_real_f32m2, vector_length);
1064
+ y_real_f32m2 = __riscv_vfnmsac_vf_f32m2(y_real_f32m2, beta_imag, c_imag_f32m2, vector_length);
1065
+ y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(y_imag_f32m2, beta_real, c_imag_f32m2, vector_length);
1066
+ y_imag_f32m2 = __riscv_vfmacc_vf_f32m2(y_imag_f32m2, beta_imag, c_real_f32m2, vector_length);
1067
+ vfloat32m2x2_t out_f32m2x2 = __riscv_vcreate_v_f32m2x2(y_real_f32m2, y_imag_f32m2);
1068
+ __riscv_vsseg2e32_v_f32m2x2(result_f32, out_f32m2x2, vector_length);
1069
+ }
1070
+ }
1071
+
1072
+ NK_PUBLIC void nk_each_fma_f64c_rvv(nk_f64c_t const *a, nk_f64c_t const *b, nk_f64c_t const *c, nk_size_t n,
1073
+ nk_f64c_t const *alpha, nk_f64c_t const *beta, nk_f64c_t *result) {
1074
+ nk_f64_t alpha_real = alpha->real, alpha_imag = alpha->imag;
1075
+ nk_f64_t beta_real = beta->real, beta_imag = beta->imag;
1076
+ nk_f64_t const *a_f64 = (nk_f64_t const *)a;
1077
+ nk_f64_t const *b_f64 = (nk_f64_t const *)b;
1078
+ nk_f64_t const *c_f64 = (nk_f64_t const *)c;
1079
+ nk_f64_t *result_f64 = (nk_f64_t *)result;
1080
+ for (nk_size_t vector_length; n > 0; n -= vector_length, a_f64 += vector_length * 2, b_f64 += vector_length * 2,
1081
+ c_f64 += vector_length * 2, result_f64 += vector_length * 2) {
1082
+ vector_length = __riscv_vsetvl_e64m2(n);
1083
+ vfloat64m2x2_t a_f64m2x2 = __riscv_vlseg2e64_v_f64m2x2(a_f64, vector_length);
1084
+ vfloat64m2x2_t b_f64m2x2 = __riscv_vlseg2e64_v_f64m2x2(b_f64, vector_length);
1085
+ vfloat64m2x2_t c_f64m2x2 = __riscv_vlseg2e64_v_f64m2x2(c_f64, vector_length);
1086
+ vfloat64m2_t a_real_f64m2 = __riscv_vget_v_f64m2x2_f64m2(a_f64m2x2, 0);
1087
+ vfloat64m2_t a_imag_f64m2 = __riscv_vget_v_f64m2x2_f64m2(a_f64m2x2, 1);
1088
+ vfloat64m2_t b_real_f64m2 = __riscv_vget_v_f64m2x2_f64m2(b_f64m2x2, 0);
1089
+ vfloat64m2_t b_imag_f64m2 = __riscv_vget_v_f64m2x2_f64m2(b_f64m2x2, 1);
1090
+ vfloat64m2_t c_real_f64m2 = __riscv_vget_v_f64m2x2_f64m2(c_f64m2x2, 0);
1091
+ vfloat64m2_t c_imag_f64m2 = __riscv_vget_v_f64m2x2_f64m2(c_f64m2x2, 1);
1092
+ vfloat64m2_t ab_real_f64m2 = __riscv_vfmul_vv_f64m2(a_real_f64m2, b_real_f64m2, vector_length);
1093
+ ab_real_f64m2 = __riscv_vfnmsac_vv_f64m2(ab_real_f64m2, a_imag_f64m2, b_imag_f64m2, vector_length);
1094
+ vfloat64m2_t ab_imag_f64m2 = __riscv_vfmul_vv_f64m2(a_real_f64m2, b_imag_f64m2, vector_length);
1095
+ ab_imag_f64m2 = __riscv_vfmacc_vv_f64m2(ab_imag_f64m2, a_imag_f64m2, b_real_f64m2, vector_length);
1096
+ vfloat64m2_t y_real_f64m2 = __riscv_vfmul_vf_f64m2(ab_real_f64m2, alpha_real, vector_length);
1097
+ y_real_f64m2 = __riscv_vfnmsac_vf_f64m2(y_real_f64m2, alpha_imag, ab_imag_f64m2, vector_length);
1098
+ vfloat64m2_t y_imag_f64m2 = __riscv_vfmul_vf_f64m2(ab_imag_f64m2, alpha_real, vector_length);
1099
+ y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(y_imag_f64m2, alpha_imag, ab_real_f64m2, vector_length);
1100
+ y_real_f64m2 = __riscv_vfmacc_vf_f64m2(y_real_f64m2, beta_real, c_real_f64m2, vector_length);
1101
+ y_real_f64m2 = __riscv_vfnmsac_vf_f64m2(y_real_f64m2, beta_imag, c_imag_f64m2, vector_length);
1102
+ y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(y_imag_f64m2, beta_real, c_imag_f64m2, vector_length);
1103
+ y_imag_f64m2 = __riscv_vfmacc_vf_f64m2(y_imag_f64m2, beta_imag, c_real_f64m2, vector_length);
1104
+ vfloat64m2x2_t out_f64m2x2 = __riscv_vcreate_v_f64m2x2(y_real_f64m2, y_imag_f64m2);
1105
+ __riscv_vsseg2e64_v_f64m2x2(result_f64, out_f64m2x2, vector_length);
1106
+ }
1107
+ }
1108
+
1109
+ #if defined(__cplusplus)
1110
+ } // extern "C"
1111
+ #endif
1112
+
1113
+ #if defined(__clang__)
1114
+ #pragma clang attribute pop
1115
+ #elif defined(__GNUC__)
1116
+ #pragma GCC pop_options
1117
+ #endif
1118
+
1119
+ #endif // NK_TARGET_RVV
1120
+ #endif // NK_TARGET_RISCV64_
1121
+ #endif // NK_EACH_RVV_H