usearch 2.21.4 → 2.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (604) hide show
  1. package/README.md +65 -69
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +8 -8
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +124 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +224 -48
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/README.md +1 -1
  16. package/javascript/dist/cjs/usearch.d.ts +8 -3
  17. package/javascript/dist/cjs/usearch.js +12 -4
  18. package/javascript/dist/esm/usearch.d.ts +8 -3
  19. package/javascript/dist/esm/usearch.js +12 -4
  20. package/javascript/lib.cpp +17 -0
  21. package/javascript/usearch.test.js +117 -94
  22. package/javascript/usearch.ts +14 -6
  23. package/numkong/.clang-tidy +30 -0
  24. package/{simsimd → numkong}/.cmake-format.py +18 -0
  25. package/numkong/.editorconfig +12 -0
  26. package/numkong/.git-blame-ignore-revs +123 -0
  27. package/numkong/.mailmap +15 -0
  28. package/numkong/.spi.yml +11 -0
  29. package/numkong/.swift-format +14 -0
  30. package/numkong/CITATION.cff +81 -0
  31. package/numkong/CONTRIBUTING.md +500 -0
  32. package/numkong/Cargo.toml +54 -0
  33. package/numkong/MANIFEST.in +8 -0
  34. package/numkong/Package.swift +84 -0
  35. package/numkong/README.md +569 -0
  36. package/numkong/bench/README.md +217 -0
  37. package/numkong/bench/bench-browser.html +323 -0
  38. package/numkong/bench/bench.cpp +344 -0
  39. package/numkong/bench/bench.hpp +816 -0
  40. package/numkong/bench/bench.js +148 -0
  41. package/numkong/bench/bench.mjs +616 -0
  42. package/numkong/bench/bench_attention.cpp +113 -0
  43. package/numkong/bench/bench_cast.cpp +264 -0
  44. package/numkong/bench/bench_cross_amx.cpp +62 -0
  45. package/numkong/bench/bench_cross_arm.cpp +245 -0
  46. package/numkong/bench/bench_cross_blas.cpp +309 -0
  47. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  48. package/numkong/bench/bench_cross_power.cpp +84 -0
  49. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  50. package/numkong/bench/bench_cross_serial.cpp +151 -0
  51. package/numkong/bench/bench_cross_sme.cpp +163 -0
  52. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  53. package/numkong/bench/bench_cross_x86.cpp +455 -0
  54. package/numkong/bench/bench_curved.cpp +210 -0
  55. package/numkong/bench/bench_dot.cpp +315 -0
  56. package/numkong/bench/bench_each.cpp +503 -0
  57. package/numkong/bench/bench_geospatial.cpp +112 -0
  58. package/numkong/bench/bench_maxsim.cpp +154 -0
  59. package/numkong/bench/bench_mesh.cpp +148 -0
  60. package/numkong/bench/bench_probability.cpp +60 -0
  61. package/numkong/bench/bench_reduce.cpp +371 -0
  62. package/numkong/bench/bench_set.cpp +73 -0
  63. package/numkong/bench/bench_sparse.cpp +198 -0
  64. package/numkong/bench/bench_spatial.cpp +347 -0
  65. package/numkong/bench/bench_trigonometry.cpp +137 -0
  66. package/numkong/bench/swift/Bench.swift +506 -0
  67. package/numkong/binding.gyp +169 -0
  68. package/numkong/build.rs +484 -0
  69. package/numkong/c/dispatch.h +512 -0
  70. package/numkong/c/dispatch_bf16.c +447 -0
  71. package/numkong/c/dispatch_bf16c.c +52 -0
  72. package/numkong/c/dispatch_e2m3.c +296 -0
  73. package/numkong/c/dispatch_e3m2.c +284 -0
  74. package/numkong/c/dispatch_e4m3.c +318 -0
  75. package/numkong/c/dispatch_e5m2.c +316 -0
  76. package/numkong/c/dispatch_f16.c +452 -0
  77. package/numkong/c/dispatch_f16c.c +58 -0
  78. package/numkong/c/dispatch_f32.c +434 -0
  79. package/numkong/c/dispatch_f32c.c +99 -0
  80. package/numkong/c/dispatch_f64.c +348 -0
  81. package/numkong/c/dispatch_f64c.c +98 -0
  82. package/numkong/c/dispatch_i16.c +96 -0
  83. package/numkong/c/dispatch_i32.c +89 -0
  84. package/numkong/c/dispatch_i4.c +153 -0
  85. package/numkong/c/dispatch_i64.c +86 -0
  86. package/numkong/c/dispatch_i8.c +348 -0
  87. package/numkong/c/dispatch_other.c +348 -0
  88. package/numkong/c/dispatch_u1.c +193 -0
  89. package/numkong/c/dispatch_u16.c +124 -0
  90. package/numkong/c/dispatch_u32.c +118 -0
  91. package/numkong/c/dispatch_u4.c +153 -0
  92. package/numkong/c/dispatch_u64.c +102 -0
  93. package/numkong/c/dispatch_u8.c +364 -0
  94. package/numkong/c/numkong.c +940 -0
  95. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  96. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  97. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_math_lib.cmake +46 -0
  99. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  100. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  101. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  102. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  103. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  104. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  105. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  106. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  107. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  108. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  109. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  110. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  111. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  112. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  113. package/numkong/include/README.md +643 -0
  114. package/numkong/include/module.modulemap +129 -0
  115. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  116. package/numkong/include/numkong/attention/sme.h +2075 -0
  117. package/numkong/include/numkong/attention.h +49 -0
  118. package/numkong/include/numkong/capabilities.h +930 -0
  119. package/numkong/include/numkong/cast/README.md +262 -0
  120. package/numkong/include/numkong/cast/diamond.h +64 -0
  121. package/numkong/include/numkong/cast/haswell.h +823 -0
  122. package/numkong/include/numkong/cast/icelake.h +475 -0
  123. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  124. package/numkong/include/numkong/cast/neon.h +1174 -0
  125. package/numkong/include/numkong/cast/powervsx.h +449 -0
  126. package/numkong/include/numkong/cast/rvv.h +970 -0
  127. package/numkong/include/numkong/cast/sapphire.h +262 -0
  128. package/numkong/include/numkong/cast/serial.h +2336 -0
  129. package/numkong/include/numkong/cast/skylake.h +915 -0
  130. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  131. package/numkong/include/numkong/cast.h +254 -0
  132. package/numkong/include/numkong/cast.hpp +44 -0
  133. package/numkong/include/numkong/curved/README.md +223 -0
  134. package/numkong/include/numkong/curved/genoa.h +182 -0
  135. package/numkong/include/numkong/curved/haswell.h +276 -0
  136. package/numkong/include/numkong/curved/neon.h +329 -0
  137. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  138. package/numkong/include/numkong/curved/rvv.h +305 -0
  139. package/numkong/include/numkong/curved/serial.h +207 -0
  140. package/numkong/include/numkong/curved/skylake.h +457 -0
  141. package/numkong/include/numkong/curved/smef64.h +530 -0
  142. package/numkong/include/numkong/curved.h +513 -0
  143. package/numkong/include/numkong/curved.hpp +144 -0
  144. package/numkong/include/numkong/dot/README.md +442 -0
  145. package/numkong/include/numkong/dot/alder.h +563 -0
  146. package/numkong/include/numkong/dot/diamond.h +158 -0
  147. package/numkong/include/numkong/dot/genoa.h +289 -0
  148. package/numkong/include/numkong/dot/haswell.h +1723 -0
  149. package/numkong/include/numkong/dot/icelake.h +997 -0
  150. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  151. package/numkong/include/numkong/dot/neon.h +869 -0
  152. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  153. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  154. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  155. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  156. package/numkong/include/numkong/dot/powervsx.h +752 -0
  157. package/numkong/include/numkong/dot/rvv.h +722 -0
  158. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  159. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  160. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  161. package/numkong/include/numkong/dot/sapphire.h +141 -0
  162. package/numkong/include/numkong/dot/serial.h +874 -0
  163. package/numkong/include/numkong/dot/sierra.h +405 -0
  164. package/numkong/include/numkong/dot/skylake.h +1116 -0
  165. package/numkong/include/numkong/dot/sve.h +420 -0
  166. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  167. package/numkong/include/numkong/dot/svehalf.h +168 -0
  168. package/numkong/include/numkong/dot/svesdot.h +90 -0
  169. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  170. package/numkong/include/numkong/dot.h +1136 -0
  171. package/numkong/include/numkong/dot.hpp +94 -0
  172. package/numkong/include/numkong/dots/README.md +496 -0
  173. package/numkong/include/numkong/dots/alder.h +117 -0
  174. package/numkong/include/numkong/dots/diamond.h +86 -0
  175. package/numkong/include/numkong/dots/genoa.h +100 -0
  176. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  177. package/numkong/include/numkong/dots/haswell.h +310 -0
  178. package/numkong/include/numkong/dots/icelake.h +180 -0
  179. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  180. package/numkong/include/numkong/dots/neon.h +123 -0
  181. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  182. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  183. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  184. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  185. package/numkong/include/numkong/dots/powervsx.h +194 -0
  186. package/numkong/include/numkong/dots/rvv.h +2593 -0
  187. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  188. package/numkong/include/numkong/dots/serial.h +2867 -0
  189. package/numkong/include/numkong/dots/sierra.h +100 -0
  190. package/numkong/include/numkong/dots/skylake.h +207 -0
  191. package/numkong/include/numkong/dots/sme.h +5041 -0
  192. package/numkong/include/numkong/dots/smebi32.h +477 -0
  193. package/numkong/include/numkong/dots/smef64.h +1328 -0
  194. package/numkong/include/numkong/dots/smehalf.h +47 -0
  195. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  196. package/numkong/include/numkong/dots.h +3110 -0
  197. package/numkong/include/numkong/dots.hpp +641 -0
  198. package/numkong/include/numkong/each/README.md +465 -0
  199. package/numkong/include/numkong/each/haswell.h +1658 -0
  200. package/numkong/include/numkong/each/icelake.h +272 -0
  201. package/numkong/include/numkong/each/neon.h +1138 -0
  202. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  203. package/numkong/include/numkong/each/neonhalf.h +318 -0
  204. package/numkong/include/numkong/each/rvv.h +1121 -0
  205. package/numkong/include/numkong/each/sapphire.h +343 -0
  206. package/numkong/include/numkong/each/serial.h +282 -0
  207. package/numkong/include/numkong/each/skylake.h +1562 -0
  208. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  209. package/numkong/include/numkong/each.h +2232 -0
  210. package/numkong/include/numkong/each.hpp +434 -0
  211. package/numkong/include/numkong/geospatial/README.md +147 -0
  212. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  213. package/numkong/include/numkong/geospatial/neon.h +615 -0
  214. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  215. package/numkong/include/numkong/geospatial/serial.h +309 -0
  216. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  217. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  218. package/numkong/include/numkong/geospatial.h +453 -0
  219. package/numkong/include/numkong/geospatial.hpp +235 -0
  220. package/numkong/include/numkong/matrix.hpp +336 -0
  221. package/numkong/include/numkong/maxsim/README.md +187 -0
  222. package/numkong/include/numkong/maxsim/alder.h +511 -0
  223. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  224. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  225. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  226. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  227. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  228. package/numkong/include/numkong/maxsim/serial.h +490 -0
  229. package/numkong/include/numkong/maxsim/sme.h +993 -0
  230. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  231. package/numkong/include/numkong/maxsim.h +571 -0
  232. package/numkong/include/numkong/maxsim.hpp +133 -0
  233. package/numkong/include/numkong/mesh/README.md +219 -0
  234. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  235. package/numkong/include/numkong/mesh/neon.h +1821 -0
  236. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  237. package/numkong/include/numkong/mesh/rvv.h +936 -0
  238. package/numkong/include/numkong/mesh/serial.h +721 -0
  239. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  240. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  241. package/numkong/include/numkong/mesh.h +666 -0
  242. package/numkong/include/numkong/mesh.hpp +701 -0
  243. package/numkong/include/numkong/numkong.h +78 -0
  244. package/numkong/include/numkong/numkong.hpp +58 -0
  245. package/numkong/include/numkong/probability/README.md +177 -0
  246. package/numkong/include/numkong/probability/haswell.h +267 -0
  247. package/numkong/include/numkong/probability/neon.h +255 -0
  248. package/numkong/include/numkong/probability/rvv.h +414 -0
  249. package/numkong/include/numkong/probability/serial.h +181 -0
  250. package/numkong/include/numkong/probability/skylake.h +324 -0
  251. package/numkong/include/numkong/probability.h +380 -0
  252. package/numkong/include/numkong/probability.hpp +120 -0
  253. package/numkong/include/numkong/random.h +50 -0
  254. package/numkong/include/numkong/random.hpp +285 -0
  255. package/numkong/include/numkong/reduce/README.md +552 -0
  256. package/numkong/include/numkong/reduce/alder.h +636 -0
  257. package/numkong/include/numkong/reduce/genoa.h +201 -0
  258. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  259. package/numkong/include/numkong/reduce/icelake.h +549 -0
  260. package/numkong/include/numkong/reduce/neon.h +3980 -0
  261. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  262. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  263. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  264. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  265. package/numkong/include/numkong/reduce/serial.h +753 -0
  266. package/numkong/include/numkong/reduce/sierra.h +338 -0
  267. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  268. package/numkong/include/numkong/reduce/sve.h +52 -0
  269. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  270. package/numkong/include/numkong/reduce.h +1582 -0
  271. package/numkong/include/numkong/reduce.hpp +778 -0
  272. package/numkong/include/numkong/scalar/README.md +89 -0
  273. package/numkong/include/numkong/scalar/haswell.h +122 -0
  274. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  275. package/numkong/include/numkong/scalar/neon.h +122 -0
  276. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  277. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  278. package/numkong/include/numkong/scalar/rvv.h +211 -0
  279. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  280. package/numkong/include/numkong/scalar/serial.h +332 -0
  281. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  282. package/numkong/include/numkong/scalar.h +696 -0
  283. package/numkong/include/numkong/set/README.md +179 -0
  284. package/numkong/include/numkong/set/haswell.h +334 -0
  285. package/numkong/include/numkong/set/icelake.h +485 -0
  286. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  287. package/numkong/include/numkong/set/neon.h +363 -0
  288. package/numkong/include/numkong/set/powervsx.h +326 -0
  289. package/numkong/include/numkong/set/rvv.h +226 -0
  290. package/numkong/include/numkong/set/rvvbb.h +117 -0
  291. package/numkong/include/numkong/set/serial.h +174 -0
  292. package/numkong/include/numkong/set/sve.h +187 -0
  293. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  294. package/numkong/include/numkong/set.h +471 -0
  295. package/numkong/include/numkong/set.hpp +114 -0
  296. package/numkong/include/numkong/sets/README.md +149 -0
  297. package/numkong/include/numkong/sets/haswell.h +63 -0
  298. package/numkong/include/numkong/sets/icelake.h +66 -0
  299. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  300. package/numkong/include/numkong/sets/neon.h +61 -0
  301. package/numkong/include/numkong/sets/powervsx.h +65 -0
  302. package/numkong/include/numkong/sets/serial.h +43 -0
  303. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  304. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  305. package/numkong/include/numkong/sets.h +382 -0
  306. package/numkong/include/numkong/sparse/README.md +156 -0
  307. package/numkong/include/numkong/sparse/icelake.h +463 -0
  308. package/numkong/include/numkong/sparse/neon.h +288 -0
  309. package/numkong/include/numkong/sparse/serial.h +117 -0
  310. package/numkong/include/numkong/sparse/sve2.h +504 -0
  311. package/numkong/include/numkong/sparse/turin.h +322 -0
  312. package/numkong/include/numkong/sparse.h +363 -0
  313. package/numkong/include/numkong/sparse.hpp +113 -0
  314. package/numkong/include/numkong/spatial/README.md +439 -0
  315. package/numkong/include/numkong/spatial/alder.h +604 -0
  316. package/numkong/include/numkong/spatial/diamond.h +240 -0
  317. package/numkong/include/numkong/spatial/genoa.h +222 -0
  318. package/numkong/include/numkong/spatial/haswell.h +979 -0
  319. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  320. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  321. package/numkong/include/numkong/spatial/neon.h +846 -0
  322. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  323. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  324. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  325. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  326. package/numkong/include/numkong/spatial/rvv.h +991 -0
  327. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  328. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  329. package/numkong/include/numkong/spatial/serial.h +347 -0
  330. package/numkong/include/numkong/spatial/sierra.h +516 -0
  331. package/numkong/include/numkong/spatial/skylake.h +605 -0
  332. package/numkong/include/numkong/spatial/sve.h +246 -0
  333. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  334. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  335. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  336. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  337. package/numkong/include/numkong/spatial.h +1681 -0
  338. package/numkong/include/numkong/spatial.hpp +183 -0
  339. package/numkong/include/numkong/spatials/README.md +580 -0
  340. package/numkong/include/numkong/spatials/alder.h +94 -0
  341. package/numkong/include/numkong/spatials/diamond.h +82 -0
  342. package/numkong/include/numkong/spatials/genoa.h +94 -0
  343. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  344. package/numkong/include/numkong/spatials/haswell.h +219 -0
  345. package/numkong/include/numkong/spatials/icelake.h +113 -0
  346. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  347. package/numkong/include/numkong/spatials/neon.h +109 -0
  348. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  349. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  350. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  351. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  352. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  353. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  354. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  355. package/numkong/include/numkong/spatials/serial.h +248 -0
  356. package/numkong/include/numkong/spatials/sierra.h +96 -0
  357. package/numkong/include/numkong/spatials/skylake.h +184 -0
  358. package/numkong/include/numkong/spatials/sme.h +1927 -0
  359. package/numkong/include/numkong/spatials/smef64.h +483 -0
  360. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  361. package/numkong/include/numkong/spatials.h +3374 -0
  362. package/numkong/include/numkong/spatials.hpp +509 -0
  363. package/numkong/include/numkong/tensor.hpp +1981 -0
  364. package/numkong/include/numkong/trigonometry/README.md +184 -0
  365. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  366. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  367. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  368. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  369. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  370. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  371. package/numkong/include/numkong/trigonometry.h +467 -0
  372. package/numkong/include/numkong/trigonometry.hpp +166 -0
  373. package/numkong/include/numkong/types.h +1695 -0
  374. package/numkong/include/numkong/types.hpp +6027 -0
  375. package/numkong/include/numkong/vector.hpp +766 -0
  376. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  377. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  378. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  380. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  381. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  382. package/numkong/javascript/README.md +254 -0
  383. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  384. package/numkong/javascript/numkong-browser.ts +40 -0
  385. package/numkong/javascript/numkong-wasm.ts +750 -0
  386. package/numkong/javascript/numkong.c +712 -0
  387. package/numkong/javascript/numkong.ts +613 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  389. package/numkong/javascript/tsconfig-cjs.json +13 -0
  390. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  391. package/numkong/javascript/types.ts +784 -0
  392. package/numkong/numkong.gypi +104 -0
  393. package/numkong/package.json +108 -0
  394. package/numkong/probes/arm_neon.c +8 -0
  395. package/numkong/probes/arm_neon_bfdot.c +9 -0
  396. package/numkong/probes/arm_neon_fhm.c +9 -0
  397. package/numkong/probes/arm_neon_half.c +8 -0
  398. package/numkong/probes/arm_neon_sdot.c +9 -0
  399. package/numkong/probes/arm_neonfp8.c +9 -0
  400. package/numkong/probes/arm_sme.c +16 -0
  401. package/numkong/probes/arm_sme2.c +16 -0
  402. package/numkong/probes/arm_sme2p1.c +16 -0
  403. package/numkong/probes/arm_sme_bf16.c +16 -0
  404. package/numkong/probes/arm_sme_bi32.c +16 -0
  405. package/numkong/probes/arm_sme_f64.c +16 -0
  406. package/numkong/probes/arm_sme_fa64.c +14 -0
  407. package/numkong/probes/arm_sme_half.c +16 -0
  408. package/numkong/probes/arm_sme_lut2.c +15 -0
  409. package/numkong/probes/arm_sve.c +18 -0
  410. package/numkong/probes/arm_sve2.c +20 -0
  411. package/numkong/probes/arm_sve2p1.c +18 -0
  412. package/numkong/probes/arm_sve_bfdot.c +20 -0
  413. package/numkong/probes/arm_sve_half.c +18 -0
  414. package/numkong/probes/arm_sve_sdot.c +21 -0
  415. package/numkong/probes/loongarch_lasx.c +12 -0
  416. package/numkong/probes/power_vsx.c +12 -0
  417. package/numkong/probes/probe.js +127 -0
  418. package/numkong/probes/riscv_rvv.c +14 -0
  419. package/numkong/probes/riscv_rvv_bb.c +15 -0
  420. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  421. package/numkong/probes/riscv_rvv_half.c +14 -0
  422. package/numkong/probes/wasm_v128relaxed.c +11 -0
  423. package/numkong/probes/x86_alder.c +17 -0
  424. package/numkong/probes/x86_diamond.c +17 -0
  425. package/numkong/probes/x86_genoa.c +17 -0
  426. package/numkong/probes/x86_graniteamx.c +19 -0
  427. package/numkong/probes/x86_haswell.c +11 -0
  428. package/numkong/probes/x86_icelake.c +17 -0
  429. package/numkong/probes/x86_sapphire.c +16 -0
  430. package/numkong/probes/x86_sapphireamx.c +18 -0
  431. package/numkong/probes/x86_sierra.c +17 -0
  432. package/numkong/probes/x86_skylake.c +15 -0
  433. package/numkong/probes/x86_turin.c +17 -0
  434. package/numkong/rust/README.md +698 -0
  435. package/numkong/rust/capabilities.rs +96 -0
  436. package/numkong/rust/cast.rs +314 -0
  437. package/numkong/rust/curved.rs +391 -0
  438. package/numkong/rust/each.rs +2721 -0
  439. package/numkong/rust/geospatial.rs +343 -0
  440. package/numkong/rust/matrix.rs +5424 -0
  441. package/numkong/rust/maxsim.rs +540 -0
  442. package/numkong/rust/mesh.rs +837 -0
  443. package/numkong/rust/numkong.rs +409 -0
  444. package/numkong/rust/probability.rs +272 -0
  445. package/numkong/rust/reduce.rs +1249 -0
  446. package/numkong/rust/set.rs +194 -0
  447. package/numkong/rust/sparse.rs +513 -0
  448. package/numkong/rust/spatial.rs +1815 -0
  449. package/numkong/rust/tensor.rs +7565 -0
  450. package/numkong/rust/types.rs +3666 -0
  451. package/numkong/rust/vector.rs +1899 -0
  452. package/numkong/rustfmt.toml +2 -0
  453. package/numkong/swift/Matrix.swift +1388 -0
  454. package/numkong/swift/MaxSim.swift +151 -0
  455. package/numkong/swift/NumKong.swift +283 -0
  456. package/numkong/swift/Numerics.swift +329 -0
  457. package/numkong/swift/README.md +433 -0
  458. package/numkong/swift/Scalar.swift +741 -0
  459. package/numkong/swift/Tensor.swift +209 -0
  460. package/numkong/test/README.md +315 -0
  461. package/numkong/test/playwright.config.ts +16 -0
  462. package/numkong/test/swift/Test.swift +534 -0
  463. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  464. package/numkong/test/test-browser-bundle.html +188 -0
  465. package/numkong/test/test-browser-runner.mjs +43 -0
  466. package/numkong/test/test-browser.html +305 -0
  467. package/numkong/test/test-dtypes.mjs +453 -0
  468. package/numkong/test/test-wasm.mjs +267 -0
  469. package/numkong/test/test.cpp +442 -0
  470. package/numkong/test/test.hpp +580 -0
  471. package/numkong/test/test.mjs +363 -0
  472. package/numkong/test/test_base.py +1041 -0
  473. package/numkong/test/test_cast.cpp +242 -0
  474. package/numkong/test/test_cross.hpp +551 -0
  475. package/numkong/test/test_cross.py +467 -0
  476. package/numkong/test/test_cross_amx.cpp +50 -0
  477. package/numkong/test/test_cross_arm.cpp +225 -0
  478. package/numkong/test/test_cross_blas.cpp +354 -0
  479. package/numkong/test/test_cross_loongarch.cpp +90 -0
  480. package/numkong/test/test_cross_power.cpp +73 -0
  481. package/numkong/test/test_cross_rvv.cpp +109 -0
  482. package/numkong/test/test_cross_serial.cpp +181 -0
  483. package/numkong/test/test_cross_sme.cpp +148 -0
  484. package/numkong/test/test_cross_wasm.cpp +141 -0
  485. package/numkong/test/test_cross_x86.cpp +442 -0
  486. package/numkong/test/test_curved.cpp +182 -0
  487. package/numkong/test/test_curved.py +207 -0
  488. package/numkong/test/test_dot.cpp +353 -0
  489. package/numkong/test/test_dot.py +296 -0
  490. package/numkong/test/test_each.cpp +437 -0
  491. package/numkong/test/test_each.py +756 -0
  492. package/numkong/test/test_geospatial.cpp +155 -0
  493. package/numkong/test/test_geospatial.py +241 -0
  494. package/numkong/test/test_maxsim.cpp +138 -0
  495. package/numkong/test/test_maxsim.py +209 -0
  496. package/numkong/test/test_mesh.cpp +203 -0
  497. package/numkong/test/test_mesh.py +394 -0
  498. package/numkong/test/test_probability.cpp +136 -0
  499. package/numkong/test/test_probability.py +166 -0
  500. package/numkong/test/test_reduce.cpp +563 -0
  501. package/numkong/test/test_reduce.py +781 -0
  502. package/numkong/test/test_set.cpp +151 -0
  503. package/numkong/test/test_set.py +120 -0
  504. package/numkong/test/test_similarities.py +579 -0
  505. package/numkong/test/test_sparse.cpp +121 -0
  506. package/numkong/test/test_sparse.py +118 -0
  507. package/numkong/test/test_spatial.cpp +518 -0
  508. package/numkong/test/test_spatial.py +269 -0
  509. package/numkong/test/test_tensor.cpp +884 -0
  510. package/numkong/test/test_tensor.py +1417 -0
  511. package/numkong/test/test_trigonometry.cpp +156 -0
  512. package/numkong/test/test_trigonometry.py +180 -0
  513. package/package.json +7 -8
  514. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  515. package/prebuilds/linux-arm64/usearch.node +0 -0
  516. package/prebuilds/linux-x64/usearch.node +0 -0
  517. package/fp16/CMakeLists.txt +0 -176
  518. package/fp16/LICENSE +0 -11
  519. package/fp16/README.md +0 -20
  520. package/fp16/bench/alt-element.cc +0 -89
  521. package/fp16/bench/from-alt-array.cc +0 -175
  522. package/fp16/bench/from-ieee-array.cc +0 -399
  523. package/fp16/bench/ieee-element.cc +0 -272
  524. package/fp16/bench/to-alt-array.cc +0 -91
  525. package/fp16/bench/to-ieee-array.cc +0 -303
  526. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  527. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  528. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  529. package/fp16/configure.py +0 -65
  530. package/fp16/confu.yaml +0 -8
  531. package/fp16/include/fp16/__init__.py +0 -0
  532. package/fp16/include/fp16/avx.py +0 -50
  533. package/fp16/include/fp16/avx2.py +0 -53
  534. package/fp16/include/fp16/bitcasts.h +0 -92
  535. package/fp16/include/fp16/fp16.h +0 -451
  536. package/fp16/include/fp16/psimd.h +0 -131
  537. package/fp16/include/fp16.h +0 -11
  538. package/fp16/jni/Android.mk +0 -6
  539. package/fp16/jni/Application.mk +0 -4
  540. package/fp16/test/alt-from-fp32-value.cc +0 -498
  541. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  542. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  543. package/fp16/test/alt-to-fp32-value.cc +0 -348
  544. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  545. package/fp16/test/bitcasts.cc +0 -56
  546. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  547. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  548. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  549. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  550. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  551. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  552. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  553. package/fp16/test/peachpy/stubs.py +0 -38
  554. package/fp16/test/tables.cc +0 -530
  555. package/fp16/test/tables.h +0 -13
  556. package/fp16/third-party/THHalf.h +0 -92
  557. package/fp16/third-party/eigen-half.h +0 -170
  558. package/fp16/third-party/float16-compressor.h +0 -80
  559. package/fp16/third-party/half.hpp +0 -2916
  560. package/fp16/third-party/npy-halffloat.h +0 -328
  561. package/simsimd/.git-blame-ignore-revs +0 -2
  562. package/simsimd/CONTRIBUTING.md +0 -321
  563. package/simsimd/Cargo.toml +0 -49
  564. package/simsimd/MANIFEST.in +0 -4
  565. package/simsimd/Package.swift +0 -28
  566. package/simsimd/README.md +0 -1493
  567. package/simsimd/binding.gyp +0 -35
  568. package/simsimd/build.rs +0 -70
  569. package/simsimd/c/lib.c +0 -374
  570. package/simsimd/include/module.modulemap +0 -4
  571. package/simsimd/include/simsimd/binary.h +0 -486
  572. package/simsimd/include/simsimd/curved.h +0 -1541
  573. package/simsimd/include/simsimd/dot.h +0 -1853
  574. package/simsimd/include/simsimd/elementwise.h +0 -2465
  575. package/simsimd/include/simsimd/geospatial.h +0 -43
  576. package/simsimd/include/simsimd/mesh.h +0 -69
  577. package/simsimd/include/simsimd/probability.h +0 -606
  578. package/simsimd/include/simsimd/simsimd.h +0 -2567
  579. package/simsimd/include/simsimd/sparse.h +0 -1384
  580. package/simsimd/include/simsimd/spatial.h +0 -2335
  581. package/simsimd/include/simsimd/types.h +0 -666
  582. package/simsimd/javascript/fallback.ts +0 -233
  583. package/simsimd/javascript/lib.c +0 -125
  584. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  585. package/simsimd/javascript/simsimd.ts +0 -179
  586. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  587. package/simsimd/package-lock.json +0 -1761
  588. package/simsimd/package.json +0 -61
  589. package/simsimd/rust/lib.rs +0 -1821
  590. package/simsimd/scripts/bench.cxx +0 -1170
  591. package/simsimd/scripts/bench.ipynb +0 -187
  592. package/simsimd/scripts/bench.js +0 -127
  593. package/simsimd/scripts/bench.rs +0 -876
  594. package/simsimd/scripts/bench_vectors.py +0 -734
  595. package/simsimd/scripts/bench_vectors_live.py +0 -213
  596. package/simsimd/scripts/test.c +0 -187
  597. package/simsimd/scripts/test.mjs +0 -220
  598. package/simsimd/scripts/test.py +0 -1505
  599. package/simsimd/swift/SimSIMD.swift +0 -149
  600. package/simsimd/swift/Test.swift +0 -128
  601. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  602. /package/{simsimd → numkong}/LICENSE +0 -0
  603. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  604. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -0,0 +1,3533 @@
1
+ /**
2
+ * @brief SIMD-accelerated Reductions for RISC-V.
3
+ * @file include/numkong/reduce/rvv.h
4
+ * @author Ash Vardanian
5
+ * @date February 13, 2026
6
+ *
7
+ * @sa include/numkong/reduce.h
8
+ */
9
+ #ifndef NK_REDUCE_RVV_H
10
+ #define NK_REDUCE_RVV_H
11
+
12
+ #if NK_TARGET_RISCV64_
13
+ #if NK_TARGET_RVV
14
+
15
+ #include "numkong/types.h"
16
+ #include "numkong/cast/rvv.h"
17
+ #include "numkong/reduce/serial.h"
18
+
19
+ #if defined(__clang__)
20
+ #pragma clang attribute push(__attribute__((target("arch=+v"))), apply_to = function)
21
+ #elif defined(__GNUC__)
22
+ #pragma GCC push_options
23
+ #pragma GCC target("arch=+v")
24
+ #endif
25
+
26
+ #if defined(__cplusplus)
27
+ extern "C" {
28
+ #endif
29
+
30
+ /** @brief Saturating horizontal sum of u64m1 via tree fold: O(log vector_length) vector ops. */
31
+ NK_INTERNAL nk_u64_t nk_reduce_vsaddu_u64m1_rvv_(vuint64m1_t acc_u64m1, nk_size_t vector_length) {
32
+ for (nk_size_t half = vector_length >> 1; half > 0; half >>= 1) {
33
+ vuint64m1_t shifted_u64m1 = __riscv_vslidedown_vx_u64m1(acc_u64m1, half, vector_length);
34
+ acc_u64m1 = __riscv_vsaddu_vv_u64m1(acc_u64m1, shifted_u64m1, vector_length);
35
+ }
36
+ return __riscv_vmv_x_s_u64m1_u64(acc_u64m1);
37
+ }
38
+
39
+ /** @brief Saturating horizontal sum of u64m2 via tree fold: O(log vector_length) vector ops. */
40
+ NK_INTERNAL nk_u64_t nk_reduce_vsaddu_u64m2_rvv_(vuint64m2_t acc_u64m2, nk_size_t vector_length) {
41
+ for (nk_size_t half = vector_length >> 1; half > 0; half >>= 1) {
42
+ vuint64m2_t shifted_u64m2 = __riscv_vslidedown_vx_u64m2(acc_u64m2, half, vector_length);
43
+ acc_u64m2 = __riscv_vsaddu_vv_u64m2(acc_u64m2, shifted_u64m2, vector_length);
44
+ }
45
+ return __riscv_vmv_x_s_u64m2_u64(acc_u64m2);
46
+ }
47
+
48
+ /** @brief 128-bit horizontal sum of (upper:i64m1, lower:u64m1) via tree fold, then saturate to i64. */
49
+ NK_INTERNAL nk_i64_t nk_reduce_128bit_sum_i64m1_rvv_( //
50
+ vuint64m1_t sum_low_u64m1, vint64m1_t sum_high_i64m1, nk_size_t vector_length) {
51
+ for (nk_size_t half = vector_length >> 1; half > 0; half >>= 1) {
52
+ vuint64m1_t shifted_low_u64m1 = __riscv_vslidedown_vx_u64m1(sum_low_u64m1, half, vector_length);
53
+ vint64m1_t shifted_high_i64m1 = __riscv_vslidedown_vx_i64m1(sum_high_i64m1, half, vector_length);
54
+ vuint64m1_t new_low_u64m1 = __riscv_vadd_vv_u64m1(sum_low_u64m1, shifted_low_u64m1, vector_length);
55
+ vbool64_t carry_b64 = __riscv_vmsltu_vv_u64m1_b64(new_low_u64m1, sum_low_u64m1, vector_length);
56
+ vint64m1_t carry_i64m1 = __riscv_vmerge_vxm_i64m1(__riscv_vmv_v_x_i64m1(0, vector_length), 1, carry_b64,
57
+ vector_length);
58
+ sum_high_i64m1 = __riscv_vadd_vv_i64m1(sum_high_i64m1, shifted_high_i64m1, vector_length);
59
+ sum_high_i64m1 = __riscv_vadd_vv_i64m1(sum_high_i64m1, carry_i64m1, vector_length);
60
+ sum_low_u64m1 = new_low_u64m1;
61
+ }
62
+ nk_u64_t total_low = __riscv_vmv_x_s_u64m1_u64(sum_low_u64m1);
63
+ nk_i64_t total_high = __riscv_vmv_x_s_i64m1_i64(sum_high_i64m1);
64
+ nk_i64_t total_low_signed = (nk_i64_t)total_low;
65
+ if (total_high == (total_low_signed >> 63)) return total_low_signed;
66
+ else if (total_high >= 0) return NK_I64_MAX;
67
+ else return NK_I64_MIN;
68
+ }
69
+
70
+ /** @brief 128-bit horizontal sum of (upper:i64m2, lower:u64m2) via tree fold, then saturate to i64. */
71
+ NK_INTERNAL nk_i64_t nk_reduce_128bit_sum_i64m2_rvv_( //
72
+ vuint64m2_t sum_low_u64m2, vint64m2_t sum_high_i64m2, nk_size_t vector_length) {
73
+ for (nk_size_t half = vector_length >> 1; half > 0; half >>= 1) {
74
+ vuint64m2_t shifted_low_u64m2 = __riscv_vslidedown_vx_u64m2(sum_low_u64m2, half, vector_length);
75
+ vint64m2_t shifted_high_i64m2 = __riscv_vslidedown_vx_i64m2(sum_high_i64m2, half, vector_length);
76
+ vuint64m2_t new_low_u64m2 = __riscv_vadd_vv_u64m2(sum_low_u64m2, shifted_low_u64m2, vector_length);
77
+ vbool32_t carry_b32 = __riscv_vmsltu_vv_u64m2_b32(new_low_u64m2, sum_low_u64m2, vector_length);
78
+ vint64m2_t carry_i64m2 = __riscv_vmerge_vxm_i64m2(__riscv_vmv_v_x_i64m2(0, vector_length), 1, carry_b32,
79
+ vector_length);
80
+ sum_high_i64m2 = __riscv_vadd_vv_i64m2(sum_high_i64m2, shifted_high_i64m2, vector_length);
81
+ sum_high_i64m2 = __riscv_vadd_vv_i64m2(sum_high_i64m2, carry_i64m2, vector_length);
82
+ sum_low_u64m2 = new_low_u64m2;
83
+ }
84
+ nk_u64_t total_low = __riscv_vmv_x_s_u64m2_u64(sum_low_u64m2);
85
+ nk_i64_t total_high = __riscv_vmv_x_s_i64m2_i64(sum_high_i64m2);
86
+ nk_i64_t total_low_signed = (nk_i64_t)total_low;
87
+ if (total_high == (total_low_signed >> 63)) return total_low_signed;
88
+ else if (total_high >= 0) return NK_I64_MAX;
89
+ else return NK_I64_MIN;
90
+ }
91
+
92
+ NK_INTERNAL void nk_reduce_moments_f32_rvv_contiguous_( //
93
+ nk_f32_t const *data, nk_size_t count, //
94
+ nk_f64_t *sum_ptr, nk_f64_t *sumsq_ptr) {
95
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
96
+ vfloat64m2_t sum_f64m2 = __riscv_vfmv_v_f_f64m2(0.0, max_vector_length);
97
+ vfloat64m2_t sumsq_f64m2 = __riscv_vfmv_v_f_f64m2(0.0, max_vector_length);
98
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data += vector_length) {
99
+ vector_length = __riscv_vsetvl_e32m1(count);
100
+ vfloat32m1_t data_f32m1 = __riscv_vle32_v_f32m1(data, vector_length);
101
+ sum_f64m2 = __riscv_vfwadd_wv_f64m2_tu(sum_f64m2, sum_f64m2, data_f32m1, vector_length);
102
+ sumsq_f64m2 = __riscv_vfwmacc_vv_f64m2_tu(sumsq_f64m2, data_f32m1, data_f32m1, vector_length);
103
+ }
104
+ vfloat64m1_t zero = __riscv_vfmv_v_f_f64m1(0.0, 1);
105
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m2_f64m1(sum_f64m2, zero, max_vector_length)),
106
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m2_f64m1(sumsq_f64m2, zero, max_vector_length));
107
+ }
108
+
109
+ NK_INTERNAL void nk_reduce_moments_f32_rvv_strided_( //
110
+ nk_f32_t const *data, nk_size_t count, nk_size_t stride_bytes, //
111
+ nk_f64_t *sum_ptr, nk_f64_t *sumsq_ptr) {
112
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
113
+ vfloat64m2_t sum_f64m2 = __riscv_vfmv_v_f_f64m2(0.0, max_vector_length);
114
+ vfloat64m2_t sumsq_f64m2 = __riscv_vfmv_v_f_f64m2(0.0, max_vector_length);
115
+ unsigned char const *ptr = (unsigned char const *)data;
116
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
117
+ vector_length = __riscv_vsetvl_e32m1(count);
118
+ vfloat32m1_t data_f32m1 = __riscv_vlse32_v_f32m1((nk_f32_t const *)ptr, (nk_ssize_t)stride_bytes,
119
+ vector_length);
120
+ sum_f64m2 = __riscv_vfwadd_wv_f64m2_tu(sum_f64m2, sum_f64m2, data_f32m1, vector_length);
121
+ sumsq_f64m2 = __riscv_vfwmacc_vv_f64m2_tu(sumsq_f64m2, data_f32m1, data_f32m1, vector_length);
122
+ }
123
+ vfloat64m1_t zero = __riscv_vfmv_v_f_f64m1(0.0, 1);
124
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m2_f64m1(sum_f64m2, zero, max_vector_length)),
125
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m2_f64m1(sumsq_f64m2, zero, max_vector_length));
126
+ }
127
+
128
+ NK_PUBLIC void nk_reduce_moments_f32_rvv( //
129
+ nk_f32_t const *data, nk_size_t count, nk_size_t stride_bytes, //
130
+ nk_f64_t *sum, nk_f64_t *sumsq) {
131
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_f32_t);
132
+ int aligned = (stride_bytes % sizeof(nk_f32_t) == 0);
133
+ if (count == 0) *sum = 0, *sumsq = 0;
134
+ else if (!aligned) nk_reduce_moments_f32_serial(data, count, stride_bytes, sum, sumsq);
135
+ else if (stride_elements == 1) nk_reduce_moments_f32_rvv_contiguous_(data, count, sum, sumsq);
136
+ else nk_reduce_moments_f32_rvv_strided_(data, count, stride_bytes, sum, sumsq);
137
+ }
138
+
139
+ NK_INTERNAL void nk_reduce_minmax_f32_rvv_contiguous_( //
140
+ nk_f32_t const *data, nk_size_t count, //
141
+ nk_f32_t *min_value, nk_size_t *min_index, //
142
+ nk_f32_t *max_value, nk_size_t *max_index) {
143
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m1();
144
+ vfloat32m1_t min = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, max_vector_length);
145
+ vfloat32m1_t max = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, max_vector_length);
146
+ vuint64m2_t min_indices = __riscv_vmv_v_x_u64m2(0, max_vector_length);
147
+ vuint64m2_t max_indices = __riscv_vmv_v_x_u64m2(0, max_vector_length);
148
+ nk_size_t offset = 0;
149
+ for (nk_size_t remaining = count, max_vector_length; remaining > 0;
150
+ remaining -= max_vector_length, offset += max_vector_length) {
151
+ max_vector_length = __riscv_vsetvl_e32m1(remaining);
152
+ vfloat32m1_t data_f32m1 = __riscv_vle32_v_f32m1(data + offset, max_vector_length);
153
+ vuint64m2_t position_u64m2 = __riscv_vadd_vx_u64m2(__riscv_vid_v_u64m2(max_vector_length), (nk_u64_t)offset,
154
+ max_vector_length);
155
+ vbool32_t less_b32 = __riscv_vmflt_vv_f32m1_b32(data_f32m1, min, max_vector_length);
156
+ min = __riscv_vmerge_vvm_f32m1_tu(min, min, data_f32m1, less_b32, max_vector_length);
157
+ min_indices = __riscv_vmerge_vvm_u64m2_tu(min_indices, min_indices, position_u64m2, less_b32,
158
+ max_vector_length);
159
+ vbool32_t greater_b32 = __riscv_vmflt_vv_f32m1_b32(max, data_f32m1, max_vector_length);
160
+ max = __riscv_vmerge_vvm_f32m1_tu(max, max, data_f32m1, greater_b32, max_vector_length);
161
+ max_indices = __riscv_vmerge_vvm_u64m2_tu(max_indices, max_indices, position_u64m2, greater_b32,
162
+ max_vector_length);
163
+ }
164
+ vfloat32m1_t id_max = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, 1);
165
+ nk_f32_t mn = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredmin_vs_f32m1_f32m1(min, id_max, max_vector_length));
166
+ vfloat32m1_t id_min = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, 1);
167
+ nk_f32_t mx = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredmax_vs_f32m1_f32m1(max, id_min, max_vector_length));
168
+ if (mn == NK_F32_MAX && mx == NK_F32_MIN) {
169
+ *min_value = NK_F32_MAX, *min_index = NK_SIZE_MAX, *max_value = NK_F32_MIN, *max_index = NK_SIZE_MAX;
170
+ return;
171
+ }
172
+ vbool32_t min_match_b32 = __riscv_vmfeq_vf_f32m1_b32(min, mn, max_vector_length);
173
+ vuint64m2_t sentinel = __riscv_vmv_v_x_u64m2(NK_U64_MAX, max_vector_length);
174
+ vuint64m2_t min_cands = __riscv_vmerge_vvm_u64m2(sentinel, min_indices, min_match_b32, max_vector_length);
175
+ vuint64m1_t id_umax = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
176
+ *min_value = mn, *min_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
177
+ __riscv_vredminu_vs_u64m2_u64m1(min_cands, id_umax, max_vector_length));
178
+ vbool32_t max_match_b32 = __riscv_vmfeq_vf_f32m1_b32(max, mx, max_vector_length);
179
+ vuint64m2_t max_cands = __riscv_vmerge_vvm_u64m2(sentinel, max_indices, max_match_b32, max_vector_length);
180
+ *max_value = mx, *max_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
181
+ __riscv_vredminu_vs_u64m2_u64m1(max_cands, id_umax, max_vector_length));
182
+ }
183
+
184
+ NK_INTERNAL void nk_reduce_minmax_f32_rvv_strided_( //
185
+ nk_f32_t const *data, nk_size_t count, nk_size_t stride_bytes, //
186
+ nk_f32_t *min_value, nk_size_t *min_index, //
187
+ nk_f32_t *max_value, nk_size_t *max_index) {
188
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m1();
189
+ vfloat32m1_t min = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, max_vector_length);
190
+ vfloat32m1_t max = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, max_vector_length);
191
+ vuint64m2_t min_indices = __riscv_vmv_v_x_u64m2(0, max_vector_length);
192
+ vuint64m2_t max_indices = __riscv_vmv_v_x_u64m2(0, max_vector_length);
193
+ unsigned char const *ptr = (unsigned char const *)data;
194
+ nk_size_t offset = 0;
195
+ for (nk_size_t remaining = count, max_vector_length; remaining > 0;
196
+ remaining -= max_vector_length, offset += max_vector_length, ptr += max_vector_length * stride_bytes) {
197
+ max_vector_length = __riscv_vsetvl_e32m1(remaining);
198
+ vfloat32m1_t data_f32m1 = __riscv_vlse32_v_f32m1((nk_f32_t const *)ptr, (nk_ssize_t)stride_bytes,
199
+ max_vector_length);
200
+ vuint64m2_t position_u64m2 = __riscv_vadd_vx_u64m2(__riscv_vid_v_u64m2(max_vector_length), (nk_u64_t)offset,
201
+ max_vector_length);
202
+ vbool32_t less_b32 = __riscv_vmflt_vv_f32m1_b32(data_f32m1, min, max_vector_length);
203
+ min = __riscv_vmerge_vvm_f32m1_tu(min, min, data_f32m1, less_b32, max_vector_length);
204
+ min_indices = __riscv_vmerge_vvm_u64m2_tu(min_indices, min_indices, position_u64m2, less_b32,
205
+ max_vector_length);
206
+ vbool32_t greater_b32 = __riscv_vmflt_vv_f32m1_b32(max, data_f32m1, max_vector_length);
207
+ max = __riscv_vmerge_vvm_f32m1_tu(max, max, data_f32m1, greater_b32, max_vector_length);
208
+ max_indices = __riscv_vmerge_vvm_u64m2_tu(max_indices, max_indices, position_u64m2, greater_b32,
209
+ max_vector_length);
210
+ }
211
+ vfloat32m1_t id_max = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, 1);
212
+ nk_f32_t mn = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredmin_vs_f32m1_f32m1(min, id_max, max_vector_length));
213
+ vfloat32m1_t id_min = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, 1);
214
+ nk_f32_t mx = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredmax_vs_f32m1_f32m1(max, id_min, max_vector_length));
215
+ if (mn == NK_F32_MAX && mx == NK_F32_MIN) {
216
+ *min_value = NK_F32_MAX, *min_index = NK_SIZE_MAX, *max_value = NK_F32_MIN, *max_index = NK_SIZE_MAX;
217
+ return;
218
+ }
219
+ vbool32_t min_match_b32 = __riscv_vmfeq_vf_f32m1_b32(min, mn, max_vector_length);
220
+ vuint64m2_t sentinel = __riscv_vmv_v_x_u64m2(NK_U64_MAX, max_vector_length);
221
+ vuint64m2_t min_cands = __riscv_vmerge_vvm_u64m2(sentinel, min_indices, min_match_b32, max_vector_length);
222
+ vuint64m1_t id_umax = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
223
+ *min_value = mn, *min_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
224
+ __riscv_vredminu_vs_u64m2_u64m1(min_cands, id_umax, max_vector_length));
225
+ vbool32_t max_match_b32 = __riscv_vmfeq_vf_f32m1_b32(max, mx, max_vector_length);
226
+ vuint64m2_t max_cands = __riscv_vmerge_vvm_u64m2(sentinel, max_indices, max_match_b32, max_vector_length);
227
+ *max_value = mx, *max_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
228
+ __riscv_vredminu_vs_u64m2_u64m1(max_cands, id_umax, max_vector_length));
229
+ }
230
+
231
+ NK_PUBLIC void nk_reduce_minmax_f32_rvv( //
232
+ nk_f32_t const *data, nk_size_t count, nk_size_t stride_bytes, //
233
+ nk_f32_t *min_value, nk_size_t *min_index, //
234
+ nk_f32_t *max_value, nk_size_t *max_index) {
235
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_f32_t);
236
+ int aligned = (stride_bytes % sizeof(nk_f32_t) == 0);
237
+ if (count == 0)
238
+ *min_value = NK_F32_MAX, *min_index = NK_SIZE_MAX, *max_value = NK_F32_MIN, *max_index = NK_SIZE_MAX;
239
+ else if (!aligned)
240
+ nk_reduce_minmax_f32_serial(data, count, stride_bytes, min_value, min_index, max_value, max_index);
241
+ else if (stride_elements == 1)
242
+ nk_reduce_minmax_f32_rvv_contiguous_(data, count, min_value, min_index, max_value, max_index);
243
+ else nk_reduce_minmax_f32_rvv_strided_(data, count, stride_bytes, min_value, min_index, max_value, max_index);
244
+ }
245
+
246
+ NK_INTERNAL void nk_reduce_moments_f64_rvv_contiguous_( //
247
+ nk_f64_t const *data, nk_size_t count, //
248
+ nk_f64_t *sum_ptr, nk_f64_t *sumsq_ptr) {
249
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
250
+ vfloat64m4_t sum_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
251
+ vfloat64m4_t sumsq_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
252
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data += vector_length) {
253
+ vector_length = __riscv_vsetvl_e64m4(count);
254
+ vfloat64m4_t data_f64m4 = __riscv_vle64_v_f64m4(data, vector_length);
255
+ sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(sum_f64m4, sum_f64m4, data_f64m4, vector_length);
256
+ sumsq_f64m4 = __riscv_vfmacc_vv_f64m4_tu(sumsq_f64m4, data_f64m4, data_f64m4, vector_length);
257
+ }
258
+ vfloat64m1_t zero = __riscv_vfmv_v_f_f64m1(0.0, 1);
259
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sum_f64m4, zero, max_vector_length)),
260
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sumsq_f64m4, zero, max_vector_length));
261
+ }
262
+
263
+ NK_INTERNAL void nk_reduce_moments_f64_rvv_strided_( //
264
+ nk_f64_t const *data, nk_size_t count, nk_size_t stride_bytes, //
265
+ nk_f64_t *sum_ptr, nk_f64_t *sumsq_ptr) {
266
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
267
+ vfloat64m4_t sum_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
268
+ vfloat64m4_t sumsq_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
269
+ unsigned char const *ptr = (unsigned char const *)data;
270
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
271
+ vector_length = __riscv_vsetvl_e64m4(count);
272
+ vfloat64m4_t data_f64m4 = __riscv_vlse64_v_f64m4((nk_f64_t const *)ptr, (nk_ssize_t)stride_bytes,
273
+ vector_length);
274
+ sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(sum_f64m4, sum_f64m4, data_f64m4, vector_length);
275
+ sumsq_f64m4 = __riscv_vfmacc_vv_f64m4_tu(sumsq_f64m4, data_f64m4, data_f64m4, vector_length);
276
+ }
277
+ vfloat64m1_t zero = __riscv_vfmv_v_f_f64m1(0.0, 1);
278
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sum_f64m4, zero, max_vector_length)),
279
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sumsq_f64m4, zero, max_vector_length));
280
+ }
281
+
282
+ NK_PUBLIC void nk_reduce_moments_f64_rvv( //
283
+ nk_f64_t const *data, nk_size_t count, nk_size_t stride_bytes, //
284
+ nk_f64_t *sum, nk_f64_t *sumsq) {
285
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_f64_t);
286
+ int aligned = (stride_bytes % sizeof(nk_f64_t) == 0);
287
+ if (count == 0) *sum = 0, *sumsq = 0;
288
+ else if (!aligned) nk_reduce_moments_f64_serial(data, count, stride_bytes, sum, sumsq);
289
+ else if (stride_elements == 1) nk_reduce_moments_f64_rvv_contiguous_(data, count, sum, sumsq);
290
+ else nk_reduce_moments_f64_rvv_strided_(data, count, stride_bytes, sum, sumsq);
291
+ }
292
+
293
+ NK_INTERNAL void nk_reduce_minmax_f64_rvv_contiguous_( //
294
+ nk_f64_t const *data, nk_size_t count, //
295
+ nk_f64_t *min_value, nk_size_t *min_index, //
296
+ nk_f64_t *max_value, nk_size_t *max_index) {
297
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
298
+ vfloat64m1_t min = __riscv_vfmv_v_f_f64m1(NK_F64_MAX, max_vector_length);
299
+ vfloat64m1_t max = __riscv_vfmv_v_f_f64m1(NK_F64_MIN, max_vector_length);
300
+ vuint64m1_t min_indices = __riscv_vmv_v_x_u64m1(0, max_vector_length);
301
+ vuint64m1_t max_indices = __riscv_vmv_v_x_u64m1(0, max_vector_length);
302
+ nk_size_t offset = 0;
303
+ for (nk_size_t remaining = count, max_vector_length; remaining > 0;
304
+ remaining -= max_vector_length, offset += max_vector_length) {
305
+ max_vector_length = __riscv_vsetvl_e64m1(remaining);
306
+ vfloat64m1_t data_f64m1 = __riscv_vle64_v_f64m1(data + offset, max_vector_length);
307
+ vuint64m1_t position_u64m1 = __riscv_vadd_vx_u64m1(__riscv_vid_v_u64m1(max_vector_length), (nk_u64_t)offset,
308
+ max_vector_length);
309
+ vbool64_t less_b64 = __riscv_vmflt_vv_f64m1_b64(data_f64m1, min, max_vector_length);
310
+ min = __riscv_vmerge_vvm_f64m1_tu(min, min, data_f64m1, less_b64, max_vector_length);
311
+ min_indices = __riscv_vmerge_vvm_u64m1_tu(min_indices, min_indices, position_u64m1, less_b64,
312
+ max_vector_length);
313
+ vbool64_t greater_b64 = __riscv_vmflt_vv_f64m1_b64(max, data_f64m1, max_vector_length);
314
+ max = __riscv_vmerge_vvm_f64m1_tu(max, max, data_f64m1, greater_b64, max_vector_length);
315
+ max_indices = __riscv_vmerge_vvm_u64m1_tu(max_indices, max_indices, position_u64m1, greater_b64,
316
+ max_vector_length);
317
+ }
318
+ vfloat64m1_t id_max = __riscv_vfmv_v_f_f64m1(NK_F64_MAX, 1);
319
+ nk_f64_t mn = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredmin_vs_f64m1_f64m1(min, id_max, max_vector_length));
320
+ vfloat64m1_t id_min = __riscv_vfmv_v_f_f64m1(NK_F64_MIN, 1);
321
+ nk_f64_t mx = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredmax_vs_f64m1_f64m1(max, id_min, max_vector_length));
322
+ if (mn == NK_F64_MAX && mx == NK_F64_MIN) {
323
+ *min_value = NK_F64_MAX, *min_index = NK_SIZE_MAX, *max_value = NK_F64_MIN, *max_index = NK_SIZE_MAX;
324
+ return;
325
+ }
326
+ vbool64_t min_match_b64 = __riscv_vmfeq_vf_f64m1_b64(min, mn, max_vector_length);
327
+ vuint64m1_t sentinel = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
328
+ vuint64m1_t min_cands = __riscv_vmerge_vvm_u64m1(sentinel, min_indices, min_match_b64, max_vector_length);
329
+ vuint64m1_t id_umax = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
330
+ *min_value = mn, *min_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
331
+ __riscv_vredminu_vs_u64m1_u64m1(min_cands, id_umax, max_vector_length));
332
+ vbool64_t max_match_b64 = __riscv_vmfeq_vf_f64m1_b64(max, mx, max_vector_length);
333
+ vuint64m1_t max_cands = __riscv_vmerge_vvm_u64m1(sentinel, max_indices, max_match_b64, max_vector_length);
334
+ *max_value = mx, *max_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
335
+ __riscv_vredminu_vs_u64m1_u64m1(max_cands, id_umax, max_vector_length));
336
+ }
337
+
338
+ NK_INTERNAL void nk_reduce_minmax_f64_rvv_strided_( //
339
+ nk_f64_t const *data, nk_size_t count, nk_size_t stride_bytes, //
340
+ nk_f64_t *min_value, nk_size_t *min_index, //
341
+ nk_f64_t *max_value, nk_size_t *max_index) {
342
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
343
+ vfloat64m1_t min = __riscv_vfmv_v_f_f64m1(NK_F64_MAX, max_vector_length);
344
+ vfloat64m1_t max = __riscv_vfmv_v_f_f64m1(NK_F64_MIN, max_vector_length);
345
+ vuint64m1_t min_indices = __riscv_vmv_v_x_u64m1(0, max_vector_length);
346
+ vuint64m1_t max_indices = __riscv_vmv_v_x_u64m1(0, max_vector_length);
347
+ unsigned char const *ptr = (unsigned char const *)data;
348
+ nk_size_t offset = 0;
349
+ for (nk_size_t remaining = count, max_vector_length; remaining > 0;
350
+ remaining -= max_vector_length, offset += max_vector_length, ptr += max_vector_length * stride_bytes) {
351
+ max_vector_length = __riscv_vsetvl_e64m1(remaining);
352
+ vfloat64m1_t data_f64m1 = __riscv_vlse64_v_f64m1((nk_f64_t const *)ptr, (nk_ssize_t)stride_bytes,
353
+ max_vector_length);
354
+ vuint64m1_t position_u64m1 = __riscv_vadd_vx_u64m1(__riscv_vid_v_u64m1(max_vector_length), (nk_u64_t)offset,
355
+ max_vector_length);
356
+ vbool64_t less_b64 = __riscv_vmflt_vv_f64m1_b64(data_f64m1, min, max_vector_length);
357
+ min = __riscv_vmerge_vvm_f64m1_tu(min, min, data_f64m1, less_b64, max_vector_length);
358
+ min_indices = __riscv_vmerge_vvm_u64m1_tu(min_indices, min_indices, position_u64m1, less_b64,
359
+ max_vector_length);
360
+ vbool64_t greater_b64 = __riscv_vmflt_vv_f64m1_b64(max, data_f64m1, max_vector_length);
361
+ max = __riscv_vmerge_vvm_f64m1_tu(max, max, data_f64m1, greater_b64, max_vector_length);
362
+ max_indices = __riscv_vmerge_vvm_u64m1_tu(max_indices, max_indices, position_u64m1, greater_b64,
363
+ max_vector_length);
364
+ }
365
+ vfloat64m1_t id_max = __riscv_vfmv_v_f_f64m1(NK_F64_MAX, 1);
366
+ nk_f64_t mn = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredmin_vs_f64m1_f64m1(min, id_max, max_vector_length));
367
+ vfloat64m1_t id_min = __riscv_vfmv_v_f_f64m1(NK_F64_MIN, 1);
368
+ nk_f64_t mx = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredmax_vs_f64m1_f64m1(max, id_min, max_vector_length));
369
+ if (mn == NK_F64_MAX && mx == NK_F64_MIN) {
370
+ *min_value = NK_F64_MAX, *min_index = NK_SIZE_MAX, *max_value = NK_F64_MIN, *max_index = NK_SIZE_MAX;
371
+ return;
372
+ }
373
+ vbool64_t min_match_b64 = __riscv_vmfeq_vf_f64m1_b64(min, mn, max_vector_length);
374
+ vuint64m1_t sentinel = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
375
+ vuint64m1_t min_cands = __riscv_vmerge_vvm_u64m1(sentinel, min_indices, min_match_b64, max_vector_length);
376
+ vuint64m1_t id_umax = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
377
+ *min_value = mn, *min_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
378
+ __riscv_vredminu_vs_u64m1_u64m1(min_cands, id_umax, max_vector_length));
379
+ vbool64_t max_match_b64 = __riscv_vmfeq_vf_f64m1_b64(max, mx, max_vector_length);
380
+ vuint64m1_t max_cands = __riscv_vmerge_vvm_u64m1(sentinel, max_indices, max_match_b64, max_vector_length);
381
+ *max_value = mx, *max_index = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
382
+ __riscv_vredminu_vs_u64m1_u64m1(max_cands, id_umax, max_vector_length));
383
+ }
384
+
385
+ NK_PUBLIC void nk_reduce_minmax_f64_rvv( //
386
+ nk_f64_t const *data, nk_size_t count, nk_size_t stride_bytes, //
387
+ nk_f64_t *min_value, nk_size_t *min_index, //
388
+ nk_f64_t *max_value, nk_size_t *max_index) {
389
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_f64_t);
390
+ int aligned = (stride_bytes % sizeof(nk_f64_t) == 0);
391
+ if (count == 0)
392
+ *min_value = NK_F64_MAX, *min_index = NK_SIZE_MAX, *max_value = NK_F64_MIN, *max_index = NK_SIZE_MAX;
393
+ else if (!aligned)
394
+ nk_reduce_minmax_f64_serial(data, count, stride_bytes, min_value, min_index, max_value, max_index);
395
+ else if (stride_elements == 1)
396
+ nk_reduce_minmax_f64_rvv_contiguous_(data, count, min_value, min_index, max_value, max_index);
397
+ else nk_reduce_minmax_f64_rvv_strided_(data, count, stride_bytes, min_value, min_index, max_value, max_index);
398
+ }
399
+
400
+ NK_INTERNAL vuint8m1_t nk_fp8m1_to_comparable_u8m1_rvv_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
401
+ // Convert FP8 (e4m3/e5m2) to comparable unsigned form (sign bit 7)
402
+ // Positive (sign=0): XOR 0x80 → [0x80, 0xFF]
403
+ // Negative (sign=1): Bitwise NOT → [0x00, 0x7F]
404
+ vbool8_t is_negative_b8 = __riscv_vmsne_vx_u8m1_b8(__riscv_vand_vx_u8m1(raw_u8m1, 0x80, vector_length), 0,
405
+ vector_length);
406
+ vuint8m1_t flip_positive_u8m1 = __riscv_vxor_vx_u8m1(raw_u8m1, 0x80, vector_length);
407
+ vuint8m1_t flip_negative_u8m1 = __riscv_vnot_v_u8m1(raw_u8m1, vector_length);
408
+ return __riscv_vmerge_vvm_u8m1(flip_positive_u8m1, flip_negative_u8m1, is_negative_b8, vector_length);
409
+ }
410
+
411
+ NK_INTERNAL vuint8m1_t nk_comparable_to_fp8m1_rvv_(vuint8m1_t comparable_u8m1, nk_size_t vector_length) {
412
+ // Reverse: if >= 0x80 (was positive), XOR; else NOT
413
+ vbool8_t was_positive_b8 = __riscv_vmsgeu_vx_u8m1_b8(comparable_u8m1, 0x80, vector_length);
414
+ vuint8m1_t from_positive_u8m1 = __riscv_vxor_vx_u8m1(comparable_u8m1, 0x80, vector_length);
415
+ vuint8m1_t from_negative_u8m1 = __riscv_vnot_v_u8m1(comparable_u8m1, vector_length);
416
+ return __riscv_vmerge_vvm_u8m1(from_negative_u8m1, from_positive_u8m1, was_positive_b8, vector_length);
417
+ }
418
+
419
+ NK_INTERNAL vuint8m1_t nk_fp6m1_to_comparable_u8m1_rvv_(vuint8m1_t raw_u8m1, nk_size_t vector_length) {
420
+ // Convert FP6 (e2m3/e3m2) to comparable unsigned form (sign bit 5)
421
+ // Positive (sign=0): XOR 0x20 → [0x20, 0x3F]
422
+ // Negative (sign=1): XOR 0x3F (NOT lower 6 bits) → [0x00, 0x1F]
423
+ vbool8_t is_negative_b8 = __riscv_vmsne_vx_u8m1_b8(__riscv_vand_vx_u8m1(raw_u8m1, 0x20, vector_length), 0,
424
+ vector_length);
425
+ vuint8m1_t flip_positive_u8m1 = __riscv_vxor_vx_u8m1(raw_u8m1, 0x20, vector_length);
426
+ vuint8m1_t flip_negative_u8m1 = __riscv_vxor_vx_u8m1(raw_u8m1, 0x3F, vector_length);
427
+ return __riscv_vmerge_vvm_u8m1(flip_positive_u8m1, flip_negative_u8m1, is_negative_b8, vector_length);
428
+ }
429
+
430
+ NK_INTERNAL vuint8m1_t nk_comparable_to_fp6m1_rvv_(vuint8m1_t comparable_u8m1, nk_size_t vector_length) {
431
+ // Reverse: if >= 0x20 (was positive), XOR 0x20; else XOR 0x3F (NOT lower 6 bits)
432
+ vbool8_t was_positive_b8 = __riscv_vmsgeu_vx_u8m1_b8(comparable_u8m1, 0x20, vector_length);
433
+ vuint8m1_t from_positive_u8m1 = __riscv_vxor_vx_u8m1(comparable_u8m1, 0x20, vector_length);
434
+ vuint8m1_t from_negative_u8m1 = __riscv_vxor_vx_u8m1(comparable_u8m1, 0x3F, vector_length);
435
+ return __riscv_vmerge_vvm_u8m1(from_negative_u8m1, from_positive_u8m1, was_positive_b8, vector_length);
436
+ }
437
+
438
+ NK_INTERNAL void nk_reduce_moments_i8_rvv_contiguous_( //
439
+ nk_i8_t const *data_ptr, nk_size_t count, //
440
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
441
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
442
+ nk_size_t vlmax_elements = __riscv_vsetvlmax_e8m1();
443
+ vint64m4_t sum_i64m4 = __riscv_vmv_v_x_i64m4(0, max_vector_length);
444
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
445
+ vint8m1_t zero_i8m1 = __riscv_vmv_v_x_i8m1(0, vlmax_elements);
446
+
447
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
448
+ vector_length = __riscv_vsetvl_e8m1(count);
449
+ vint8m1_t data_i8m1 = __riscv_vle8_v_i8m1_tu(zero_i8m1, data_ptr, vector_length);
450
+
451
+ // Widen i8 → i16 → i32 → i64 for sum
452
+ vint16m2_t data_i16m2 = __riscv_vsext_vf2_i16m2(data_i8m1, vlmax_elements);
453
+ vint32m4_t data_i32m4 = __riscv_vsext_vf2_i32m4(data_i16m2, vlmax_elements);
454
+ vint64m8_t data_i64m8 = __riscv_vsext_vf2_i64m8(data_i32m4, vlmax_elements);
455
+
456
+ // Accumulate sum (split m8 into two m4)
457
+ sum_i64m4 = __riscv_vadd_vv_i64m4(sum_i64m4, __riscv_vget_v_i64m8_i64m4(data_i64m8, 0), vector_length);
458
+ sum_i64m4 = __riscv_vadd_vv_i64m4(sum_i64m4, __riscv_vget_v_i64m8_i64m4(data_i64m8, 1), vector_length);
459
+
460
+ // Sumsq: i8 × i8 → i16 (widening multiply)
461
+ vint16m2_t squares_i16m2 = __riscv_vwmul_vv_i16m2(data_i8m1, data_i8m1, vlmax_elements);
462
+ // Widen i16 → u32 → u64
463
+ vuint32m4_t squares_u32m4 = __riscv_vwcvtu_x_x_v_u32m4(__riscv_vreinterpret_v_i16m2_u16m2(squares_i16m2),
464
+ vlmax_elements);
465
+ vuint64m8_t squares_u64m8 = __riscv_vwcvtu_x_x_v_u64m8(squares_u32m4, vlmax_elements);
466
+
467
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 0), vector_length);
468
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 1), vector_length);
469
+ }
470
+
471
+ // Horizontal reduction
472
+ vint64m1_t zero_i64m1 = __riscv_vmv_v_x_i64m1(0, 1);
473
+ *sum_ptr = __riscv_vmv_x_s_i64m1_i64(__riscv_vredsum_vs_i64m4_i64m1(sum_i64m4, zero_i64m1, max_vector_length));
474
+
475
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
476
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
477
+ }
478
+
479
+ NK_INTERNAL void nk_reduce_moments_i8_rvv_strided_( //
480
+ nk_i8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
481
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
482
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
483
+ nk_size_t vlmax_elements = __riscv_vsetvlmax_e8m1();
484
+ vint64m4_t sum_i64m4 = __riscv_vmv_v_x_i64m4(0, max_vector_length);
485
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
486
+ vint8m1_t zero_i8m1 = __riscv_vmv_v_x_i8m1(0, vlmax_elements);
487
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
488
+
489
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
490
+ vector_length = __riscv_vsetvl_e8m1(count);
491
+ vint8m1_t data_i8m1 = __riscv_vlse8_v_i8m1_tu(zero_i8m1, (nk_i8_t const *)ptr, (nk_ssize_t)stride_bytes,
492
+ vector_length);
493
+
494
+ // Widen i8 → i16 → i32 → i64 for sum
495
+ vint16m2_t data_i16m2 = __riscv_vsext_vf2_i16m2(data_i8m1, vlmax_elements);
496
+ vint32m4_t data_i32m4 = __riscv_vsext_vf2_i32m4(data_i16m2, vlmax_elements);
497
+ vint64m8_t data_i64m8 = __riscv_vsext_vf2_i64m8(data_i32m4, vlmax_elements);
498
+
499
+ // Accumulate sum (split m8 into two m4)
500
+ sum_i64m4 = __riscv_vadd_vv_i64m4(sum_i64m4, __riscv_vget_v_i64m8_i64m4(data_i64m8, 0), vector_length);
501
+ sum_i64m4 = __riscv_vadd_vv_i64m4(sum_i64m4, __riscv_vget_v_i64m8_i64m4(data_i64m8, 1), vector_length);
502
+
503
+ // Sumsq: i8 × i8 → i16 (widening multiply)
504
+ vint16m2_t squares_i16m2 = __riscv_vwmul_vv_i16m2(data_i8m1, data_i8m1, vlmax_elements);
505
+ // Widen i16 → u32 → u64
506
+ vuint32m4_t squares_u32m4 = __riscv_vwcvtu_x_x_v_u32m4(__riscv_vreinterpret_v_i16m2_u16m2(squares_i16m2),
507
+ vlmax_elements);
508
+ vuint64m8_t squares_u64m8 = __riscv_vwcvtu_x_x_v_u64m8(squares_u32m4, vlmax_elements);
509
+
510
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 0), vector_length);
511
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 1), vector_length);
512
+ }
513
+
514
+ // Horizontal reduction
515
+ vint64m1_t zero_i64m1 = __riscv_vmv_v_x_i64m1(0, 1);
516
+ *sum_ptr = __riscv_vmv_x_s_i64m1_i64(__riscv_vredsum_vs_i64m4_i64m1(sum_i64m4, zero_i64m1, max_vector_length));
517
+
518
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
519
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
520
+ }
521
+
522
+ NK_PUBLIC void nk_reduce_moments_i8_rvv( //
523
+ nk_i8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
524
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
525
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i8_t);
526
+ int aligned = (stride_bytes % sizeof(nk_i8_t) == 0);
527
+
528
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
529
+ else if (!aligned) { nk_reduce_moments_i8_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
530
+ else if (stride_elements == 1) { nk_reduce_moments_i8_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
531
+ else { nk_reduce_moments_i8_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
532
+ }
533
+
534
+ NK_INTERNAL void nk_reduce_minmax_i8_rvv_contiguous_( //
535
+ nk_i8_t const *data_ptr, nk_size_t count, //
536
+ nk_i8_t *min_value_ptr, nk_size_t *min_index_ptr, //
537
+ nk_i8_t *max_value_ptr, nk_size_t *max_index_ptr) {
538
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
539
+ vint8m1_t min_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MAX, max_vector_length);
540
+ vint8m1_t max_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MIN, max_vector_length);
541
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
542
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
543
+
544
+ nk_size_t offset = 0;
545
+ for (nk_size_t vector_length; count > 0;
546
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
547
+ vector_length = __riscv_vsetvl_e8m1(count);
548
+ vint8m1_t data_i8m1 = __riscv_vle8_v_i8m1(data_ptr, vector_length);
549
+
550
+ // VID-based absolute indices
551
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
552
+ vector_length);
553
+
554
+ vbool8_t less_b8 = __riscv_vmslt_vv_i8m1_b8(data_i8m1, min_i8m1, vector_length);
555
+ min_i8m1 = __riscv_vmerge_vvm_i8m1_tu(min_i8m1, min_i8m1, data_i8m1, less_b8, vector_length);
556
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
557
+ vector_length);
558
+
559
+ vbool8_t greater_b8 = __riscv_vmslt_vv_i8m1_b8(max_i8m1, data_i8m1, vector_length);
560
+ max_i8m1 = __riscv_vmerge_vvm_i8m1_tu(max_i8m1, max_i8m1, data_i8m1, greater_b8, vector_length);
561
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
562
+ vector_length);
563
+ }
564
+
565
+ // Horizontal reduction for min
566
+ vint8m1_t init_max_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MAX, 1);
567
+ nk_i8_t min_val = __riscv_vmv_x_s_i8m1_i8(__riscv_vredmin_vs_i8m1_i8m1(min_i8m1, init_max_i8m1, max_vector_length));
568
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_i8m1_b8(min_i8m1, min_val, max_vector_length);
569
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
570
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
571
+ max_vector_length);
572
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
573
+ *min_value_ptr = min_val;
574
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
575
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
576
+
577
+ // Horizontal reduction for max
578
+ vint8m1_t init_min_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MIN, 1);
579
+ nk_i8_t max_val = __riscv_vmv_x_s_i8m1_i8(__riscv_vredmax_vs_i8m1_i8m1(max_i8m1, init_min_i8m1, max_vector_length));
580
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_i8m1_b8(max_i8m1, max_val, max_vector_length);
581
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
582
+ max_vector_length);
583
+ *max_value_ptr = max_val;
584
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
585
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
586
+ }
587
+
588
+ NK_INTERNAL void nk_reduce_minmax_i8_rvv_strided_( //
589
+ nk_i8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
590
+ nk_i8_t *min_value_ptr, nk_size_t *min_index_ptr, //
591
+ nk_i8_t *max_value_ptr, nk_size_t *max_index_ptr) {
592
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
593
+ vint8m1_t min_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MAX, max_vector_length);
594
+ vint8m1_t max_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MIN, max_vector_length);
595
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
596
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
597
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
598
+
599
+ nk_size_t offset = 0;
600
+ for (nk_size_t vector_length; count > 0;
601
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
602
+ vector_length = __riscv_vsetvl_e8m1(count);
603
+ vint8m1_t data_i8m1 = __riscv_vlse8_v_i8m1((nk_i8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
604
+
605
+ // VID-based absolute indices
606
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
607
+ vector_length);
608
+
609
+ vbool8_t less_b8 = __riscv_vmslt_vv_i8m1_b8(data_i8m1, min_i8m1, vector_length);
610
+ min_i8m1 = __riscv_vmerge_vvm_i8m1_tu(min_i8m1, min_i8m1, data_i8m1, less_b8, vector_length);
611
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
612
+ vector_length);
613
+
614
+ vbool8_t greater_b8 = __riscv_vmslt_vv_i8m1_b8(max_i8m1, data_i8m1, vector_length);
615
+ max_i8m1 = __riscv_vmerge_vvm_i8m1_tu(max_i8m1, max_i8m1, data_i8m1, greater_b8, vector_length);
616
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
617
+ vector_length);
618
+ }
619
+
620
+ // Horizontal reduction for min
621
+ vint8m1_t init_max_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MAX, 1);
622
+ nk_i8_t min_val = __riscv_vmv_x_s_i8m1_i8(__riscv_vredmin_vs_i8m1_i8m1(min_i8m1, init_max_i8m1, max_vector_length));
623
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_i8m1_b8(min_i8m1, min_val, max_vector_length);
624
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
625
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
626
+ max_vector_length);
627
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
628
+ *min_value_ptr = min_val;
629
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
630
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
631
+
632
+ // Horizontal reduction for max
633
+ vint8m1_t init_min_i8m1 = __riscv_vmv_v_x_i8m1(NK_I8_MIN, 1);
634
+ nk_i8_t max_val = __riscv_vmv_x_s_i8m1_i8(__riscv_vredmax_vs_i8m1_i8m1(max_i8m1, init_min_i8m1, max_vector_length));
635
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_i8m1_b8(max_i8m1, max_val, max_vector_length);
636
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
637
+ max_vector_length);
638
+ *max_value_ptr = max_val;
639
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
640
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
641
+ }
642
+
643
+ NK_PUBLIC void nk_reduce_minmax_i8_rvv( //
644
+ nk_i8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
645
+ nk_i8_t *min_value_ptr, nk_size_t *min_index_ptr, //
646
+ nk_i8_t *max_value_ptr, nk_size_t *max_index_ptr) {
647
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i8_t);
648
+ int aligned = (stride_bytes % sizeof(nk_i8_t) == 0);
649
+
650
+ if (count == 0)
651
+ *min_value_ptr = NK_I8_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_I8_MIN,
652
+ *max_index_ptr = NK_SIZE_MAX;
653
+ else if (!aligned)
654
+ nk_reduce_minmax_i8_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
655
+ max_index_ptr);
656
+ else if (stride_elements == 1)
657
+ nk_reduce_minmax_i8_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
658
+ max_index_ptr);
659
+ else
660
+ nk_reduce_minmax_i8_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
661
+ max_index_ptr);
662
+ }
663
+
664
+ NK_INTERNAL void nk_reduce_moments_u8_rvv_contiguous_( //
665
+ nk_u8_t const *data_ptr, nk_size_t count, //
666
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
667
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
668
+ nk_size_t vlmax_elements = __riscv_vsetvlmax_e8m1();
669
+ vuint64m4_t sum_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
670
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
671
+ vuint8m1_t zero_u8m1 = __riscv_vmv_v_x_u8m1(0, vlmax_elements);
672
+
673
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
674
+ vector_length = __riscv_vsetvl_e8m1(count);
675
+ vuint8m1_t data_u8m1 = __riscv_vle8_v_u8m1_tu(zero_u8m1, data_ptr, vector_length);
676
+
677
+ // Widen u8 → u16 → u32 → u64 for sum
678
+ vuint16m2_t data_u16m2 = __riscv_vzext_vf2_u16m2(data_u8m1, vlmax_elements);
679
+ vuint32m4_t data_u32m4 = __riscv_vzext_vf2_u32m4(data_u16m2, vlmax_elements);
680
+ vuint64m8_t data_u64m8 = __riscv_vzext_vf2_u64m8(data_u32m4, vlmax_elements);
681
+
682
+ // Accumulate sum (split m8 into two m4)
683
+ sum_u64m4 = __riscv_vadd_vv_u64m4(sum_u64m4, __riscv_vget_v_u64m8_u64m4(data_u64m8, 0), vector_length);
684
+ sum_u64m4 = __riscv_vadd_vv_u64m4(sum_u64m4, __riscv_vget_v_u64m8_u64m4(data_u64m8, 1), vector_length);
685
+
686
+ // Sumsq: u8 × u8 → u16 (widening multiply)
687
+ vuint16m2_t squares_u16m2 = __riscv_vwmulu_vv_u16m2(data_u8m1, data_u8m1, vlmax_elements);
688
+ // Widen u16 → u32 → u64
689
+ vuint32m4_t squares_u32m4 = __riscv_vzext_vf2_u32m4(squares_u16m2, vlmax_elements);
690
+ vuint64m8_t squares_u64m8 = __riscv_vzext_vf2_u64m8(squares_u32m4, vlmax_elements);
691
+
692
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 0), vector_length);
693
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 1), vector_length);
694
+ }
695
+
696
+ // Horizontal reduction
697
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
698
+ *sum_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sum_u64m4, zero_u64m1, max_vector_length)),
699
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
700
+ }
701
+
702
+ NK_INTERNAL void nk_reduce_moments_u8_rvv_strided_( //
703
+ nk_u8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
704
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
705
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
706
+ nk_size_t vlmax_elements = __riscv_vsetvlmax_e8m1();
707
+ vuint64m4_t sum_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
708
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
709
+ vuint8m1_t zero_u8m1 = __riscv_vmv_v_x_u8m1(0, vlmax_elements);
710
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
711
+
712
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
713
+ vector_length = __riscv_vsetvl_e8m1(count);
714
+ vuint8m1_t data_u8m1 = __riscv_vlse8_v_u8m1_tu(zero_u8m1, (nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes,
715
+ vector_length);
716
+
717
+ // Widen u8 → u16 → u32 → u64 for sum
718
+ vuint16m2_t data_u16m2 = __riscv_vzext_vf2_u16m2(data_u8m1, vlmax_elements);
719
+ vuint32m4_t data_u32m4 = __riscv_vzext_vf2_u32m4(data_u16m2, vlmax_elements);
720
+ vuint64m8_t data_u64m8 = __riscv_vzext_vf2_u64m8(data_u32m4, vlmax_elements);
721
+
722
+ // Accumulate sum (split m8 into two m4)
723
+ sum_u64m4 = __riscv_vadd_vv_u64m4(sum_u64m4, __riscv_vget_v_u64m8_u64m4(data_u64m8, 0), vector_length);
724
+ sum_u64m4 = __riscv_vadd_vv_u64m4(sum_u64m4, __riscv_vget_v_u64m8_u64m4(data_u64m8, 1), vector_length);
725
+
726
+ // Sumsq: u8 × u8 → u16 (widening multiply)
727
+ vuint16m2_t squares_u16m2 = __riscv_vwmulu_vv_u16m2(data_u8m1, data_u8m1, vlmax_elements);
728
+ // Widen u16 → u32 → u64
729
+ vuint32m4_t squares_u32m4 = __riscv_vzext_vf2_u32m4(squares_u16m2, vlmax_elements);
730
+ vuint64m8_t squares_u64m8 = __riscv_vzext_vf2_u64m8(squares_u32m4, vlmax_elements);
731
+
732
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 0), vector_length);
733
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4(sumsq_u64m4, __riscv_vget_v_u64m8_u64m4(squares_u64m8, 1), vector_length);
734
+ }
735
+
736
+ // Horizontal reduction
737
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
738
+ *sum_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sum_u64m4, zero_u64m1, max_vector_length)),
739
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
740
+ }
741
+
742
+ NK_PUBLIC void nk_reduce_moments_u8_rvv( //
743
+ nk_u8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
744
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
745
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u8_t);
746
+ int aligned = (stride_bytes % sizeof(nk_u8_t) == 0);
747
+
748
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
749
+ else if (!aligned) { nk_reduce_moments_u8_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
750
+ else if (stride_elements == 1) { nk_reduce_moments_u8_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
751
+ else { nk_reduce_moments_u8_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
752
+ }
753
+
754
+ NK_INTERNAL void nk_reduce_minmax_u8_rvv_contiguous_( //
755
+ nk_u8_t const *data_ptr, nk_size_t count, //
756
+ nk_u8_t *min_value_ptr, nk_size_t *min_index_ptr, //
757
+ nk_u8_t *max_value_ptr, nk_size_t *max_index_ptr) {
758
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
759
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MAX, max_vector_length);
760
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MIN, max_vector_length);
761
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
762
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
763
+
764
+ nk_size_t offset = 0;
765
+ for (nk_size_t vector_length; count > 0;
766
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
767
+ vector_length = __riscv_vsetvl_e8m1(count);
768
+ vuint8m1_t data_u8m1 = __riscv_vle8_v_u8m1(data_ptr, vector_length);
769
+
770
+ // VID-based absolute indices
771
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
772
+ vector_length);
773
+
774
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(data_u8m1, min_u8m1, vector_length);
775
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, data_u8m1, less_b8, vector_length);
776
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
777
+ vector_length);
778
+
779
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, data_u8m1, vector_length);
780
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, data_u8m1, greater_b8, vector_length);
781
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
782
+ vector_length);
783
+ }
784
+
785
+ // Horizontal reduction for min
786
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MAX, 1);
787
+ nk_u8_t min_val = __riscv_vmv_x_s_u8m1_u8(
788
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
789
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_val, max_vector_length);
790
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
791
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
792
+ max_vector_length);
793
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
794
+ *min_value_ptr = min_val;
795
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
796
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
797
+
798
+ // Horizontal reduction for max
799
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MIN, 1);
800
+ nk_u8_t max_val = __riscv_vmv_x_s_u8m1_u8(
801
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
802
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_val, max_vector_length);
803
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
804
+ max_vector_length);
805
+ *max_value_ptr = max_val;
806
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
807
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
808
+ }
809
+
810
+ NK_INTERNAL void nk_reduce_minmax_u8_rvv_strided_( //
811
+ nk_u8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
812
+ nk_u8_t *min_value_ptr, nk_size_t *min_index_ptr, //
813
+ nk_u8_t *max_value_ptr, nk_size_t *max_index_ptr) {
814
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
815
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MAX, max_vector_length);
816
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MIN, max_vector_length);
817
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
818
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
819
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
820
+
821
+ nk_size_t offset = 0;
822
+ for (nk_size_t vector_length; count > 0;
823
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
824
+ vector_length = __riscv_vsetvl_e8m1(count);
825
+ vuint8m1_t data_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
826
+
827
+ // VID-based absolute indices
828
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
829
+ vector_length);
830
+
831
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(data_u8m1, min_u8m1, vector_length);
832
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, data_u8m1, less_b8, vector_length);
833
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
834
+ vector_length);
835
+
836
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, data_u8m1, vector_length);
837
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, data_u8m1, greater_b8, vector_length);
838
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
839
+ vector_length);
840
+ }
841
+
842
+ // Horizontal reduction for min
843
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MAX, 1);
844
+ nk_u8_t min_val = __riscv_vmv_x_s_u8m1_u8(
845
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
846
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_val, max_vector_length);
847
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
848
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
849
+ max_vector_length);
850
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
851
+ *min_value_ptr = min_val;
852
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
853
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
854
+
855
+ // Horizontal reduction for max
856
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(NK_U8_MIN, 1);
857
+ nk_u8_t max_val = __riscv_vmv_x_s_u8m1_u8(
858
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
859
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_val, max_vector_length);
860
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
861
+ max_vector_length);
862
+ *max_value_ptr = max_val;
863
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
864
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
865
+ }
866
+
867
+ NK_PUBLIC void nk_reduce_minmax_u8_rvv( //
868
+ nk_u8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
869
+ nk_u8_t *min_value_ptr, nk_size_t *min_index_ptr, //
870
+ nk_u8_t *max_value_ptr, nk_size_t *max_index_ptr) {
871
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u8_t);
872
+ int aligned = (stride_bytes % sizeof(nk_u8_t) == 0);
873
+
874
+ if (count == 0)
875
+ *min_value_ptr = NK_U8_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_U8_MIN,
876
+ *max_index_ptr = NK_SIZE_MAX;
877
+ else if (!aligned)
878
+ nk_reduce_minmax_u8_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
879
+ max_index_ptr);
880
+ else if (stride_elements == 1)
881
+ nk_reduce_minmax_u8_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
882
+ max_index_ptr);
883
+ else
884
+ nk_reduce_minmax_u8_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
885
+ max_index_ptr);
886
+ }
887
+
888
+ NK_INTERNAL void nk_reduce_moments_i16_rvv_contiguous_( //
889
+ nk_i16_t const *data_ptr, nk_size_t count, //
890
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
891
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
892
+ vint64m4_t sum_i64m4 = __riscv_vmv_v_x_i64m4(0, max_vector_length);
893
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
894
+
895
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
896
+ vector_length = __riscv_vsetvl_e16m1(count);
897
+ vint16m1_t data_i16m1 = __riscv_vle16_v_i16m1(data_ptr, vector_length);
898
+
899
+ // Widen i16 → i32 → i64 for sum
900
+ vint32m2_t data_i32m2 = __riscv_vsext_vf2_i32m2(data_i16m1, vector_length);
901
+ vint64m4_t data_i64m4 = __riscv_vsext_vf2_i64m4(data_i32m2, vector_length);
902
+ sum_i64m4 = __riscv_vadd_vv_i64m4_tu(sum_i64m4, sum_i64m4, data_i64m4, vector_length);
903
+
904
+ // Sumsq: i16 × i16 → i32 (widening multiply)
905
+ vint32m2_t squares_i32m2 = __riscv_vwmul_vv_i32m2(data_i16m1, data_i16m1, vector_length);
906
+ // Widen i32 → u64
907
+ vuint64m4_t squares_u64m4 = __riscv_vwcvtu_x_x_v_u64m4(__riscv_vreinterpret_v_i32m2_u32m2(squares_i32m2),
908
+ vector_length);
909
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4_tu(sumsq_u64m4, sumsq_u64m4, squares_u64m4, vector_length);
910
+ }
911
+
912
+ // Horizontal reduction
913
+ vint64m1_t zero_i64m1 = __riscv_vmv_v_x_i64m1(0, 1);
914
+ *sum_ptr = __riscv_vmv_x_s_i64m1_i64(__riscv_vredsum_vs_i64m4_i64m1(sum_i64m4, zero_i64m1, max_vector_length));
915
+
916
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
917
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
918
+ }
919
+
920
+ NK_INTERNAL void nk_reduce_moments_i16_rvv_strided_( //
921
+ nk_i16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
922
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
923
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
924
+ vint64m4_t sum_i64m4 = __riscv_vmv_v_x_i64m4(0, max_vector_length);
925
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
926
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
927
+
928
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
929
+ vector_length = __riscv_vsetvl_e16m1(count);
930
+ vint16m1_t data_i16m1 = __riscv_vlse16_v_i16m1((nk_i16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
931
+
932
+ // Widen i16 → i32 → i64 for sum
933
+ vint32m2_t data_i32m2 = __riscv_vsext_vf2_i32m2(data_i16m1, vector_length);
934
+ vint64m4_t data_i64m4 = __riscv_vsext_vf2_i64m4(data_i32m2, vector_length);
935
+ sum_i64m4 = __riscv_vadd_vv_i64m4_tu(sum_i64m4, sum_i64m4, data_i64m4, vector_length);
936
+
937
+ // Sumsq: i16 × i16 → i32 (widening multiply)
938
+ vint32m2_t squares_i32m2 = __riscv_vwmul_vv_i32m2(data_i16m1, data_i16m1, vector_length);
939
+ // Widen i32 → u64
940
+ vuint64m4_t squares_u64m4 = __riscv_vwcvtu_x_x_v_u64m4(__riscv_vreinterpret_v_i32m2_u32m2(squares_i32m2),
941
+ vector_length);
942
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4_tu(sumsq_u64m4, sumsq_u64m4, squares_u64m4, vector_length);
943
+ }
944
+
945
+ // Horizontal reduction
946
+ vint64m1_t zero_i64m1 = __riscv_vmv_v_x_i64m1(0, 1);
947
+ *sum_ptr = __riscv_vmv_x_s_i64m1_i64(__riscv_vredsum_vs_i64m4_i64m1(sum_i64m4, zero_i64m1, max_vector_length));
948
+
949
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
950
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
951
+ }
952
+
953
+ NK_PUBLIC void nk_reduce_moments_i16_rvv( //
954
+ nk_i16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
955
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
956
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i16_t);
957
+ int aligned = (stride_bytes % sizeof(nk_i16_t) == 0);
958
+
959
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
960
+ else if (!aligned) { nk_reduce_moments_i16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
961
+ else if (stride_elements == 1) { nk_reduce_moments_i16_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
962
+ else { nk_reduce_moments_i16_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
963
+ }
964
+
965
+ NK_INTERNAL void nk_reduce_minmax_i16_rvv_contiguous_( //
966
+ nk_i16_t const *data_ptr, nk_size_t count, //
967
+ nk_i16_t *min_value_ptr, nk_size_t *min_index_ptr, //
968
+ nk_i16_t *max_value_ptr, nk_size_t *max_index_ptr) {
969
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
970
+ vint16m1_t min_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MAX, max_vector_length);
971
+ vint16m1_t max_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MIN, max_vector_length);
972
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
973
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
974
+
975
+ nk_size_t offset = 0;
976
+ for (nk_size_t vector_length; count > 0;
977
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
978
+ vector_length = __riscv_vsetvl_e16m1(count);
979
+ vint16m1_t data_i16m1 = __riscv_vle16_v_i16m1(data_ptr, vector_length);
980
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
981
+ vector_length);
982
+
983
+ vbool16_t less_b16 = __riscv_vmslt_vv_i16m1_b16(data_i16m1, min_i16m1, vector_length);
984
+ min_i16m1 = __riscv_vmerge_vvm_i16m1_tu(min_i16m1, min_i16m1, data_i16m1, less_b16, vector_length);
985
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
986
+ vector_length);
987
+
988
+ vbool16_t greater_b16 = __riscv_vmslt_vv_i16m1_b16(max_i16m1, data_i16m1, vector_length);
989
+ max_i16m1 = __riscv_vmerge_vvm_i16m1_tu(max_i16m1, max_i16m1, data_i16m1, greater_b16, vector_length);
990
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
991
+ vector_length);
992
+ }
993
+
994
+ // Horizontal reduction for min
995
+ vint16m1_t init_max_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MAX, 1);
996
+ nk_i16_t min_val = __riscv_vmv_x_s_i16m1_i16(
997
+ __riscv_vredmin_vs_i16m1_i16m1(min_i16m1, init_max_i16m1, max_vector_length));
998
+ vbool16_t min_match_b16 = __riscv_vmseq_vx_i16m1_b16(min_i16m1, min_val, max_vector_length);
999
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
1000
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
1001
+ max_vector_length);
1002
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1003
+ *min_value_ptr = min_val;
1004
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1005
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
1006
+
1007
+ // Horizontal reduction for max
1008
+ vint16m1_t init_min_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MIN, 1);
1009
+ nk_i16_t max_val = __riscv_vmv_x_s_i16m1_i16(
1010
+ __riscv_vredmax_vs_i16m1_i16m1(max_i16m1, init_min_i16m1, max_vector_length));
1011
+ vbool16_t max_match_b16 = __riscv_vmseq_vx_i16m1_b16(max_i16m1, max_val, max_vector_length);
1012
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
1013
+ max_vector_length);
1014
+ *max_value_ptr = max_val;
1015
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1016
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
1017
+ }
1018
+
1019
+ NK_INTERNAL void nk_reduce_minmax_i16_rvv_strided_( //
1020
+ nk_i16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1021
+ nk_i16_t *min_value_ptr, nk_size_t *min_index_ptr, //
1022
+ nk_i16_t *max_value_ptr, nk_size_t *max_index_ptr) {
1023
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
1024
+ vint16m1_t min_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MAX, max_vector_length);
1025
+ vint16m1_t max_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MIN, max_vector_length);
1026
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1027
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1028
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1029
+
1030
+ nk_size_t offset = 0;
1031
+ for (nk_size_t vector_length; count > 0;
1032
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
1033
+ vector_length = __riscv_vsetvl_e16m1(count);
1034
+ vint16m1_t data_i16m1 = __riscv_vlse16_v_i16m1((nk_i16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1035
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
1036
+ vector_length);
1037
+
1038
+ vbool16_t less_b16 = __riscv_vmslt_vv_i16m1_b16(data_i16m1, min_i16m1, vector_length);
1039
+ min_i16m1 = __riscv_vmerge_vvm_i16m1_tu(min_i16m1, min_i16m1, data_i16m1, less_b16, vector_length);
1040
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
1041
+ vector_length);
1042
+
1043
+ vbool16_t greater_b16 = __riscv_vmslt_vv_i16m1_b16(max_i16m1, data_i16m1, vector_length);
1044
+ max_i16m1 = __riscv_vmerge_vvm_i16m1_tu(max_i16m1, max_i16m1, data_i16m1, greater_b16, vector_length);
1045
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
1046
+ vector_length);
1047
+ }
1048
+
1049
+ // Horizontal reduction for min
1050
+ vint16m1_t init_max_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MAX, 1);
1051
+ nk_i16_t min_val = __riscv_vmv_x_s_i16m1_i16(
1052
+ __riscv_vredmin_vs_i16m1_i16m1(min_i16m1, init_max_i16m1, max_vector_length));
1053
+ vbool16_t min_match_b16 = __riscv_vmseq_vx_i16m1_b16(min_i16m1, min_val, max_vector_length);
1054
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
1055
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
1056
+ max_vector_length);
1057
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1058
+ *min_value_ptr = min_val;
1059
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1060
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
1061
+
1062
+ // Horizontal reduction for max
1063
+ vint16m1_t init_min_i16m1 = __riscv_vmv_v_x_i16m1(NK_I16_MIN, 1);
1064
+ nk_i16_t max_val = __riscv_vmv_x_s_i16m1_i16(
1065
+ __riscv_vredmax_vs_i16m1_i16m1(max_i16m1, init_min_i16m1, max_vector_length));
1066
+ vbool16_t max_match_b16 = __riscv_vmseq_vx_i16m1_b16(max_i16m1, max_val, max_vector_length);
1067
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
1068
+ max_vector_length);
1069
+ *max_value_ptr = max_val;
1070
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1071
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
1072
+ }
1073
+
1074
+ NK_PUBLIC void nk_reduce_minmax_i16_rvv( //
1075
+ nk_i16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1076
+ nk_i16_t *min_value_ptr, nk_size_t *min_index_ptr, //
1077
+ nk_i16_t *max_value_ptr, nk_size_t *max_index_ptr) {
1078
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i16_t);
1079
+ int aligned = (stride_bytes % sizeof(nk_i16_t) == 0);
1080
+
1081
+ if (count == 0)
1082
+ *min_value_ptr = NK_I16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_I16_MIN,
1083
+ *max_index_ptr = NK_SIZE_MAX;
1084
+ else if (!aligned)
1085
+ nk_reduce_minmax_i16_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1086
+ max_index_ptr);
1087
+ else if (stride_elements == 1)
1088
+ nk_reduce_minmax_i16_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
1089
+ max_index_ptr);
1090
+ else
1091
+ nk_reduce_minmax_i16_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1092
+ max_index_ptr);
1093
+ }
1094
+
1095
+ NK_INTERNAL void nk_reduce_moments_u16_rvv_contiguous_( //
1096
+ nk_u16_t const *data_ptr, nk_size_t count, //
1097
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1098
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
1099
+ vuint64m4_t sum_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1100
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1101
+
1102
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
1103
+ vector_length = __riscv_vsetvl_e16m1(count);
1104
+ vuint16m1_t data_u16m1 = __riscv_vle16_v_u16m1(data_ptr, vector_length);
1105
+
1106
+ // Widen u16 → u32 → u64 for sum
1107
+ vuint32m2_t data_u32m2 = __riscv_vzext_vf2_u32m2(data_u16m1, vector_length);
1108
+ vuint64m4_t data_u64m4 = __riscv_vzext_vf2_u64m4(data_u32m2, vector_length);
1109
+ sum_u64m4 = __riscv_vadd_vv_u64m4_tu(sum_u64m4, sum_u64m4, data_u64m4, vector_length);
1110
+
1111
+ // Sumsq: u16 × u16 → u32 (widening multiply)
1112
+ vuint32m2_t squares_u32m2 = __riscv_vwmulu_vv_u32m2(data_u16m1, data_u16m1, vector_length);
1113
+ // Widen u32 → u64
1114
+ vuint64m4_t squares_u64m4 = __riscv_vzext_vf2_u64m4(squares_u32m2, vector_length);
1115
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4_tu(sumsq_u64m4, sumsq_u64m4, squares_u64m4, vector_length);
1116
+ }
1117
+
1118
+ // Horizontal reduction
1119
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
1120
+ *sum_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sum_u64m4, zero_u64m1, max_vector_length)),
1121
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
1122
+ }
1123
+
1124
+ NK_INTERNAL void nk_reduce_moments_u16_rvv_strided_( //
1125
+ nk_u16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1126
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1127
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
1128
+ vuint64m4_t sum_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1129
+ vuint64m4_t sumsq_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1130
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1131
+
1132
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
1133
+ vector_length = __riscv_vsetvl_e16m1(count);
1134
+ vuint16m1_t data_u16m1 = __riscv_vlse16_v_u16m1((nk_u16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1135
+
1136
+ // Widen u16 → u32 → u64 for sum
1137
+ vuint32m2_t data_u32m2 = __riscv_vzext_vf2_u32m2(data_u16m1, vector_length);
1138
+ vuint64m4_t data_u64m4 = __riscv_vzext_vf2_u64m4(data_u32m2, vector_length);
1139
+ sum_u64m4 = __riscv_vadd_vv_u64m4_tu(sum_u64m4, sum_u64m4, data_u64m4, vector_length);
1140
+
1141
+ // Sumsq: u16 × u16 → u32 (widening multiply)
1142
+ vuint32m2_t squares_u32m2 = __riscv_vwmulu_vv_u32m2(data_u16m1, data_u16m1, vector_length);
1143
+ // Widen u32 → u64
1144
+ vuint64m4_t squares_u64m4 = __riscv_vzext_vf2_u64m4(squares_u32m2, vector_length);
1145
+ sumsq_u64m4 = __riscv_vadd_vv_u64m4_tu(sumsq_u64m4, sumsq_u64m4, squares_u64m4, vector_length);
1146
+ }
1147
+
1148
+ // Horizontal reduction
1149
+ vuint64m1_t zero_u64m1 = __riscv_vmv_v_x_u64m1(0, 1);
1150
+ *sum_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sum_u64m4, zero_u64m1, max_vector_length)),
1151
+ *sumsq_ptr = __riscv_vmv_x_s_u64m1_u64(__riscv_vredsum_vs_u64m4_u64m1(sumsq_u64m4, zero_u64m1, max_vector_length));
1152
+ }
1153
+
1154
+ NK_PUBLIC void nk_reduce_moments_u16_rvv( //
1155
+ nk_u16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1156
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1157
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u16_t);
1158
+ int aligned = (stride_bytes % sizeof(nk_u16_t) == 0);
1159
+
1160
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
1161
+ else if (!aligned) { nk_reduce_moments_u16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1162
+ else if (stride_elements == 1) { nk_reduce_moments_u16_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
1163
+ else { nk_reduce_moments_u16_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1164
+ }
1165
+
1166
+ NK_INTERNAL void nk_reduce_minmax_u16_rvv_contiguous_( //
1167
+ nk_u16_t const *data_ptr, nk_size_t count, //
1168
+ nk_u16_t *min_value_ptr, nk_size_t *min_index_ptr, //
1169
+ nk_u16_t *max_value_ptr, nk_size_t *max_index_ptr) {
1170
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
1171
+ vuint16m1_t min_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MAX, max_vector_length);
1172
+ vuint16m1_t max_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MIN, max_vector_length);
1173
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1174
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1175
+
1176
+ nk_size_t offset = 0;
1177
+ for (nk_size_t vector_length; count > 0;
1178
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
1179
+ vector_length = __riscv_vsetvl_e16m1(count);
1180
+ vuint16m1_t data_u16m1 = __riscv_vle16_v_u16m1(data_ptr, vector_length);
1181
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
1182
+ vector_length);
1183
+
1184
+ vbool16_t less_b16 = __riscv_vmsltu_vv_u16m1_b16(data_u16m1, min_u16m1, vector_length);
1185
+ min_u16m1 = __riscv_vmerge_vvm_u16m1_tu(min_u16m1, min_u16m1, data_u16m1, less_b16, vector_length);
1186
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
1187
+ vector_length);
1188
+
1189
+ vbool16_t greater_b16 = __riscv_vmsltu_vv_u16m1_b16(max_u16m1, data_u16m1, vector_length);
1190
+ max_u16m1 = __riscv_vmerge_vvm_u16m1_tu(max_u16m1, max_u16m1, data_u16m1, greater_b16, vector_length);
1191
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
1192
+ vector_length);
1193
+ }
1194
+
1195
+ // Horizontal reduction for min
1196
+ vuint16m1_t init_max_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MAX, 1);
1197
+ nk_u16_t min_val = __riscv_vmv_x_s_u16m1_u16(
1198
+ __riscv_vredminu_vs_u16m1_u16m1(min_u16m1, init_max_u16m1, max_vector_length));
1199
+ vbool16_t min_match_b16 = __riscv_vmseq_vx_u16m1_b16(min_u16m1, min_val, max_vector_length);
1200
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
1201
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
1202
+ max_vector_length);
1203
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1204
+ *min_value_ptr = min_val;
1205
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1206
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
1207
+
1208
+ // Horizontal reduction for max
1209
+ vuint16m1_t init_min_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MIN, 1);
1210
+ nk_u16_t max_val = __riscv_vmv_x_s_u16m1_u16(
1211
+ __riscv_vredmaxu_vs_u16m1_u16m1(max_u16m1, init_min_u16m1, max_vector_length));
1212
+ vbool16_t max_match_b16 = __riscv_vmseq_vx_u16m1_b16(max_u16m1, max_val, max_vector_length);
1213
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
1214
+ max_vector_length);
1215
+ *max_value_ptr = max_val;
1216
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1217
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
1218
+ }
1219
+
1220
+ NK_INTERNAL void nk_reduce_minmax_u16_rvv_strided_( //
1221
+ nk_u16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1222
+ nk_u16_t *min_value_ptr, nk_size_t *min_index_ptr, //
1223
+ nk_u16_t *max_value_ptr, nk_size_t *max_index_ptr) {
1224
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
1225
+ vuint16m1_t min_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MAX, max_vector_length);
1226
+ vuint16m1_t max_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MIN, max_vector_length);
1227
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1228
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
1229
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1230
+
1231
+ nk_size_t offset = 0;
1232
+ for (nk_size_t vector_length; count > 0;
1233
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
1234
+ vector_length = __riscv_vsetvl_e16m1(count);
1235
+ vuint16m1_t data_u16m1 = __riscv_vlse16_v_u16m1((nk_u16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1236
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
1237
+ vector_length);
1238
+
1239
+ vbool16_t less_b16 = __riscv_vmsltu_vv_u16m1_b16(data_u16m1, min_u16m1, vector_length);
1240
+ min_u16m1 = __riscv_vmerge_vvm_u16m1_tu(min_u16m1, min_u16m1, data_u16m1, less_b16, vector_length);
1241
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
1242
+ vector_length);
1243
+
1244
+ vbool16_t greater_b16 = __riscv_vmsltu_vv_u16m1_b16(max_u16m1, data_u16m1, vector_length);
1245
+ max_u16m1 = __riscv_vmerge_vvm_u16m1_tu(max_u16m1, max_u16m1, data_u16m1, greater_b16, vector_length);
1246
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
1247
+ vector_length);
1248
+ }
1249
+
1250
+ // Horizontal reduction for min
1251
+ vuint16m1_t init_max_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MAX, 1);
1252
+ nk_u16_t min_val = __riscv_vmv_x_s_u16m1_u16(
1253
+ __riscv_vredminu_vs_u16m1_u16m1(min_u16m1, init_max_u16m1, max_vector_length));
1254
+ vbool16_t min_match_b16 = __riscv_vmseq_vx_u16m1_b16(min_u16m1, min_val, max_vector_length);
1255
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
1256
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
1257
+ max_vector_length);
1258
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1259
+ *min_value_ptr = min_val;
1260
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1261
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
1262
+
1263
+ // Horizontal reduction for max
1264
+ vuint16m1_t init_min_u16m1 = __riscv_vmv_v_x_u16m1(NK_U16_MIN, 1);
1265
+ nk_u16_t max_val = __riscv_vmv_x_s_u16m1_u16(
1266
+ __riscv_vredmaxu_vs_u16m1_u16m1(max_u16m1, init_min_u16m1, max_vector_length));
1267
+ vbool16_t max_match_b16 = __riscv_vmseq_vx_u16m1_b16(max_u16m1, max_val, max_vector_length);
1268
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
1269
+ max_vector_length);
1270
+ *max_value_ptr = max_val;
1271
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1272
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
1273
+ }
1274
+
1275
+ NK_PUBLIC void nk_reduce_minmax_u16_rvv( //
1276
+ nk_u16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1277
+ nk_u16_t *min_value_ptr, nk_size_t *min_index_ptr, //
1278
+ nk_u16_t *max_value_ptr, nk_size_t *max_index_ptr) {
1279
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u16_t);
1280
+ int aligned = (stride_bytes % sizeof(nk_u16_t) == 0);
1281
+
1282
+ if (count == 0)
1283
+ *min_value_ptr = NK_U16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_U16_MIN,
1284
+ *max_index_ptr = NK_SIZE_MAX;
1285
+ else if (!aligned)
1286
+ nk_reduce_minmax_u16_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1287
+ max_index_ptr);
1288
+ else if (stride_elements == 1)
1289
+ nk_reduce_minmax_u16_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
1290
+ max_index_ptr);
1291
+ else
1292
+ nk_reduce_minmax_u16_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1293
+ max_index_ptr);
1294
+ }
1295
+
1296
+ NK_INTERNAL void nk_reduce_moments_i32_rvv_contiguous_( //
1297
+ nk_i32_t const *data_ptr, nk_size_t count, //
1298
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1299
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
1300
+ // 128-bit per-lane accumulator for sum: (sum_high, sum_low)
1301
+ vuint64m2_t sum_low_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1302
+ vint64m2_t sum_high_i64m2 = __riscv_vmv_v_x_i64m2(0, max_vector_length);
1303
+ vuint64m2_t sumsq_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1304
+
1305
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
1306
+ vector_length = __riscv_vsetvl_e32m1(count);
1307
+ vint32m1_t data_i32m1 = __riscv_vle32_v_i32m1(data_ptr, vector_length);
1308
+
1309
+ // Widen i32 → i64
1310
+ vint64m2_t data_i64m2 = __riscv_vsext_vf2_i64m2(data_i32m1, vector_length);
1311
+ vuint64m2_t data_u64m2 = __riscv_vreinterpret_v_i64m2_u64m2(data_i64m2);
1312
+
1313
+ // 128-bit accumulation: wrapping add on lower half
1314
+ vuint64m2_t sum_before_u64m2 = sum_low_u64m2;
1315
+ sum_low_u64m2 = __riscv_vadd_vv_u64m2_tu(sum_low_u64m2, sum_low_u64m2, data_u64m2, vector_length);
1316
+
1317
+ // Carry: new < old means unsigned overflow occurred
1318
+ vbool32_t carry_b32 = __riscv_vmsltu_vv_u64m2_b32(sum_low_u64m2, sum_before_u64m2, vector_length);
1319
+ vint64m2_t carry_i64m2 = __riscv_vmerge_vxm_i64m2(__riscv_vmv_v_x_i64m2(0, vector_length), 1, carry_b32,
1320
+ vector_length);
1321
+ sum_high_i64m2 = __riscv_vadd_vv_i64m2_tu(sum_high_i64m2, sum_high_i64m2, carry_i64m2, vector_length);
1322
+
1323
+ // Sign extension: -1 for negative, 0 for non-negative
1324
+ vint64m2_t sign_ext_i64m2 = __riscv_vsra_vx_i64m2(data_i64m2, 63, vector_length);
1325
+ sum_high_i64m2 = __riscv_vadd_vv_i64m2_tu(sum_high_i64m2, sum_high_i64m2, sign_ext_i64m2, vector_length);
1326
+
1327
+ // Sumsq: i32 × i32 → i64 (widening multiply, result ≤ 2^62), saturating accumulation
1328
+ vint64m2_t squares_i64m2 = __riscv_vwmul_vv_i64m2(data_i32m1, data_i32m1, vector_length);
1329
+ sumsq_u64m2 = __riscv_vsaddu_vv_u64m2_tu(sumsq_u64m2, sumsq_u64m2,
1330
+ __riscv_vreinterpret_v_i64m2_u64m2(squares_i64m2), vector_length);
1331
+ }
1332
+
1333
+ *sum_ptr = nk_reduce_128bit_sum_i64m2_rvv_(sum_low_u64m2, sum_high_i64m2, max_vector_length);
1334
+ *sumsq_ptr = nk_reduce_vsaddu_u64m2_rvv_(sumsq_u64m2, max_vector_length);
1335
+ }
1336
+
1337
+ NK_INTERNAL void nk_reduce_moments_i32_rvv_strided_( //
1338
+ nk_i32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1339
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1340
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
1341
+ // 128-bit per-lane accumulator for sum: (sum_high, sum_low)
1342
+ vuint64m2_t sum_low_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1343
+ vint64m2_t sum_high_i64m2 = __riscv_vmv_v_x_i64m2(0, max_vector_length);
1344
+ vuint64m2_t sumsq_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1345
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1346
+
1347
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
1348
+ vector_length = __riscv_vsetvl_e32m1(count);
1349
+ vint32m1_t data_i32m1 = __riscv_vlse32_v_i32m1((nk_i32_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1350
+
1351
+ // Widen i32 → i64
1352
+ vint64m2_t data_i64m2 = __riscv_vsext_vf2_i64m2(data_i32m1, vector_length);
1353
+ vuint64m2_t data_u64m2 = __riscv_vreinterpret_v_i64m2_u64m2(data_i64m2);
1354
+
1355
+ // 128-bit accumulation: wrapping add on lower half
1356
+ vuint64m2_t sum_before_u64m2 = sum_low_u64m2;
1357
+ sum_low_u64m2 = __riscv_vadd_vv_u64m2_tu(sum_low_u64m2, sum_low_u64m2, data_u64m2, vector_length);
1358
+
1359
+ // Carry: new < old means unsigned overflow occurred
1360
+ vbool32_t carry_b32 = __riscv_vmsltu_vv_u64m2_b32(sum_low_u64m2, sum_before_u64m2, vector_length);
1361
+ vint64m2_t carry_i64m2 = __riscv_vmerge_vxm_i64m2(__riscv_vmv_v_x_i64m2(0, vector_length), 1, carry_b32,
1362
+ vector_length);
1363
+ sum_high_i64m2 = __riscv_vadd_vv_i64m2_tu(sum_high_i64m2, sum_high_i64m2, carry_i64m2, vector_length);
1364
+
1365
+ // Sign extension: -1 for negative, 0 for non-negative
1366
+ vint64m2_t sign_ext_i64m2 = __riscv_vsra_vx_i64m2(data_i64m2, 63, vector_length);
1367
+ sum_high_i64m2 = __riscv_vadd_vv_i64m2_tu(sum_high_i64m2, sum_high_i64m2, sign_ext_i64m2, vector_length);
1368
+
1369
+ // Sumsq: i32 × i32 → i64 (widening multiply, result ≤ 2^62), saturating accumulation
1370
+ vint64m2_t squares_i64m2 = __riscv_vwmul_vv_i64m2(data_i32m1, data_i32m1, vector_length);
1371
+ sumsq_u64m2 = __riscv_vsaddu_vv_u64m2_tu(sumsq_u64m2, sumsq_u64m2,
1372
+ __riscv_vreinterpret_v_i64m2_u64m2(squares_i64m2), vector_length);
1373
+ }
1374
+
1375
+ *sum_ptr = nk_reduce_128bit_sum_i64m2_rvv_(sum_low_u64m2, sum_high_i64m2, max_vector_length);
1376
+ *sumsq_ptr = nk_reduce_vsaddu_u64m2_rvv_(sumsq_u64m2, max_vector_length);
1377
+ }
1378
+
1379
+ NK_PUBLIC void nk_reduce_moments_i32_rvv( //
1380
+ nk_i32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1381
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1382
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i32_t);
1383
+ int aligned = (stride_bytes % sizeof(nk_i32_t) == 0);
1384
+
1385
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
1386
+ else if (!aligned) { nk_reduce_moments_i32_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1387
+ else if (stride_elements == 1) { nk_reduce_moments_i32_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
1388
+ else { nk_reduce_moments_i32_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1389
+ }
1390
+
1391
+ NK_INTERNAL void nk_reduce_minmax_i32_rvv_contiguous_( //
1392
+ nk_i32_t const *data_ptr, nk_size_t count, //
1393
+ nk_i32_t *min_value_ptr, nk_size_t *min_index_ptr, //
1394
+ nk_i32_t *max_value_ptr, nk_size_t *max_index_ptr) {
1395
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m1();
1396
+ vint32m1_t min_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MAX, max_vector_length);
1397
+ vint32m1_t max_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MIN, max_vector_length);
1398
+ vuint64m2_t min_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1399
+ vuint64m2_t max_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1400
+
1401
+ nk_size_t offset = 0;
1402
+ for (nk_size_t vector_length; count > 0;
1403
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
1404
+ vector_length = __riscv_vsetvl_e32m1(count);
1405
+ vint32m1_t data_i32m1 = __riscv_vle32_v_i32m1(data_ptr, vector_length);
1406
+ vuint64m2_t pos_u64m2 = __riscv_vadd_vx_u64m2(__riscv_vid_v_u64m2(vector_length), (nk_u64_t)offset,
1407
+ vector_length);
1408
+
1409
+ vbool32_t less_b32 = __riscv_vmslt_vv_i32m1_b32(data_i32m1, min_i32m1, vector_length);
1410
+ min_i32m1 = __riscv_vmerge_vvm_i32m1_tu(min_i32m1, min_i32m1, data_i32m1, less_b32, vector_length);
1411
+ min_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(min_indices_u64m2, min_indices_u64m2, pos_u64m2, less_b32,
1412
+ vector_length);
1413
+
1414
+ vbool32_t greater_b32 = __riscv_vmslt_vv_i32m1_b32(max_i32m1, data_i32m1, vector_length);
1415
+ max_i32m1 = __riscv_vmerge_vvm_i32m1_tu(max_i32m1, max_i32m1, data_i32m1, greater_b32, vector_length);
1416
+ max_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(max_indices_u64m2, max_indices_u64m2, pos_u64m2, greater_b32,
1417
+ vector_length);
1418
+ }
1419
+
1420
+ // Horizontal reduction for min
1421
+ vint32m1_t init_max_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MAX, 1);
1422
+ nk_i32_t min_val = __riscv_vmv_x_s_i32m1_i32(
1423
+ __riscv_vredmin_vs_i32m1_i32m1(min_i32m1, init_max_i32m1, max_vector_length));
1424
+ vbool32_t min_match_b32 = __riscv_vmseq_vx_i32m1_b32(min_i32m1, min_val, max_vector_length);
1425
+ vuint64m2_t sentinel_u64m2 = __riscv_vmv_v_x_u64m2(NK_U64_MAX, max_vector_length);
1426
+ vuint64m2_t min_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, min_indices_u64m2, min_match_b32,
1427
+ max_vector_length);
1428
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1429
+ *min_value_ptr = min_val;
1430
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1431
+ __riscv_vredminu_vs_u64m2_u64m1(min_cands_u64m2, init_umax_u64m1, max_vector_length));
1432
+
1433
+ // Horizontal reduction for max
1434
+ vint32m1_t init_min_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MIN, 1);
1435
+ nk_i32_t max_val = __riscv_vmv_x_s_i32m1_i32(
1436
+ __riscv_vredmax_vs_i32m1_i32m1(max_i32m1, init_min_i32m1, max_vector_length));
1437
+ vbool32_t max_match_b32 = __riscv_vmseq_vx_i32m1_b32(max_i32m1, max_val, max_vector_length);
1438
+ vuint64m2_t max_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, max_indices_u64m2, max_match_b32,
1439
+ max_vector_length);
1440
+ *max_value_ptr = max_val;
1441
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1442
+ __riscv_vredminu_vs_u64m2_u64m1(max_cands_u64m2, init_umax_u64m1, max_vector_length));
1443
+ }
1444
+
1445
+ NK_INTERNAL void nk_reduce_minmax_i32_rvv_strided_( //
1446
+ nk_i32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1447
+ nk_i32_t *min_value_ptr, nk_size_t *min_index_ptr, //
1448
+ nk_i32_t *max_value_ptr, nk_size_t *max_index_ptr) {
1449
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m1();
1450
+ vint32m1_t min_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MAX, max_vector_length);
1451
+ vint32m1_t max_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MIN, max_vector_length);
1452
+ vuint64m2_t min_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1453
+ vuint64m2_t max_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1454
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1455
+
1456
+ nk_size_t offset = 0;
1457
+ for (nk_size_t vector_length; count > 0;
1458
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
1459
+ vector_length = __riscv_vsetvl_e32m1(count);
1460
+ vint32m1_t data_i32m1 = __riscv_vlse32_v_i32m1((nk_i32_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1461
+ vuint64m2_t pos_u64m2 = __riscv_vadd_vx_u64m2(__riscv_vid_v_u64m2(vector_length), (nk_u64_t)offset,
1462
+ vector_length);
1463
+
1464
+ vbool32_t less_b32 = __riscv_vmslt_vv_i32m1_b32(data_i32m1, min_i32m1, vector_length);
1465
+ min_i32m1 = __riscv_vmerge_vvm_i32m1_tu(min_i32m1, min_i32m1, data_i32m1, less_b32, vector_length);
1466
+ min_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(min_indices_u64m2, min_indices_u64m2, pos_u64m2, less_b32,
1467
+ vector_length);
1468
+
1469
+ vbool32_t greater_b32 = __riscv_vmslt_vv_i32m1_b32(max_i32m1, data_i32m1, vector_length);
1470
+ max_i32m1 = __riscv_vmerge_vvm_i32m1_tu(max_i32m1, max_i32m1, data_i32m1, greater_b32, vector_length);
1471
+ max_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(max_indices_u64m2, max_indices_u64m2, pos_u64m2, greater_b32,
1472
+ vector_length);
1473
+ }
1474
+
1475
+ // Horizontal reduction for min
1476
+ vint32m1_t init_max_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MAX, 1);
1477
+ nk_i32_t min_val = __riscv_vmv_x_s_i32m1_i32(
1478
+ __riscv_vredmin_vs_i32m1_i32m1(min_i32m1, init_max_i32m1, max_vector_length));
1479
+ vbool32_t min_match_b32 = __riscv_vmseq_vx_i32m1_b32(min_i32m1, min_val, max_vector_length);
1480
+ vuint64m2_t sentinel_u64m2 = __riscv_vmv_v_x_u64m2(NK_U64_MAX, max_vector_length);
1481
+ vuint64m2_t min_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, min_indices_u64m2, min_match_b32,
1482
+ max_vector_length);
1483
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1484
+ *min_value_ptr = min_val;
1485
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1486
+ __riscv_vredminu_vs_u64m2_u64m1(min_cands_u64m2, init_umax_u64m1, max_vector_length));
1487
+
1488
+ // Horizontal reduction for max
1489
+ vint32m1_t init_min_i32m1 = __riscv_vmv_v_x_i32m1(NK_I32_MIN, 1);
1490
+ nk_i32_t max_val = __riscv_vmv_x_s_i32m1_i32(
1491
+ __riscv_vredmax_vs_i32m1_i32m1(max_i32m1, init_min_i32m1, max_vector_length));
1492
+ vbool32_t max_match_b32 = __riscv_vmseq_vx_i32m1_b32(max_i32m1, max_val, max_vector_length);
1493
+ vuint64m2_t max_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, max_indices_u64m2, max_match_b32,
1494
+ max_vector_length);
1495
+ *max_value_ptr = max_val;
1496
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1497
+ __riscv_vredminu_vs_u64m2_u64m1(max_cands_u64m2, init_umax_u64m1, max_vector_length));
1498
+ }
1499
+
1500
+ NK_PUBLIC void nk_reduce_minmax_i32_rvv( //
1501
+ nk_i32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1502
+ nk_i32_t *min_value_ptr, nk_size_t *min_index_ptr, //
1503
+ nk_i32_t *max_value_ptr, nk_size_t *max_index_ptr) {
1504
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i32_t);
1505
+ int aligned = (stride_bytes % sizeof(nk_i32_t) == 0);
1506
+
1507
+ if (count == 0)
1508
+ *min_value_ptr = NK_I32_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_I32_MIN,
1509
+ *max_index_ptr = NK_SIZE_MAX;
1510
+ else if (!aligned)
1511
+ nk_reduce_minmax_i32_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1512
+ max_index_ptr);
1513
+ else if (stride_elements == 1)
1514
+ nk_reduce_minmax_i32_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
1515
+ max_index_ptr);
1516
+ else
1517
+ nk_reduce_minmax_i32_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1518
+ max_index_ptr);
1519
+ }
1520
+
1521
+ NK_INTERNAL void nk_reduce_moments_u32_rvv_contiguous_( //
1522
+ nk_u32_t const *data_ptr, nk_size_t count, //
1523
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1524
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
1525
+ vuint64m2_t sum_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1526
+ vuint64m2_t sumsq_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1527
+
1528
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
1529
+ vector_length = __riscv_vsetvl_e32m1(count);
1530
+ vuint32m1_t data_u32m1 = __riscv_vle32_v_u32m1(data_ptr, vector_length);
1531
+
1532
+ // Widen u32 → u64 for saturating sum
1533
+ vuint64m2_t data_u64m2 = __riscv_vzext_vf2_u64m2(data_u32m1, vector_length);
1534
+ sum_u64m2 = __riscv_vsaddu_vv_u64m2_tu(sum_u64m2, sum_u64m2, data_u64m2, vector_length);
1535
+
1536
+ // Sumsq: u32 × u32 → u64 (widening multiply, no overflow), saturating accumulation
1537
+ vuint64m2_t squares_u64m2 = __riscv_vwmulu_vv_u64m2(data_u32m1, data_u32m1, vector_length);
1538
+ sumsq_u64m2 = __riscv_vsaddu_vv_u64m2_tu(sumsq_u64m2, sumsq_u64m2, squares_u64m2, vector_length);
1539
+ }
1540
+
1541
+ *sum_ptr = nk_reduce_vsaddu_u64m2_rvv_(sum_u64m2, max_vector_length);
1542
+ *sumsq_ptr = nk_reduce_vsaddu_u64m2_rvv_(sumsq_u64m2, max_vector_length);
1543
+ }
1544
+
1545
+ NK_INTERNAL void nk_reduce_moments_u32_rvv_strided_( //
1546
+ nk_u32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1547
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1548
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
1549
+ vuint64m2_t sum_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1550
+ vuint64m2_t sumsq_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1551
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1552
+
1553
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
1554
+ vector_length = __riscv_vsetvl_e32m1(count);
1555
+ vuint32m1_t data_u32m1 = __riscv_vlse32_v_u32m1((nk_u32_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1556
+
1557
+ // Widen u32 → u64 for saturating sum
1558
+ vuint64m2_t data_u64m2 = __riscv_vzext_vf2_u64m2(data_u32m1, vector_length);
1559
+ sum_u64m2 = __riscv_vsaddu_vv_u64m2_tu(sum_u64m2, sum_u64m2, data_u64m2, vector_length);
1560
+
1561
+ // Sumsq: u32 × u32 → u64 (widening multiply, no overflow), saturating accumulation
1562
+ vuint64m2_t squares_u64m2 = __riscv_vwmulu_vv_u64m2(data_u32m1, data_u32m1, vector_length);
1563
+ sumsq_u64m2 = __riscv_vsaddu_vv_u64m2_tu(sumsq_u64m2, sumsq_u64m2, squares_u64m2, vector_length);
1564
+ }
1565
+
1566
+ *sum_ptr = nk_reduce_vsaddu_u64m2_rvv_(sum_u64m2, max_vector_length);
1567
+ *sumsq_ptr = nk_reduce_vsaddu_u64m2_rvv_(sumsq_u64m2, max_vector_length);
1568
+ }
1569
+
1570
+ NK_PUBLIC void nk_reduce_moments_u32_rvv( //
1571
+ nk_u32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1572
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1573
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u32_t);
1574
+ int aligned = (stride_bytes % sizeof(nk_u32_t) == 0);
1575
+
1576
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
1577
+ else if (!aligned) { nk_reduce_moments_u32_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1578
+ else if (stride_elements == 1) { nk_reduce_moments_u32_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
1579
+ else { nk_reduce_moments_u32_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1580
+ }
1581
+
1582
+ NK_INTERNAL void nk_reduce_minmax_u32_rvv_contiguous_( //
1583
+ nk_u32_t const *data_ptr, nk_size_t count, //
1584
+ nk_u32_t *min_value_ptr, nk_size_t *min_index_ptr, //
1585
+ nk_u32_t *max_value_ptr, nk_size_t *max_index_ptr) {
1586
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m1();
1587
+ vuint32m1_t min_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MAX, max_vector_length);
1588
+ vuint32m1_t max_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MIN, max_vector_length);
1589
+ vuint64m2_t min_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1590
+ vuint64m2_t max_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1591
+
1592
+ nk_size_t offset = 0;
1593
+ for (nk_size_t vector_length; count > 0;
1594
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
1595
+ vector_length = __riscv_vsetvl_e32m1(count);
1596
+ vuint32m1_t data_u32m1 = __riscv_vle32_v_u32m1(data_ptr, vector_length);
1597
+ vuint64m2_t pos_u64m2 = __riscv_vadd_vx_u64m2(__riscv_vid_v_u64m2(vector_length), (nk_u64_t)offset,
1598
+ vector_length);
1599
+
1600
+ vbool32_t less_b32 = __riscv_vmsltu_vv_u32m1_b32(data_u32m1, min_u32m1, vector_length);
1601
+ min_u32m1 = __riscv_vmerge_vvm_u32m1_tu(min_u32m1, min_u32m1, data_u32m1, less_b32, vector_length);
1602
+ min_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(min_indices_u64m2, min_indices_u64m2, pos_u64m2, less_b32,
1603
+ vector_length);
1604
+
1605
+ vbool32_t greater_b32 = __riscv_vmsltu_vv_u32m1_b32(max_u32m1, data_u32m1, vector_length);
1606
+ max_u32m1 = __riscv_vmerge_vvm_u32m1_tu(max_u32m1, max_u32m1, data_u32m1, greater_b32, vector_length);
1607
+ max_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(max_indices_u64m2, max_indices_u64m2, pos_u64m2, greater_b32,
1608
+ vector_length);
1609
+ }
1610
+
1611
+ // Horizontal reduction for min
1612
+ vuint32m1_t init_max_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MAX, 1);
1613
+ nk_u32_t min_val = __riscv_vmv_x_s_u32m1_u32(
1614
+ __riscv_vredminu_vs_u32m1_u32m1(min_u32m1, init_max_u32m1, max_vector_length));
1615
+ vbool32_t min_match_b32 = __riscv_vmseq_vx_u32m1_b32(min_u32m1, min_val, max_vector_length);
1616
+ vuint64m2_t sentinel_u64m2 = __riscv_vmv_v_x_u64m2(NK_U64_MAX, max_vector_length);
1617
+ vuint64m2_t min_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, min_indices_u64m2, min_match_b32,
1618
+ max_vector_length);
1619
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1620
+ *min_value_ptr = min_val;
1621
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1622
+ __riscv_vredminu_vs_u64m2_u64m1(min_cands_u64m2, init_umax_u64m1, max_vector_length));
1623
+
1624
+ // Horizontal reduction for max
1625
+ vuint32m1_t init_min_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MIN, 1);
1626
+ nk_u32_t max_val = __riscv_vmv_x_s_u32m1_u32(
1627
+ __riscv_vredmaxu_vs_u32m1_u32m1(max_u32m1, init_min_u32m1, max_vector_length));
1628
+ vbool32_t max_match_b32 = __riscv_vmseq_vx_u32m1_b32(max_u32m1, max_val, max_vector_length);
1629
+ vuint64m2_t max_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, max_indices_u64m2, max_match_b32,
1630
+ max_vector_length);
1631
+ *max_value_ptr = max_val;
1632
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1633
+ __riscv_vredminu_vs_u64m2_u64m1(max_cands_u64m2, init_umax_u64m1, max_vector_length));
1634
+ }
1635
+
1636
+ NK_INTERNAL void nk_reduce_minmax_u32_rvv_strided_( //
1637
+ nk_u32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1638
+ nk_u32_t *min_value_ptr, nk_size_t *min_index_ptr, //
1639
+ nk_u32_t *max_value_ptr, nk_size_t *max_index_ptr) {
1640
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m1();
1641
+ vuint32m1_t min_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MAX, max_vector_length);
1642
+ vuint32m1_t max_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MIN, max_vector_length);
1643
+ vuint64m2_t min_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1644
+ vuint64m2_t max_indices_u64m2 = __riscv_vmv_v_x_u64m2(0, max_vector_length);
1645
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1646
+
1647
+ nk_size_t offset = 0;
1648
+ for (nk_size_t vector_length; count > 0;
1649
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
1650
+ vector_length = __riscv_vsetvl_e32m1(count);
1651
+ vuint32m1_t data_u32m1 = __riscv_vlse32_v_u32m1((nk_u32_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1652
+ vuint64m2_t pos_u64m2 = __riscv_vadd_vx_u64m2(__riscv_vid_v_u64m2(vector_length), (nk_u64_t)offset,
1653
+ vector_length);
1654
+
1655
+ vbool32_t less_b32 = __riscv_vmsltu_vv_u32m1_b32(data_u32m1, min_u32m1, vector_length);
1656
+ min_u32m1 = __riscv_vmerge_vvm_u32m1_tu(min_u32m1, min_u32m1, data_u32m1, less_b32, vector_length);
1657
+ min_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(min_indices_u64m2, min_indices_u64m2, pos_u64m2, less_b32,
1658
+ vector_length);
1659
+
1660
+ vbool32_t greater_b32 = __riscv_vmsltu_vv_u32m1_b32(max_u32m1, data_u32m1, vector_length);
1661
+ max_u32m1 = __riscv_vmerge_vvm_u32m1_tu(max_u32m1, max_u32m1, data_u32m1, greater_b32, vector_length);
1662
+ max_indices_u64m2 = __riscv_vmerge_vvm_u64m2_tu(max_indices_u64m2, max_indices_u64m2, pos_u64m2, greater_b32,
1663
+ vector_length);
1664
+ }
1665
+
1666
+ // Horizontal reduction for min
1667
+ vuint32m1_t init_max_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MAX, 1);
1668
+ nk_u32_t min_val = __riscv_vmv_x_s_u32m1_u32(
1669
+ __riscv_vredminu_vs_u32m1_u32m1(min_u32m1, init_max_u32m1, max_vector_length));
1670
+ vbool32_t min_match_b32 = __riscv_vmseq_vx_u32m1_b32(min_u32m1, min_val, max_vector_length);
1671
+ vuint64m2_t sentinel_u64m2 = __riscv_vmv_v_x_u64m2(NK_U64_MAX, max_vector_length);
1672
+ vuint64m2_t min_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, min_indices_u64m2, min_match_b32,
1673
+ max_vector_length);
1674
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1675
+ *min_value_ptr = min_val;
1676
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1677
+ __riscv_vredminu_vs_u64m2_u64m1(min_cands_u64m2, init_umax_u64m1, max_vector_length));
1678
+
1679
+ // Horizontal reduction for max
1680
+ vuint32m1_t init_min_u32m1 = __riscv_vmv_v_x_u32m1(NK_U32_MIN, 1);
1681
+ nk_u32_t max_val = __riscv_vmv_x_s_u32m1_u32(
1682
+ __riscv_vredmaxu_vs_u32m1_u32m1(max_u32m1, init_min_u32m1, max_vector_length));
1683
+ vbool32_t max_match_b32 = __riscv_vmseq_vx_u32m1_b32(max_u32m1, max_val, max_vector_length);
1684
+ vuint64m2_t max_cands_u64m2 = __riscv_vmerge_vvm_u64m2(sentinel_u64m2, max_indices_u64m2, max_match_b32,
1685
+ max_vector_length);
1686
+ *max_value_ptr = max_val;
1687
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1688
+ __riscv_vredminu_vs_u64m2_u64m1(max_cands_u64m2, init_umax_u64m1, max_vector_length));
1689
+ }
1690
+
1691
+ NK_PUBLIC void nk_reduce_minmax_u32_rvv( //
1692
+ nk_u32_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1693
+ nk_u32_t *min_value_ptr, nk_size_t *min_index_ptr, //
1694
+ nk_u32_t *max_value_ptr, nk_size_t *max_index_ptr) {
1695
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u32_t);
1696
+ int aligned = (stride_bytes % sizeof(nk_u32_t) == 0);
1697
+
1698
+ if (count == 0)
1699
+ *min_value_ptr = NK_U32_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_U32_MIN,
1700
+ *max_index_ptr = NK_SIZE_MAX;
1701
+ else if (!aligned)
1702
+ nk_reduce_minmax_u32_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1703
+ max_index_ptr);
1704
+ else if (stride_elements == 1)
1705
+ nk_reduce_minmax_u32_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
1706
+ max_index_ptr);
1707
+ else
1708
+ nk_reduce_minmax_u32_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1709
+ max_index_ptr);
1710
+ }
1711
+
1712
+ NK_INTERNAL void nk_reduce_moments_i64_rvv_contiguous_( //
1713
+ nk_i64_t const *data_ptr, nk_size_t count, //
1714
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1715
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
1716
+ // 128-bit per-lane accumulator for sum: (sum_high, sum_low)
1717
+ vuint64m1_t sum_low_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1718
+ vint64m1_t sum_high_i64m1 = __riscv_vmv_v_x_i64m1(0, max_vector_length);
1719
+ vuint64m1_t sumsq_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1720
+
1721
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
1722
+ vector_length = __riscv_vsetvl_e64m1(count);
1723
+ vint64m1_t data_i64m1 = __riscv_vle64_v_i64m1(data_ptr, vector_length);
1724
+
1725
+ // 128-bit sum accumulation: wrapping add on lower half
1726
+ vuint64m1_t data_u64m1 = __riscv_vreinterpret_v_i64m1_u64m1(data_i64m1);
1727
+ vuint64m1_t sum_before_u64m1 = sum_low_u64m1;
1728
+ sum_low_u64m1 = __riscv_vadd_vv_u64m1_tu(sum_low_u64m1, sum_low_u64m1, data_u64m1, vector_length);
1729
+
1730
+ // Carry: new < old means unsigned overflow occurred
1731
+ vbool64_t carry_b64 = __riscv_vmsltu_vv_u64m1_b64(sum_low_u64m1, sum_before_u64m1, vector_length);
1732
+ vint64m1_t carry_i64m1 = __riscv_vmerge_vxm_i64m1(__riscv_vmv_v_x_i64m1(0, vector_length), 1, carry_b64,
1733
+ vector_length);
1734
+ sum_high_i64m1 = __riscv_vadd_vv_i64m1_tu(sum_high_i64m1, sum_high_i64m1, carry_i64m1, vector_length);
1735
+
1736
+ // Sign extension: -1 for negative, 0 for non-negative
1737
+ vint64m1_t sign_ext_i64m1 = __riscv_vsra_vx_i64m1(data_i64m1, 63, vector_length);
1738
+ sum_high_i64m1 = __riscv_vadd_vv_i64m1_tu(sum_high_i64m1, sum_high_i64m1, sign_ext_i64m1, vector_length);
1739
+
1740
+ // Sumsq: abs(val)² with overflow detection
1741
+ vint64m1_t negated_i64m1 = __riscv_vneg_v_i64m1(data_i64m1, vector_length);
1742
+ vint64m1_t absolute_i64m1 = __riscv_vmax_vv_i64m1(data_i64m1, negated_i64m1, vector_length);
1743
+ vuint64m1_t absolute_u64m1 = __riscv_vreinterpret_v_i64m1_u64m1(absolute_i64m1);
1744
+ vuint64m1_t product_low_u64m1 = __riscv_vmul_vv_u64m1(absolute_u64m1, absolute_u64m1, vector_length);
1745
+ vuint64m1_t product_high_u64m1 = __riscv_vmulhu_vv_u64m1(absolute_u64m1, absolute_u64m1, vector_length);
1746
+ vbool64_t overflow_b64 = __riscv_vmsne_vx_u64m1_b64(product_high_u64m1, 0, vector_length);
1747
+ vuint64m1_t squares_u64m1 = __riscv_vmerge_vxm_u64m1(product_low_u64m1, NK_U64_MAX, overflow_b64,
1748
+ vector_length);
1749
+ sumsq_u64m1 = __riscv_vsaddu_vv_u64m1_tu(sumsq_u64m1, sumsq_u64m1, squares_u64m1, vector_length);
1750
+ }
1751
+
1752
+ *sum_ptr = nk_reduce_128bit_sum_i64m1_rvv_(sum_low_u64m1, sum_high_i64m1, max_vector_length);
1753
+ *sumsq_ptr = nk_reduce_vsaddu_u64m1_rvv_(sumsq_u64m1, max_vector_length);
1754
+ }
1755
+
1756
+ NK_INTERNAL void nk_reduce_moments_i64_rvv_strided_( //
1757
+ nk_i64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1758
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1759
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
1760
+ // 128-bit per-lane accumulator for sum: (sum_high, sum_low)
1761
+ vuint64m1_t sum_low_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1762
+ vint64m1_t sum_high_i64m1 = __riscv_vmv_v_x_i64m1(0, max_vector_length);
1763
+ vuint64m1_t sumsq_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1764
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1765
+
1766
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
1767
+ vector_length = __riscv_vsetvl_e64m1(count);
1768
+ vint64m1_t data_i64m1 = __riscv_vlse64_v_i64m1((nk_i64_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1769
+
1770
+ // 128-bit sum accumulation: wrapping add on lower half
1771
+ vuint64m1_t data_u64m1 = __riscv_vreinterpret_v_i64m1_u64m1(data_i64m1);
1772
+ vuint64m1_t sum_before_u64m1 = sum_low_u64m1;
1773
+ sum_low_u64m1 = __riscv_vadd_vv_u64m1_tu(sum_low_u64m1, sum_low_u64m1, data_u64m1, vector_length);
1774
+
1775
+ // Carry: new < old means unsigned overflow occurred
1776
+ vbool64_t carry_b64 = __riscv_vmsltu_vv_u64m1_b64(sum_low_u64m1, sum_before_u64m1, vector_length);
1777
+ vint64m1_t carry_i64m1 = __riscv_vmerge_vxm_i64m1(__riscv_vmv_v_x_i64m1(0, vector_length), 1, carry_b64,
1778
+ vector_length);
1779
+ sum_high_i64m1 = __riscv_vadd_vv_i64m1_tu(sum_high_i64m1, sum_high_i64m1, carry_i64m1, vector_length);
1780
+
1781
+ // Sign extension: -1 for negative, 0 for non-negative
1782
+ vint64m1_t sign_ext_i64m1 = __riscv_vsra_vx_i64m1(data_i64m1, 63, vector_length);
1783
+ sum_high_i64m1 = __riscv_vadd_vv_i64m1_tu(sum_high_i64m1, sum_high_i64m1, sign_ext_i64m1, vector_length);
1784
+
1785
+ // Sumsq: abs(val)² with overflow detection
1786
+ vint64m1_t negated_i64m1 = __riscv_vneg_v_i64m1(data_i64m1, vector_length);
1787
+ vint64m1_t absolute_i64m1 = __riscv_vmax_vv_i64m1(data_i64m1, negated_i64m1, vector_length);
1788
+ vuint64m1_t absolute_u64m1 = __riscv_vreinterpret_v_i64m1_u64m1(absolute_i64m1);
1789
+ vuint64m1_t product_low_u64m1 = __riscv_vmul_vv_u64m1(absolute_u64m1, absolute_u64m1, vector_length);
1790
+ vuint64m1_t product_high_u64m1 = __riscv_vmulhu_vv_u64m1(absolute_u64m1, absolute_u64m1, vector_length);
1791
+ vbool64_t overflow_b64 = __riscv_vmsne_vx_u64m1_b64(product_high_u64m1, 0, vector_length);
1792
+ vuint64m1_t squares_u64m1 = __riscv_vmerge_vxm_u64m1(product_low_u64m1, NK_U64_MAX, overflow_b64,
1793
+ vector_length);
1794
+ sumsq_u64m1 = __riscv_vsaddu_vv_u64m1_tu(sumsq_u64m1, sumsq_u64m1, squares_u64m1, vector_length);
1795
+ }
1796
+
1797
+ *sum_ptr = nk_reduce_128bit_sum_i64m1_rvv_(sum_low_u64m1, sum_high_i64m1, max_vector_length);
1798
+ *sumsq_ptr = nk_reduce_vsaddu_u64m1_rvv_(sumsq_u64m1, max_vector_length);
1799
+ }
1800
+
1801
+ NK_PUBLIC void nk_reduce_moments_i64_rvv( //
1802
+ nk_i64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1803
+ nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1804
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i64_t);
1805
+ int aligned = (stride_bytes % sizeof(nk_i64_t) == 0);
1806
+
1807
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
1808
+ else if (!aligned) { nk_reduce_moments_i64_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1809
+ else if (stride_elements == 1) { nk_reduce_moments_i64_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
1810
+ else { nk_reduce_moments_i64_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
1811
+ }
1812
+
1813
+ NK_INTERNAL void nk_reduce_minmax_i64_rvv_contiguous_( //
1814
+ nk_i64_t const *data_ptr, nk_size_t count, //
1815
+ nk_i64_t *min_value_ptr, nk_size_t *min_index_ptr, //
1816
+ nk_i64_t *max_value_ptr, nk_size_t *max_index_ptr) {
1817
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
1818
+ vint64m1_t min_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MAX, max_vector_length);
1819
+ vint64m1_t max_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MIN, max_vector_length);
1820
+ vuint64m1_t min_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1821
+ vuint64m1_t max_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1822
+
1823
+ nk_size_t offset = 0;
1824
+ for (nk_size_t vector_length; count > 0;
1825
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
1826
+ vector_length = __riscv_vsetvl_e64m1(count);
1827
+ vint64m1_t data_i64m1 = __riscv_vle64_v_i64m1(data_ptr, vector_length);
1828
+ vuint64m1_t pos_u64m1 = __riscv_vadd_vx_u64m1(__riscv_vid_v_u64m1(vector_length), (nk_u64_t)offset,
1829
+ vector_length);
1830
+
1831
+ vbool64_t less_b64 = __riscv_vmslt_vv_i64m1_b64(data_i64m1, min_i64m1, vector_length);
1832
+ min_i64m1 = __riscv_vmerge_vvm_i64m1_tu(min_i64m1, min_i64m1, data_i64m1, less_b64, vector_length);
1833
+ min_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(min_indices_u64m1, min_indices_u64m1, pos_u64m1, less_b64,
1834
+ vector_length);
1835
+
1836
+ vbool64_t greater_b64 = __riscv_vmslt_vv_i64m1_b64(max_i64m1, data_i64m1, vector_length);
1837
+ max_i64m1 = __riscv_vmerge_vvm_i64m1_tu(max_i64m1, max_i64m1, data_i64m1, greater_b64, vector_length);
1838
+ max_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(max_indices_u64m1, max_indices_u64m1, pos_u64m1, greater_b64,
1839
+ vector_length);
1840
+ }
1841
+
1842
+ // Horizontal reduction for min
1843
+ vint64m1_t init_max_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MAX, 1);
1844
+ nk_i64_t min_val = __riscv_vmv_x_s_i64m1_i64(
1845
+ __riscv_vredmin_vs_i64m1_i64m1(min_i64m1, init_max_i64m1, max_vector_length));
1846
+ vbool64_t min_match_b64 = __riscv_vmseq_vx_i64m1_b64(min_i64m1, min_val, max_vector_length);
1847
+ vuint64m1_t sentinel_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
1848
+ vuint64m1_t min_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, min_indices_u64m1, min_match_b64,
1849
+ max_vector_length);
1850
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1851
+ *min_value_ptr = min_val;
1852
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1853
+ __riscv_vredminu_vs_u64m1_u64m1(min_cands_u64m1, init_umax_u64m1, max_vector_length));
1854
+
1855
+ // Horizontal reduction for max
1856
+ vint64m1_t init_min_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MIN, 1);
1857
+ nk_i64_t max_val = __riscv_vmv_x_s_i64m1_i64(
1858
+ __riscv_vredmax_vs_i64m1_i64m1(max_i64m1, init_min_i64m1, max_vector_length));
1859
+ vbool64_t max_match_b64 = __riscv_vmseq_vx_i64m1_b64(max_i64m1, max_val, max_vector_length);
1860
+ vuint64m1_t max_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, max_indices_u64m1, max_match_b64,
1861
+ max_vector_length);
1862
+ *max_value_ptr = max_val;
1863
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1864
+ __riscv_vredminu_vs_u64m1_u64m1(max_cands_u64m1, init_umax_u64m1, max_vector_length));
1865
+ }
1866
+
1867
+ NK_INTERNAL void nk_reduce_minmax_i64_rvv_strided_( //
1868
+ nk_i64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1869
+ nk_i64_t *min_value_ptr, nk_size_t *min_index_ptr, //
1870
+ nk_i64_t *max_value_ptr, nk_size_t *max_index_ptr) {
1871
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
1872
+ vint64m1_t min_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MAX, max_vector_length);
1873
+ vint64m1_t max_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MIN, max_vector_length);
1874
+ vuint64m1_t min_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1875
+ vuint64m1_t max_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1876
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1877
+
1878
+ nk_size_t offset = 0;
1879
+ for (nk_size_t vector_length; count > 0;
1880
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
1881
+ vector_length = __riscv_vsetvl_e64m1(count);
1882
+ vint64m1_t data_i64m1 = __riscv_vlse64_v_i64m1((nk_i64_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1883
+ vuint64m1_t pos_u64m1 = __riscv_vadd_vx_u64m1(__riscv_vid_v_u64m1(vector_length), (nk_u64_t)offset,
1884
+ vector_length);
1885
+
1886
+ vbool64_t less_b64 = __riscv_vmslt_vv_i64m1_b64(data_i64m1, min_i64m1, vector_length);
1887
+ min_i64m1 = __riscv_vmerge_vvm_i64m1_tu(min_i64m1, min_i64m1, data_i64m1, less_b64, vector_length);
1888
+ min_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(min_indices_u64m1, min_indices_u64m1, pos_u64m1, less_b64,
1889
+ vector_length);
1890
+
1891
+ vbool64_t greater_b64 = __riscv_vmslt_vv_i64m1_b64(max_i64m1, data_i64m1, vector_length);
1892
+ max_i64m1 = __riscv_vmerge_vvm_i64m1_tu(max_i64m1, max_i64m1, data_i64m1, greater_b64, vector_length);
1893
+ max_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(max_indices_u64m1, max_indices_u64m1, pos_u64m1, greater_b64,
1894
+ vector_length);
1895
+ }
1896
+
1897
+ // Horizontal reduction for min
1898
+ vint64m1_t init_max_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MAX, 1);
1899
+ nk_i64_t min_val = __riscv_vmv_x_s_i64m1_i64(
1900
+ __riscv_vredmin_vs_i64m1_i64m1(min_i64m1, init_max_i64m1, max_vector_length));
1901
+ vbool64_t min_match_b64 = __riscv_vmseq_vx_i64m1_b64(min_i64m1, min_val, max_vector_length);
1902
+ vuint64m1_t sentinel_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
1903
+ vuint64m1_t min_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, min_indices_u64m1, min_match_b64,
1904
+ max_vector_length);
1905
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
1906
+ *min_value_ptr = min_val;
1907
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1908
+ __riscv_vredminu_vs_u64m1_u64m1(min_cands_u64m1, init_umax_u64m1, max_vector_length));
1909
+
1910
+ // Horizontal reduction for max
1911
+ vint64m1_t init_min_i64m1 = __riscv_vmv_v_x_i64m1(NK_I64_MIN, 1);
1912
+ nk_i64_t max_val = __riscv_vmv_x_s_i64m1_i64(
1913
+ __riscv_vredmax_vs_i64m1_i64m1(max_i64m1, init_min_i64m1, max_vector_length));
1914
+ vbool64_t max_match_b64 = __riscv_vmseq_vx_i64m1_b64(max_i64m1, max_val, max_vector_length);
1915
+ vuint64m1_t max_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, max_indices_u64m1, max_match_b64,
1916
+ max_vector_length);
1917
+ *max_value_ptr = max_val;
1918
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
1919
+ __riscv_vredminu_vs_u64m1_u64m1(max_cands_u64m1, init_umax_u64m1, max_vector_length));
1920
+ }
1921
+
1922
+ NK_PUBLIC void nk_reduce_minmax_i64_rvv( //
1923
+ nk_i64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1924
+ nk_i64_t *min_value_ptr, nk_size_t *min_index_ptr, //
1925
+ nk_i64_t *max_value_ptr, nk_size_t *max_index_ptr) {
1926
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_i64_t);
1927
+ int aligned = (stride_bytes % sizeof(nk_i64_t) == 0);
1928
+
1929
+ if (count == 0)
1930
+ *min_value_ptr = NK_I64_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_I64_MIN,
1931
+ *max_index_ptr = NK_SIZE_MAX;
1932
+ else if (!aligned)
1933
+ nk_reduce_minmax_i64_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1934
+ max_index_ptr);
1935
+ else if (stride_elements == 1)
1936
+ nk_reduce_minmax_i64_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
1937
+ max_index_ptr);
1938
+ else
1939
+ nk_reduce_minmax_i64_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
1940
+ max_index_ptr);
1941
+ }
1942
+
1943
+ NK_INTERNAL void nk_reduce_moments_u64_rvv_contiguous_( //
1944
+ nk_u64_t const *data_ptr, nk_size_t count, //
1945
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1946
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
1947
+ vuint64m1_t sum_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1948
+ vuint64m1_t sumsq_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1949
+
1950
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
1951
+ vector_length = __riscv_vsetvl_e64m1(count);
1952
+ vuint64m1_t data_u64m1 = __riscv_vle64_v_u64m1(data_ptr, vector_length);
1953
+
1954
+ // Saturating unsigned sum
1955
+ sum_u64m1 = __riscv_vsaddu_vv_u64m1_tu(sum_u64m1, sum_u64m1, data_u64m1, vector_length);
1956
+
1957
+ // Sumsq: u64 × u64 with overflow detection via vmul + vmulhu
1958
+ vuint64m1_t product_low_u64m1 = __riscv_vmul_vv_u64m1(data_u64m1, data_u64m1, vector_length);
1959
+ vuint64m1_t product_high_u64m1 = __riscv_vmulhu_vv_u64m1(data_u64m1, data_u64m1, vector_length);
1960
+ vbool64_t overflow_b64 = __riscv_vmsne_vx_u64m1_b64(product_high_u64m1, 0, vector_length);
1961
+ vuint64m1_t squares_u64m1 = __riscv_vmerge_vxm_u64m1(product_low_u64m1, NK_U64_MAX, overflow_b64,
1962
+ vector_length);
1963
+ sumsq_u64m1 = __riscv_vsaddu_vv_u64m1_tu(sumsq_u64m1, sumsq_u64m1, squares_u64m1, vector_length);
1964
+ }
1965
+
1966
+ *sum_ptr = nk_reduce_vsaddu_u64m1_rvv_(sum_u64m1, max_vector_length);
1967
+ *sumsq_ptr = nk_reduce_vsaddu_u64m1_rvv_(sumsq_u64m1, max_vector_length);
1968
+ }
1969
+
1970
+ NK_INTERNAL void nk_reduce_moments_u64_rvv_strided_( //
1971
+ nk_u64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
1972
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
1973
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
1974
+ vuint64m1_t sum_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1975
+ vuint64m1_t sumsq_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
1976
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
1977
+
1978
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
1979
+ vector_length = __riscv_vsetvl_e64m1(count);
1980
+ vuint64m1_t data_u64m1 = __riscv_vlse64_v_u64m1((nk_u64_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
1981
+
1982
+ // Saturating unsigned sum
1983
+ sum_u64m1 = __riscv_vsaddu_vv_u64m1_tu(sum_u64m1, sum_u64m1, data_u64m1, vector_length);
1984
+
1985
+ // Sumsq: u64 × u64 with overflow detection via vmul + vmulhu
1986
+ vuint64m1_t product_low_u64m1 = __riscv_vmul_vv_u64m1(data_u64m1, data_u64m1, vector_length);
1987
+ vuint64m1_t product_high_u64m1 = __riscv_vmulhu_vv_u64m1(data_u64m1, data_u64m1, vector_length);
1988
+ vbool64_t overflow_b64 = __riscv_vmsne_vx_u64m1_b64(product_high_u64m1, 0, vector_length);
1989
+ vuint64m1_t squares_u64m1 = __riscv_vmerge_vxm_u64m1(product_low_u64m1, NK_U64_MAX, overflow_b64,
1990
+ vector_length);
1991
+ sumsq_u64m1 = __riscv_vsaddu_vv_u64m1_tu(sumsq_u64m1, sumsq_u64m1, squares_u64m1, vector_length);
1992
+ }
1993
+
1994
+ *sum_ptr = nk_reduce_vsaddu_u64m1_rvv_(sum_u64m1, max_vector_length);
1995
+ *sumsq_ptr = nk_reduce_vsaddu_u64m1_rvv_(sumsq_u64m1, max_vector_length);
1996
+ }
1997
+
1998
+ NK_PUBLIC void nk_reduce_moments_u64_rvv( //
1999
+ nk_u64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2000
+ nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
2001
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u64_t);
2002
+ int aligned = (stride_bytes % sizeof(nk_u64_t) == 0);
2003
+
2004
+ if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
2005
+ else if (!aligned) { nk_reduce_moments_u64_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
2006
+ else if (stride_elements == 1) { nk_reduce_moments_u64_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr); }
2007
+ else { nk_reduce_moments_u64_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr); }
2008
+ }
2009
+
2010
+ NK_INTERNAL void nk_reduce_minmax_u64_rvv_contiguous_( //
2011
+ nk_u64_t const *data_ptr, nk_size_t count, //
2012
+ nk_u64_t *min_value_ptr, nk_size_t *min_index_ptr, //
2013
+ nk_u64_t *max_value_ptr, nk_size_t *max_index_ptr) {
2014
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
2015
+ vuint64m1_t min_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
2016
+ vuint64m1_t max_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MIN, max_vector_length);
2017
+ vuint64m1_t min_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
2018
+ vuint64m1_t max_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
2019
+
2020
+ nk_size_t offset = 0;
2021
+ for (nk_size_t vector_length; count > 0;
2022
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
2023
+ vector_length = __riscv_vsetvl_e64m1(count);
2024
+ vuint64m1_t data_u64m1 = __riscv_vle64_v_u64m1(data_ptr, vector_length);
2025
+ vuint64m1_t pos_u64m1 = __riscv_vadd_vx_u64m1(__riscv_vid_v_u64m1(vector_length), (nk_u64_t)offset,
2026
+ vector_length);
2027
+
2028
+ vbool64_t less_b64 = __riscv_vmsltu_vv_u64m1_b64(data_u64m1, min_u64m1, vector_length);
2029
+ min_u64m1 = __riscv_vmerge_vvm_u64m1_tu(min_u64m1, min_u64m1, data_u64m1, less_b64, vector_length);
2030
+ min_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(min_indices_u64m1, min_indices_u64m1, pos_u64m1, less_b64,
2031
+ vector_length);
2032
+
2033
+ vbool64_t greater_b64 = __riscv_vmsltu_vv_u64m1_b64(max_u64m1, data_u64m1, vector_length);
2034
+ max_u64m1 = __riscv_vmerge_vvm_u64m1_tu(max_u64m1, max_u64m1, data_u64m1, greater_b64, vector_length);
2035
+ max_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(max_indices_u64m1, max_indices_u64m1, pos_u64m1, greater_b64,
2036
+ vector_length);
2037
+ }
2038
+
2039
+ // Horizontal reduction for min
2040
+ vuint64m1_t init_max_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2041
+ nk_u64_t min_val = __riscv_vmv_x_s_u64m1_u64(
2042
+ __riscv_vredminu_vs_u64m1_u64m1(min_u64m1, init_max_u64m1, max_vector_length));
2043
+ vbool64_t min_match_b64 = __riscv_vmseq_vx_u64m1_b64(min_u64m1, min_val, max_vector_length);
2044
+ vuint64m1_t sentinel_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
2045
+ vuint64m1_t min_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, min_indices_u64m1, min_match_b64,
2046
+ max_vector_length);
2047
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2048
+ *min_value_ptr = min_val;
2049
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2050
+ __riscv_vredminu_vs_u64m1_u64m1(min_cands_u64m1, init_umax_u64m1, max_vector_length));
2051
+
2052
+ // Horizontal reduction for max
2053
+ vuint64m1_t init_min_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MIN, 1);
2054
+ nk_u64_t max_val = __riscv_vmv_x_s_u64m1_u64(
2055
+ __riscv_vredmaxu_vs_u64m1_u64m1(max_u64m1, init_min_u64m1, max_vector_length));
2056
+ vbool64_t max_match_b64 = __riscv_vmseq_vx_u64m1_b64(max_u64m1, max_val, max_vector_length);
2057
+ vuint64m1_t max_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, max_indices_u64m1, max_match_b64,
2058
+ max_vector_length);
2059
+ *max_value_ptr = max_val;
2060
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2061
+ __riscv_vredminu_vs_u64m1_u64m1(max_cands_u64m1, init_umax_u64m1, max_vector_length));
2062
+ }
2063
+
2064
+ NK_INTERNAL void nk_reduce_minmax_u64_rvv_strided_( //
2065
+ nk_u64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2066
+ nk_u64_t *min_value_ptr, nk_size_t *min_index_ptr, //
2067
+ nk_u64_t *max_value_ptr, nk_size_t *max_index_ptr) {
2068
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m1();
2069
+ vuint64m1_t min_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
2070
+ vuint64m1_t max_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MIN, max_vector_length);
2071
+ vuint64m1_t min_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
2072
+ vuint64m1_t max_indices_u64m1 = __riscv_vmv_v_x_u64m1(0, max_vector_length);
2073
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2074
+
2075
+ nk_size_t offset = 0;
2076
+ for (nk_size_t vector_length; count > 0;
2077
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
2078
+ vector_length = __riscv_vsetvl_e64m1(count);
2079
+ vuint64m1_t data_u64m1 = __riscv_vlse64_v_u64m1((nk_u64_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2080
+ vuint64m1_t pos_u64m1 = __riscv_vadd_vx_u64m1(__riscv_vid_v_u64m1(vector_length), (nk_u64_t)offset,
2081
+ vector_length);
2082
+
2083
+ vbool64_t less_b64 = __riscv_vmsltu_vv_u64m1_b64(data_u64m1, min_u64m1, vector_length);
2084
+ min_u64m1 = __riscv_vmerge_vvm_u64m1_tu(min_u64m1, min_u64m1, data_u64m1, less_b64, vector_length);
2085
+ min_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(min_indices_u64m1, min_indices_u64m1, pos_u64m1, less_b64,
2086
+ vector_length);
2087
+
2088
+ vbool64_t greater_b64 = __riscv_vmsltu_vv_u64m1_b64(max_u64m1, data_u64m1, vector_length);
2089
+ max_u64m1 = __riscv_vmerge_vvm_u64m1_tu(max_u64m1, max_u64m1, data_u64m1, greater_b64, vector_length);
2090
+ max_indices_u64m1 = __riscv_vmerge_vvm_u64m1_tu(max_indices_u64m1, max_indices_u64m1, pos_u64m1, greater_b64,
2091
+ vector_length);
2092
+ }
2093
+
2094
+ // Horizontal reduction for min
2095
+ vuint64m1_t init_max_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2096
+ nk_u64_t min_val = __riscv_vmv_x_s_u64m1_u64(
2097
+ __riscv_vredminu_vs_u64m1_u64m1(min_u64m1, init_max_u64m1, max_vector_length));
2098
+ vbool64_t min_match_b64 = __riscv_vmseq_vx_u64m1_b64(min_u64m1, min_val, max_vector_length);
2099
+ vuint64m1_t sentinel_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, max_vector_length);
2100
+ vuint64m1_t min_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, min_indices_u64m1, min_match_b64,
2101
+ max_vector_length);
2102
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2103
+ *min_value_ptr = min_val;
2104
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2105
+ __riscv_vredminu_vs_u64m1_u64m1(min_cands_u64m1, init_umax_u64m1, max_vector_length));
2106
+
2107
+ // Horizontal reduction for max
2108
+ vuint64m1_t init_min_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MIN, 1);
2109
+ nk_u64_t max_val = __riscv_vmv_x_s_u64m1_u64(
2110
+ __riscv_vredmaxu_vs_u64m1_u64m1(max_u64m1, init_min_u64m1, max_vector_length));
2111
+ vbool64_t max_match_b64 = __riscv_vmseq_vx_u64m1_b64(max_u64m1, max_val, max_vector_length);
2112
+ vuint64m1_t max_cands_u64m1 = __riscv_vmerge_vvm_u64m1(sentinel_u64m1, max_indices_u64m1, max_match_b64,
2113
+ max_vector_length);
2114
+ *max_value_ptr = max_val;
2115
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2116
+ __riscv_vredminu_vs_u64m1_u64m1(max_cands_u64m1, init_umax_u64m1, max_vector_length));
2117
+ }
2118
+
2119
+ NK_PUBLIC void nk_reduce_minmax_u64_rvv( //
2120
+ nk_u64_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2121
+ nk_u64_t *min_value_ptr, nk_size_t *min_index_ptr, //
2122
+ nk_u64_t *max_value_ptr, nk_size_t *max_index_ptr) {
2123
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_u64_t);
2124
+ int aligned = (stride_bytes % sizeof(nk_u64_t) == 0);
2125
+
2126
+ if (count == 0)
2127
+ *min_value_ptr = NK_U64_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_U64_MIN,
2128
+ *max_index_ptr = NK_SIZE_MAX;
2129
+ else if (!aligned)
2130
+ nk_reduce_minmax_u64_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2131
+ max_index_ptr);
2132
+ else if (stride_elements == 1)
2133
+ nk_reduce_minmax_u64_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
2134
+ max_index_ptr);
2135
+ else
2136
+ nk_reduce_minmax_u64_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2137
+ max_index_ptr);
2138
+ }
2139
+
2140
+ NK_INTERNAL void nk_reduce_moments_bf16_rvv_contiguous_( //
2141
+ nk_bf16_t const *data_ptr, nk_size_t count, //
2142
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2143
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
2144
+ vfloat64m4_t sum_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2145
+ vfloat64m4_t sumsq_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2146
+
2147
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
2148
+ vector_length = __riscv_vsetvl_e16m1(count);
2149
+ vuint16m1_t data_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)data_ptr, vector_length);
2150
+
2151
+ // Convert bf16 → f32 (m1 → m2)
2152
+ vfloat32m2_t data_f32m2 = nk_bf16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2153
+
2154
+ // Widen f32 → f64 (m2 → m4)
2155
+ vfloat64m4_t data_f64m4 = __riscv_vfwcvt_f_f_v_f64m4(data_f32m2, vector_length);
2156
+ sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(sum_f64m4, sum_f64m4, data_f64m4, vector_length);
2157
+
2158
+ // Sumsq via widening FMA: f32×f32 → f64
2159
+ sumsq_f64m4 = __riscv_vfwmacc_vv_f64m4_tu(sumsq_f64m4, data_f32m2, data_f32m2, vector_length);
2160
+ }
2161
+
2162
+ // Horizontal reduction
2163
+ vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
2164
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sum_f64m4, zero_f64m1, max_vector_length)),
2165
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(
2166
+ __riscv_vfredusum_vs_f64m4_f64m1(sumsq_f64m4, zero_f64m1, max_vector_length));
2167
+ }
2168
+
2169
+ NK_INTERNAL void nk_reduce_moments_bf16_rvv_strided_( //
2170
+ nk_bf16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2171
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2172
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
2173
+ vfloat64m4_t sum_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2174
+ vfloat64m4_t sumsq_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2175
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2176
+
2177
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
2178
+ vector_length = __riscv_vsetvl_e16m1(count);
2179
+ vuint16m1_t data_u16m1 = __riscv_vlse16_v_u16m1((nk_u16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2180
+
2181
+ // Convert bf16 → f32 (m1 → m2)
2182
+ vfloat32m2_t data_f32m2 = nk_bf16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2183
+
2184
+ // Widen f32 → f64 (m2 → m4)
2185
+ vfloat64m4_t data_f64m4 = __riscv_vfwcvt_f_f_v_f64m4(data_f32m2, vector_length);
2186
+ sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(sum_f64m4, sum_f64m4, data_f64m4, vector_length);
2187
+
2188
+ // Sumsq via widening FMA: f32×f32 → f64
2189
+ sumsq_f64m4 = __riscv_vfwmacc_vv_f64m4_tu(sumsq_f64m4, data_f32m2, data_f32m2, vector_length);
2190
+ }
2191
+
2192
+ // Horizontal reduction
2193
+ vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
2194
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sum_f64m4, zero_f64m1, max_vector_length)),
2195
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(
2196
+ __riscv_vfredusum_vs_f64m4_f64m1(sumsq_f64m4, zero_f64m1, max_vector_length));
2197
+ }
2198
+
2199
+ NK_PUBLIC void nk_reduce_moments_bf16_rvv( //
2200
+ nk_bf16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2201
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2202
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_bf16_t);
2203
+ int aligned = (stride_bytes % sizeof(nk_bf16_t) == 0);
2204
+
2205
+ if (count == 0) *sum_ptr = 0.0f, *sumsq_ptr = 0.0f;
2206
+ else if (!aligned) nk_reduce_moments_bf16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2207
+ else if (stride_elements == 1) nk_reduce_moments_bf16_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
2208
+ else nk_reduce_moments_bf16_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2209
+ }
2210
+
2211
+ NK_INTERNAL void nk_reduce_minmax_bf16_rvv_contiguous_( //
2212
+ nk_bf16_t const *data_ptr, nk_size_t count, //
2213
+ nk_bf16_t *min_value_ptr, nk_size_t *min_index_ptr, //
2214
+ nk_bf16_t *max_value_ptr, nk_size_t *max_index_ptr) {
2215
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
2216
+ vuint16m1_t min_u16m1 = __riscv_vmv_v_x_u16m1(0x7F80, max_vector_length); // +inf in bf16
2217
+ vuint16m1_t max_u16m1 = __riscv_vmv_v_x_u16m1(0xFF80, max_vector_length); // -inf in bf16
2218
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2219
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2220
+
2221
+ nk_size_t offset = 0;
2222
+ for (nk_size_t vector_length; count > 0;
2223
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
2224
+ vector_length = __riscv_vsetvl_e16m1(count);
2225
+ vuint16m1_t data_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)data_ptr, vector_length);
2226
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
2227
+ vector_length);
2228
+
2229
+ // Convert to f32 for comparison
2230
+ vfloat32m2_t data_f32m2 = nk_bf16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2231
+ vfloat32m2_t min_f32m2 = nk_bf16m1_to_f32m2_rvv_(min_u16m1, vector_length);
2232
+ vfloat32m2_t max_f32m2 = nk_bf16m1_to_f32m2_rvv_(max_u16m1, vector_length);
2233
+
2234
+ vbool16_t less_b16 = __riscv_vmflt_vv_f32m2_b16(data_f32m2, min_f32m2, vector_length);
2235
+ min_u16m1 = __riscv_vmerge_vvm_u16m1_tu(min_u16m1, min_u16m1, data_u16m1, less_b16, vector_length);
2236
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
2237
+ vector_length);
2238
+
2239
+ vbool16_t greater_b16 = __riscv_vmflt_vv_f32m2_b16(max_f32m2, data_f32m2, vector_length);
2240
+ max_u16m1 = __riscv_vmerge_vvm_u16m1_tu(max_u16m1, max_u16m1, data_u16m1, greater_b16, vector_length);
2241
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
2242
+ vector_length);
2243
+ }
2244
+
2245
+ // Horizontal reduction
2246
+ vfloat32m2_t final_min_f32m2 = nk_bf16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2247
+ vfloat32m1_t init_max_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, 1);
2248
+ nk_f32_t min_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2249
+ __riscv_vfredmin_vs_f32m2_f32m1(final_min_f32m2, init_max_f32m1, max_vector_length));
2250
+ vfloat32m2_t final_max_f32m2 = nk_bf16m1_to_f32m2_rvv_(max_u16m1, max_vector_length);
2251
+ vfloat32m1_t init_min_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, 1);
2252
+ nk_f32_t max_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2253
+ __riscv_vfredmax_vs_f32m2_f32m1(final_max_f32m2, init_min_f32m1, max_vector_length));
2254
+ if (min_val_f32 == NK_F32_MAX && max_val_f32 == NK_F32_MIN) {
2255
+ *min_value_ptr = NK_BF16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_BF16_MIN,
2256
+ *max_index_ptr = NK_SIZE_MAX;
2257
+ return;
2258
+ }
2259
+
2260
+ vfloat32m2_t converted_f32m2 = nk_bf16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2261
+ vbool16_t min_match_b16 = __riscv_vmfeq_vf_f32m2_b16(converted_f32m2, min_val_f32, max_vector_length);
2262
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
2263
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
2264
+ max_vector_length);
2265
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2266
+
2267
+ nk_u16_t min_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2268
+ min_u16m1, (nk_size_t)__riscv_vfirst_m_b16(min_match_b16, max_vector_length), max_vector_length));
2269
+ *min_value_ptr = *(nk_bf16_t *)&min_raw;
2270
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2271
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
2272
+
2273
+ vbool16_t max_match_b16 = __riscv_vmfeq_vf_f32m2_b16(nk_bf16m1_to_f32m2_rvv_(max_u16m1, max_vector_length),
2274
+ max_val_f32, max_vector_length);
2275
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
2276
+ max_vector_length);
2277
+
2278
+ nk_u16_t max_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2279
+ max_u16m1, (nk_size_t)__riscv_vfirst_m_b16(max_match_b16, max_vector_length), max_vector_length));
2280
+ *max_value_ptr = *(nk_bf16_t *)&max_raw;
2281
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2282
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
2283
+ }
2284
+
2285
+ NK_INTERNAL void nk_reduce_minmax_bf16_rvv_strided_( //
2286
+ nk_bf16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2287
+ nk_bf16_t *min_value_ptr, nk_size_t *min_index_ptr, //
2288
+ nk_bf16_t *max_value_ptr, nk_size_t *max_index_ptr) {
2289
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
2290
+ vuint16m1_t min_u16m1 = __riscv_vmv_v_x_u16m1(0x7F80, max_vector_length); // +inf in bf16
2291
+ vuint16m1_t max_u16m1 = __riscv_vmv_v_x_u16m1(0xFF80, max_vector_length); // -inf in bf16
2292
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2293
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2294
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2295
+
2296
+ nk_size_t offset = 0;
2297
+ for (nk_size_t vector_length; count > 0;
2298
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
2299
+ vector_length = __riscv_vsetvl_e16m1(count);
2300
+ vuint16m1_t data_u16m1 = __riscv_vlse16_v_u16m1((nk_u16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2301
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
2302
+ vector_length);
2303
+
2304
+ // Convert to f32 for comparison
2305
+ vfloat32m2_t data_f32m2 = nk_bf16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2306
+ vfloat32m2_t min_f32m2 = nk_bf16m1_to_f32m2_rvv_(min_u16m1, vector_length);
2307
+ vfloat32m2_t max_f32m2 = nk_bf16m1_to_f32m2_rvv_(max_u16m1, vector_length);
2308
+
2309
+ vbool16_t less_b16 = __riscv_vmflt_vv_f32m2_b16(data_f32m2, min_f32m2, vector_length);
2310
+ min_u16m1 = __riscv_vmerge_vvm_u16m1_tu(min_u16m1, min_u16m1, data_u16m1, less_b16, vector_length);
2311
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
2312
+ vector_length);
2313
+
2314
+ vbool16_t greater_b16 = __riscv_vmflt_vv_f32m2_b16(max_f32m2, data_f32m2, vector_length);
2315
+ max_u16m1 = __riscv_vmerge_vvm_u16m1_tu(max_u16m1, max_u16m1, data_u16m1, greater_b16, vector_length);
2316
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
2317
+ vector_length);
2318
+ }
2319
+
2320
+ // Horizontal reduction (same as contiguous)
2321
+ vfloat32m2_t final_min_f32m2 = nk_bf16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2322
+ vfloat32m1_t init_max_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, 1);
2323
+ nk_f32_t min_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2324
+ __riscv_vfredmin_vs_f32m2_f32m1(final_min_f32m2, init_max_f32m1, max_vector_length));
2325
+ vfloat32m2_t final_max_f32m2 = nk_bf16m1_to_f32m2_rvv_(max_u16m1, max_vector_length);
2326
+ vfloat32m1_t init_min_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, 1);
2327
+ nk_f32_t max_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2328
+ __riscv_vfredmax_vs_f32m2_f32m1(final_max_f32m2, init_min_f32m1, max_vector_length));
2329
+ if (min_val_f32 == NK_F32_MAX && max_val_f32 == NK_F32_MIN) {
2330
+ *min_value_ptr = NK_BF16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_BF16_MIN,
2331
+ *max_index_ptr = NK_SIZE_MAX;
2332
+ return;
2333
+ }
2334
+
2335
+ vfloat32m2_t converted_f32m2 = nk_bf16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2336
+ vbool16_t min_match_b16 = __riscv_vmfeq_vf_f32m2_b16(converted_f32m2, min_val_f32, max_vector_length);
2337
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
2338
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
2339
+ max_vector_length);
2340
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2341
+
2342
+ nk_u16_t min_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2343
+ min_u16m1, (nk_size_t)__riscv_vfirst_m_b16(min_match_b16, max_vector_length), max_vector_length));
2344
+ *min_value_ptr = *(nk_bf16_t *)&min_raw;
2345
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2346
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
2347
+
2348
+ vbool16_t max_match_b16 = __riscv_vmfeq_vf_f32m2_b16(nk_bf16m1_to_f32m2_rvv_(max_u16m1, max_vector_length),
2349
+ max_val_f32, max_vector_length);
2350
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
2351
+ max_vector_length);
2352
+
2353
+ nk_u16_t max_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2354
+ max_u16m1, (nk_size_t)__riscv_vfirst_m_b16(max_match_b16, max_vector_length), max_vector_length));
2355
+ *max_value_ptr = *(nk_bf16_t *)&max_raw;
2356
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2357
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
2358
+ }
2359
+
2360
+ NK_PUBLIC void nk_reduce_minmax_bf16_rvv( //
2361
+ nk_bf16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2362
+ nk_bf16_t *min_value_ptr, nk_size_t *min_index_ptr, //
2363
+ nk_bf16_t *max_value_ptr, nk_size_t *max_index_ptr) {
2364
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_bf16_t);
2365
+ int aligned = (stride_bytes % sizeof(nk_bf16_t) == 0);
2366
+
2367
+ if (count == 0)
2368
+ *min_value_ptr = NK_BF16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_BF16_MIN,
2369
+ *max_index_ptr = NK_SIZE_MAX;
2370
+ else if (!aligned)
2371
+ nk_reduce_minmax_bf16_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2372
+ max_index_ptr);
2373
+ else if (stride_elements == 1)
2374
+ nk_reduce_minmax_bf16_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
2375
+ max_index_ptr);
2376
+ else
2377
+ nk_reduce_minmax_bf16_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2378
+ max_index_ptr);
2379
+ }
2380
+
2381
+ NK_INTERNAL void nk_reduce_moments_f16_rvv_contiguous_( //
2382
+ nk_f16_t const *data_ptr, nk_size_t count, //
2383
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2384
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
2385
+ vfloat64m4_t sum_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2386
+ vfloat64m4_t sumsq_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2387
+
2388
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
2389
+ vector_length = __riscv_vsetvl_e16m1(count);
2390
+ vuint16m1_t data_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)data_ptr, vector_length);
2391
+
2392
+ // Convert f16 → f32 (m1 → m2)
2393
+ vfloat32m2_t data_f32m2 = nk_f16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2394
+
2395
+ // Widen f32 → f64 (m2 → m4)
2396
+ vfloat64m4_t data_f64m4 = __riscv_vfwcvt_f_f_v_f64m4(data_f32m2, vector_length);
2397
+ sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(sum_f64m4, sum_f64m4, data_f64m4, vector_length);
2398
+
2399
+ // Sumsq via widening FMA: f32×f32 → f64
2400
+ sumsq_f64m4 = __riscv_vfwmacc_vv_f64m4_tu(sumsq_f64m4, data_f32m2, data_f32m2, vector_length);
2401
+ }
2402
+
2403
+ // Horizontal reduction
2404
+ vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
2405
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sum_f64m4, zero_f64m1, max_vector_length)),
2406
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(
2407
+ __riscv_vfredusum_vs_f64m4_f64m1(sumsq_f64m4, zero_f64m1, max_vector_length));
2408
+ }
2409
+
2410
+ NK_INTERNAL void nk_reduce_moments_f16_rvv_strided_( //
2411
+ nk_f16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2412
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2413
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
2414
+ vfloat64m4_t sum_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2415
+ vfloat64m4_t sumsq_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, max_vector_length);
2416
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2417
+
2418
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
2419
+ vector_length = __riscv_vsetvl_e16m1(count);
2420
+ vuint16m1_t data_u16m1 = __riscv_vlse16_v_u16m1((nk_u16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2421
+
2422
+ // Convert f16 → f32 (m1 → m2)
2423
+ vfloat32m2_t data_f32m2 = nk_f16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2424
+
2425
+ // Widen f32 → f64 (m2 → m4)
2426
+ vfloat64m4_t data_f64m4 = __riscv_vfwcvt_f_f_v_f64m4(data_f32m2, vector_length);
2427
+ sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(sum_f64m4, sum_f64m4, data_f64m4, vector_length);
2428
+
2429
+ // Sumsq via widening FMA: f32×f32 → f64
2430
+ sumsq_f64m4 = __riscv_vfwmacc_vv_f64m4_tu(sumsq_f64m4, data_f32m2, data_f32m2, vector_length);
2431
+ }
2432
+
2433
+ // Horizontal reduction
2434
+ vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
2435
+ *sum_ptr = __riscv_vfmv_f_s_f64m1_f64(__riscv_vfredusum_vs_f64m4_f64m1(sum_f64m4, zero_f64m1, max_vector_length)),
2436
+ *sumsq_ptr = __riscv_vfmv_f_s_f64m1_f64(
2437
+ __riscv_vfredusum_vs_f64m4_f64m1(sumsq_f64m4, zero_f64m1, max_vector_length));
2438
+ }
2439
+
2440
+ NK_PUBLIC void nk_reduce_moments_f16_rvv( //
2441
+ nk_f16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2442
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2443
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_f16_t);
2444
+ int aligned = (stride_bytes % sizeof(nk_f16_t) == 0);
2445
+
2446
+ if (count == 0) *sum_ptr = 0.0f, *sumsq_ptr = 0.0f;
2447
+ else if (!aligned) nk_reduce_moments_f16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2448
+ else if (stride_elements == 1) nk_reduce_moments_f16_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
2449
+ else nk_reduce_moments_f16_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2450
+ }
2451
+
2452
+ NK_INTERNAL void nk_reduce_minmax_f16_rvv_contiguous_( //
2453
+ nk_f16_t const *data_ptr, nk_size_t count, //
2454
+ nk_f16_t *min_value_ptr, nk_size_t *min_index_ptr, //
2455
+ nk_f16_t *max_value_ptr, nk_size_t *max_index_ptr) {
2456
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
2457
+ vuint16m1_t min_u16m1 = __riscv_vmv_v_x_u16m1(0x7C00, max_vector_length); // +inf in f16
2458
+ vuint16m1_t max_u16m1 = __riscv_vmv_v_x_u16m1(0xFC00, max_vector_length); // -inf in f16
2459
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2460
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2461
+
2462
+ nk_size_t offset = 0;
2463
+ for (nk_size_t vector_length; count > 0;
2464
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
2465
+ vector_length = __riscv_vsetvl_e16m1(count);
2466
+ vuint16m1_t data_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)data_ptr, vector_length);
2467
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
2468
+ vector_length);
2469
+
2470
+ // Convert to f32 for comparison
2471
+ vfloat32m2_t data_f32m2 = nk_f16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2472
+ vfloat32m2_t min_f32m2 = nk_f16m1_to_f32m2_rvv_(min_u16m1, vector_length);
2473
+ vfloat32m2_t max_f32m2 = nk_f16m1_to_f32m2_rvv_(max_u16m1, vector_length);
2474
+
2475
+ vbool16_t less_b16 = __riscv_vmflt_vv_f32m2_b16(data_f32m2, min_f32m2, vector_length);
2476
+ min_u16m1 = __riscv_vmerge_vvm_u16m1_tu(min_u16m1, min_u16m1, data_u16m1, less_b16, vector_length);
2477
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
2478
+ vector_length);
2479
+
2480
+ vbool16_t greater_b16 = __riscv_vmflt_vv_f32m2_b16(max_f32m2, data_f32m2, vector_length);
2481
+ max_u16m1 = __riscv_vmerge_vvm_u16m1_tu(max_u16m1, max_u16m1, data_u16m1, greater_b16, vector_length);
2482
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
2483
+ vector_length);
2484
+ }
2485
+
2486
+ // Horizontal reduction
2487
+ vfloat32m2_t final_min_f32m2 = nk_f16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2488
+ vfloat32m1_t init_max_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, 1);
2489
+ nk_f32_t min_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2490
+ __riscv_vfredmin_vs_f32m2_f32m1(final_min_f32m2, init_max_f32m1, max_vector_length));
2491
+ vfloat32m2_t final_max_f32m2 = nk_f16m1_to_f32m2_rvv_(max_u16m1, max_vector_length);
2492
+ vfloat32m1_t init_min_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, 1);
2493
+ nk_f32_t max_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2494
+ __riscv_vfredmax_vs_f32m2_f32m1(final_max_f32m2, init_min_f32m1, max_vector_length));
2495
+ if (min_val_f32 == NK_F32_MAX && max_val_f32 == NK_F32_MIN) {
2496
+ *min_value_ptr = NK_F16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_F16_MIN,
2497
+ *max_index_ptr = NK_SIZE_MAX;
2498
+ return;
2499
+ }
2500
+
2501
+ vfloat32m2_t converted_f32m2 = nk_f16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2502
+ vbool16_t min_match_b16 = __riscv_vmfeq_vf_f32m2_b16(converted_f32m2, min_val_f32, max_vector_length);
2503
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
2504
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
2505
+ max_vector_length);
2506
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2507
+
2508
+ nk_u16_t min_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2509
+ min_u16m1, (nk_size_t)__riscv_vfirst_m_b16(min_match_b16, max_vector_length), max_vector_length));
2510
+ *min_value_ptr = *(nk_f16_t *)&min_raw;
2511
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2512
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
2513
+
2514
+ vbool16_t max_match_b16 = __riscv_vmfeq_vf_f32m2_b16(nk_f16m1_to_f32m2_rvv_(max_u16m1, max_vector_length),
2515
+ max_val_f32, max_vector_length);
2516
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
2517
+ max_vector_length);
2518
+
2519
+ nk_u16_t max_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2520
+ max_u16m1, (nk_size_t)__riscv_vfirst_m_b16(max_match_b16, max_vector_length), max_vector_length));
2521
+ *max_value_ptr = *(nk_f16_t *)&max_raw;
2522
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2523
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
2524
+ }
2525
+
2526
+ NK_INTERNAL void nk_reduce_minmax_f16_rvv_strided_( //
2527
+ nk_f16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2528
+ nk_f16_t *min_value_ptr, nk_size_t *min_index_ptr, //
2529
+ nk_f16_t *max_value_ptr, nk_size_t *max_index_ptr) {
2530
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e16m1();
2531
+ vuint16m1_t min_u16m1 = __riscv_vmv_v_x_u16m1(0x7C00, max_vector_length); // +inf in f16
2532
+ vuint16m1_t max_u16m1 = __riscv_vmv_v_x_u16m1(0xFC00, max_vector_length); // -inf in f16
2533
+ vuint64m4_t min_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2534
+ vuint64m4_t max_indices_u64m4 = __riscv_vmv_v_x_u64m4(0, max_vector_length);
2535
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2536
+
2537
+ nk_size_t offset = 0;
2538
+ for (nk_size_t vector_length; count > 0;
2539
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
2540
+ vector_length = __riscv_vsetvl_e16m1(count);
2541
+ vuint16m1_t data_u16m1 = __riscv_vlse16_v_u16m1((nk_u16_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2542
+ vuint64m4_t pos_u64m4 = __riscv_vadd_vx_u64m4(__riscv_vid_v_u64m4(vector_length), (nk_u64_t)offset,
2543
+ vector_length);
2544
+
2545
+ // Convert to f32 for comparison
2546
+ vfloat32m2_t data_f32m2 = nk_f16m1_to_f32m2_rvv_(data_u16m1, vector_length);
2547
+ vfloat32m2_t min_f32m2 = nk_f16m1_to_f32m2_rvv_(min_u16m1, vector_length);
2548
+ vfloat32m2_t max_f32m2 = nk_f16m1_to_f32m2_rvv_(max_u16m1, vector_length);
2549
+
2550
+ vbool16_t less_b16 = __riscv_vmflt_vv_f32m2_b16(data_f32m2, min_f32m2, vector_length);
2551
+ min_u16m1 = __riscv_vmerge_vvm_u16m1_tu(min_u16m1, min_u16m1, data_u16m1, less_b16, vector_length);
2552
+ min_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(min_indices_u64m4, min_indices_u64m4, pos_u64m4, less_b16,
2553
+ vector_length);
2554
+
2555
+ vbool16_t greater_b16 = __riscv_vmflt_vv_f32m2_b16(max_f32m2, data_f32m2, vector_length);
2556
+ max_u16m1 = __riscv_vmerge_vvm_u16m1_tu(max_u16m1, max_u16m1, data_u16m1, greater_b16, vector_length);
2557
+ max_indices_u64m4 = __riscv_vmerge_vvm_u64m4_tu(max_indices_u64m4, max_indices_u64m4, pos_u64m4, greater_b16,
2558
+ vector_length);
2559
+ }
2560
+
2561
+ // Horizontal reduction (same as contiguous)
2562
+ vfloat32m2_t final_min_f32m2 = nk_f16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2563
+ vfloat32m1_t init_max_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MAX, 1);
2564
+ nk_f32_t min_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2565
+ __riscv_vfredmin_vs_f32m2_f32m1(final_min_f32m2, init_max_f32m1, max_vector_length));
2566
+ vfloat32m2_t final_max_f32m2 = nk_f16m1_to_f32m2_rvv_(max_u16m1, max_vector_length);
2567
+ vfloat32m1_t init_min_f32m1 = __riscv_vfmv_v_f_f32m1(NK_F32_MIN, 1);
2568
+ nk_f32_t max_val_f32 = __riscv_vfmv_f_s_f32m1_f32(
2569
+ __riscv_vfredmax_vs_f32m2_f32m1(final_max_f32m2, init_min_f32m1, max_vector_length));
2570
+ if (min_val_f32 == NK_F32_MAX && max_val_f32 == NK_F32_MIN) {
2571
+ *min_value_ptr = NK_F16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_F16_MIN,
2572
+ *max_index_ptr = NK_SIZE_MAX;
2573
+ return;
2574
+ }
2575
+
2576
+ vfloat32m2_t converted_f32m2 = nk_f16m1_to_f32m2_rvv_(min_u16m1, max_vector_length);
2577
+ vbool16_t min_match_b16 = __riscv_vmfeq_vf_f32m2_b16(converted_f32m2, min_val_f32, max_vector_length);
2578
+ vuint64m4_t sentinel_u64m4 = __riscv_vmv_v_x_u64m4(NK_U64_MAX, max_vector_length);
2579
+ vuint64m4_t min_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, min_indices_u64m4, min_match_b16,
2580
+ max_vector_length);
2581
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2582
+
2583
+ nk_u16_t min_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2584
+ min_u16m1, (nk_size_t)__riscv_vfirst_m_b16(min_match_b16, max_vector_length), max_vector_length));
2585
+ *min_value_ptr = *(nk_f16_t *)&min_raw;
2586
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2587
+ __riscv_vredminu_vs_u64m4_u64m1(min_cands_u64m4, init_umax_u64m1, max_vector_length));
2588
+
2589
+ vbool16_t max_match_b16 = __riscv_vmfeq_vf_f32m2_b16(nk_f16m1_to_f32m2_rvv_(max_u16m1, max_vector_length),
2590
+ max_val_f32, max_vector_length);
2591
+ vuint64m4_t max_cands_u64m4 = __riscv_vmerge_vvm_u64m4(sentinel_u64m4, max_indices_u64m4, max_match_b16,
2592
+ max_vector_length);
2593
+
2594
+ nk_u16_t max_raw = __riscv_vmv_x_s_u16m1_u16(__riscv_vslidedown_vx_u16m1(
2595
+ max_u16m1, (nk_size_t)__riscv_vfirst_m_b16(max_match_b16, max_vector_length), max_vector_length));
2596
+ *max_value_ptr = *(nk_f16_t *)&max_raw;
2597
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2598
+ __riscv_vredminu_vs_u64m4_u64m1(max_cands_u64m4, init_umax_u64m1, max_vector_length));
2599
+ }
2600
+
2601
+ NK_PUBLIC void nk_reduce_minmax_f16_rvv( //
2602
+ nk_f16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2603
+ nk_f16_t *min_value_ptr, nk_size_t *min_index_ptr, //
2604
+ nk_f16_t *max_value_ptr, nk_size_t *max_index_ptr) {
2605
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_f16_t);
2606
+ int aligned = (stride_bytes % sizeof(nk_f16_t) == 0);
2607
+
2608
+ if (count == 0)
2609
+ *min_value_ptr = NK_F16_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_F16_MIN,
2610
+ *max_index_ptr = NK_SIZE_MAX;
2611
+ else if (!aligned)
2612
+ nk_reduce_minmax_f16_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2613
+ max_index_ptr);
2614
+ else if (stride_elements == 1)
2615
+ nk_reduce_minmax_f16_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
2616
+ max_index_ptr);
2617
+ else
2618
+ nk_reduce_minmax_f16_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2619
+ max_index_ptr);
2620
+ }
2621
+
2622
+ NK_INTERNAL void nk_reduce_moments_e4m3_rvv_contiguous_( //
2623
+ nk_e4m3_t const *data_ptr, nk_size_t count, //
2624
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2625
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
2626
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2627
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2628
+
2629
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
2630
+ vector_length = __riscv_vsetvl_e8m1(count);
2631
+ vuint8m1_t data_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
2632
+
2633
+ // Convert e4m3 → f32 (m1 → m4)
2634
+ vfloat32m4_t data_f32m4 = nk_e4m3m1_to_f32m4_rvv_(data_u8m1, vector_length);
2635
+
2636
+ // Accumulate at f32 precision
2637
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
2638
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
2639
+ }
2640
+
2641
+ // Horizontal reduction
2642
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
2643
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
2644
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
2645
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
2646
+ }
2647
+
2648
+ NK_INTERNAL void nk_reduce_moments_e4m3_rvv_strided_( //
2649
+ nk_e4m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2650
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2651
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
2652
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2653
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2654
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2655
+
2656
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
2657
+ vector_length = __riscv_vsetvl_e8m1(count);
2658
+ vuint8m1_t data_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2659
+
2660
+ // Convert e4m3 → f32 (m1 → m4)
2661
+ vfloat32m4_t data_f32m4 = nk_e4m3m1_to_f32m4_rvv_(data_u8m1, vector_length);
2662
+
2663
+ // Accumulate at f32 precision
2664
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
2665
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
2666
+ }
2667
+
2668
+ // Horizontal reduction
2669
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
2670
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
2671
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
2672
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
2673
+ }
2674
+
2675
+ NK_PUBLIC void nk_reduce_moments_e4m3_rvv( //
2676
+ nk_e4m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2677
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2678
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e4m3_t);
2679
+ int aligned = (stride_bytes % sizeof(nk_e4m3_t) == 0);
2680
+
2681
+ if (count == 0) *sum_ptr = 0.0f, *sumsq_ptr = 0.0f;
2682
+ else if (!aligned) nk_reduce_moments_e4m3_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2683
+ else if (stride_elements == 1) nk_reduce_moments_e4m3_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
2684
+ else nk_reduce_moments_e4m3_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2685
+ }
2686
+
2687
+ NK_INTERNAL void nk_reduce_minmax_e4m3_rvv_contiguous_( //
2688
+ nk_e4m3_t const *data_ptr, nk_size_t count, //
2689
+ nk_e4m3_t *min_value_ptr, nk_size_t *min_index_ptr, //
2690
+ nk_e4m3_t *max_value_ptr, nk_size_t *max_index_ptr) {
2691
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
2692
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, max_vector_length); // Largest comparable
2693
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length); // Smallest comparable
2694
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
2695
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
2696
+
2697
+ nk_size_t offset = 0;
2698
+ for (nk_size_t vector_length; count > 0;
2699
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
2700
+ vector_length = __riscv_vsetvl_e8m1(count);
2701
+ vuint8m1_t raw_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
2702
+
2703
+ // Convert to comparable form
2704
+ vuint8m1_t comparable_u8m1 = nk_fp8m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
2705
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
2706
+ vector_length);
2707
+
2708
+ // Detect E4M3 NaN: comparable == 0x00 (neg NaN) or comparable == 0xFF (pos NaN)
2709
+ vbool8_t nan_low_b8 = __riscv_vmseq_vx_u8m1_b8(comparable_u8m1, 0x00, vector_length);
2710
+ vbool8_t nan_high_b8 = __riscv_vmseq_vx_u8m1_b8(comparable_u8m1, 0xFF, vector_length);
2711
+ vbool8_t is_nan_b8 = __riscv_vmor_mm_b8(nan_low_b8, nan_high_b8, vector_length);
2712
+ vuint8m1_t data_min_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0xFF, is_nan_b8, vector_length);
2713
+ vuint8m1_t data_max_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0x00, is_nan_b8, vector_length);
2714
+
2715
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(data_min_u8m1, min_u8m1, vector_length);
2716
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, data_min_u8m1, less_b8, vector_length);
2717
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
2718
+ vector_length);
2719
+
2720
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, data_max_u8m1, vector_length);
2721
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, data_max_u8m1, greater_b8, vector_length);
2722
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
2723
+ vector_length);
2724
+ }
2725
+
2726
+ // Horizontal reduction + convert back
2727
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, 1);
2728
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
2729
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
2730
+
2731
+ // All-NaN case
2732
+ if (min_comparable == 0xFF) {
2733
+ *min_value_ptr = (nk_e4m3_t)NK_E4M3_MAX, *min_index_ptr = NK_SIZE_MAX;
2734
+ *max_value_ptr = (nk_e4m3_t)NK_E4M3_MIN, *max_index_ptr = NK_SIZE_MAX;
2735
+ return;
2736
+ }
2737
+
2738
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
2739
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
2740
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
2741
+ max_vector_length);
2742
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2743
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2744
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
2745
+
2746
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
2747
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(min_vec_u8m1, 1);
2748
+ *min_value_ptr = (nk_e4m3_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
2749
+
2750
+ // Similar for max
2751
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
2752
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
2753
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
2754
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
2755
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
2756
+ max_vector_length);
2757
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2758
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
2759
+
2760
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
2761
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(max_vec_u8m1, 1);
2762
+ *max_value_ptr = (nk_e4m3_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
2763
+ }
2764
+
2765
+ NK_INTERNAL void nk_reduce_minmax_e4m3_rvv_strided_( //
2766
+ nk_e4m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2767
+ nk_e4m3_t *min_value_ptr, nk_size_t *min_index_ptr, //
2768
+ nk_e4m3_t *max_value_ptr, nk_size_t *max_index_ptr) {
2769
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
2770
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, max_vector_length);
2771
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length);
2772
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
2773
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
2774
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2775
+
2776
+ nk_size_t offset = 0;
2777
+ for (nk_size_t vector_length; count > 0;
2778
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
2779
+ vector_length = __riscv_vsetvl_e8m1(count);
2780
+ vuint8m1_t raw_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2781
+
2782
+ vuint8m1_t comparable_u8m1 = nk_fp8m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
2783
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
2784
+ vector_length);
2785
+
2786
+ // Detect E4M3 NaN: comparable == 0x00 (neg NaN) or comparable == 0xFF (pos NaN)
2787
+ vbool8_t nan_low_b8 = __riscv_vmseq_vx_u8m1_b8(comparable_u8m1, 0x00, vector_length);
2788
+ vbool8_t nan_high_b8 = __riscv_vmseq_vx_u8m1_b8(comparable_u8m1, 0xFF, vector_length);
2789
+ vbool8_t is_nan_b8 = __riscv_vmor_mm_b8(nan_low_b8, nan_high_b8, vector_length);
2790
+ vuint8m1_t data_min_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0xFF, is_nan_b8, vector_length);
2791
+ vuint8m1_t data_max_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0x00, is_nan_b8, vector_length);
2792
+
2793
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(data_min_u8m1, min_u8m1, vector_length);
2794
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, data_min_u8m1, less_b8, vector_length);
2795
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
2796
+ vector_length);
2797
+
2798
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, data_max_u8m1, vector_length);
2799
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, data_max_u8m1, greater_b8, vector_length);
2800
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
2801
+ vector_length);
2802
+ }
2803
+
2804
+ // Horizontal reduction (same as contiguous)
2805
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, 1);
2806
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
2807
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
2808
+
2809
+ // All-NaN case
2810
+ if (min_comparable == 0xFF) {
2811
+ *min_value_ptr = (nk_e4m3_t)NK_E4M3_MAX, *min_index_ptr = NK_SIZE_MAX;
2812
+ *max_value_ptr = (nk_e4m3_t)NK_E4M3_MIN, *max_index_ptr = NK_SIZE_MAX;
2813
+ return;
2814
+ }
2815
+
2816
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
2817
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
2818
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
2819
+ max_vector_length);
2820
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2821
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2822
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
2823
+
2824
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
2825
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(min_vec_u8m1, 1);
2826
+ *min_value_ptr = (nk_e4m3_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
2827
+
2828
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
2829
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
2830
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
2831
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
2832
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
2833
+ max_vector_length);
2834
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2835
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
2836
+
2837
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
2838
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(max_vec_u8m1, 1);
2839
+ *max_value_ptr = (nk_e4m3_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
2840
+ }
2841
+
2842
+ NK_PUBLIC void nk_reduce_minmax_e4m3_rvv( //
2843
+ nk_e4m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2844
+ nk_e4m3_t *min_value_ptr, nk_size_t *min_index_ptr, //
2845
+ nk_e4m3_t *max_value_ptr, nk_size_t *max_index_ptr) {
2846
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e4m3_t);
2847
+ int aligned = (stride_bytes % sizeof(nk_e4m3_t) == 0);
2848
+
2849
+ if (count == 0)
2850
+ *min_value_ptr = NK_E4M3_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_E4M3_MIN,
2851
+ *max_index_ptr = NK_SIZE_MAX;
2852
+ else if (!aligned)
2853
+ nk_reduce_minmax_e4m3_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2854
+ max_index_ptr);
2855
+ else if (stride_elements == 1)
2856
+ nk_reduce_minmax_e4m3_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
2857
+ max_index_ptr);
2858
+ else
2859
+ nk_reduce_minmax_e4m3_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
2860
+ max_index_ptr);
2861
+ }
2862
+
2863
+ NK_INTERNAL void nk_reduce_moments_e5m2_rvv_contiguous_( //
2864
+ nk_e5m2_t const *data_ptr, nk_size_t count, //
2865
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2866
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
2867
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2868
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2869
+
2870
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
2871
+ vector_length = __riscv_vsetvl_e8m1(count);
2872
+ vuint8m1_t data_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
2873
+
2874
+ // Convert e5m2 → f32 (m1 → m4)
2875
+ vfloat32m4_t data_f32m4 = nk_e5m2m1_to_f32m4_rvv_(data_u8m1, vector_length);
2876
+
2877
+ // Accumulate at f32 precision
2878
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
2879
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
2880
+ }
2881
+
2882
+ // Horizontal reduction
2883
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
2884
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
2885
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
2886
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
2887
+ }
2888
+
2889
+ NK_INTERNAL void nk_reduce_moments_e5m2_rvv_strided_( //
2890
+ nk_e5m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2891
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2892
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
2893
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2894
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
2895
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
2896
+
2897
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
2898
+ vector_length = __riscv_vsetvl_e8m1(count);
2899
+ vuint8m1_t data_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
2900
+
2901
+ // Convert e5m2 → f32 (m1 → m4)
2902
+ vfloat32m4_t data_f32m4 = nk_e5m2m1_to_f32m4_rvv_(data_u8m1, vector_length);
2903
+
2904
+ // Accumulate at f32 precision
2905
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
2906
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
2907
+ }
2908
+
2909
+ // Horizontal reduction
2910
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
2911
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
2912
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
2913
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
2914
+ }
2915
+
2916
+ NK_PUBLIC void nk_reduce_moments_e5m2_rvv( //
2917
+ nk_e5m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
2918
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
2919
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e5m2_t);
2920
+ int aligned = (stride_bytes % sizeof(nk_e5m2_t) == 0);
2921
+
2922
+ if (count == 0) *sum_ptr = 0.0f, *sumsq_ptr = 0.0f;
2923
+ else if (!aligned) nk_reduce_moments_e5m2_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2924
+ else if (stride_elements == 1) nk_reduce_moments_e5m2_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
2925
+ else nk_reduce_moments_e5m2_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
2926
+ }
2927
+
2928
+ NK_INTERNAL void nk_reduce_minmax_e5m2_rvv_contiguous_( //
2929
+ nk_e5m2_t const *data_ptr, nk_size_t count, //
2930
+ nk_e5m2_t *min_value_ptr, nk_size_t *min_index_ptr, //
2931
+ nk_e5m2_t *max_value_ptr, nk_size_t *max_index_ptr) {
2932
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
2933
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, max_vector_length);
2934
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length);
2935
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
2936
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
2937
+
2938
+ nk_size_t offset = 0;
2939
+ for (nk_size_t vector_length; count > 0;
2940
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
2941
+ vector_length = __riscv_vsetvl_e8m1(count);
2942
+ vuint8m1_t raw_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
2943
+
2944
+ vuint8m1_t comparable_u8m1 = nk_fp8m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
2945
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
2946
+ vector_length);
2947
+
2948
+ // Detect E5M2 NaN: comparable <= 0x02 (neg NaN) or comparable >= 0xFD (pos NaN)
2949
+ vbool8_t nan_low_b8 = __riscv_vmsleu_vx_u8m1_b8(comparable_u8m1, 0x02, vector_length);
2950
+ vbool8_t nan_high_b8 = __riscv_vmsgeu_vx_u8m1_b8(comparable_u8m1, 0xFD, vector_length);
2951
+ vbool8_t is_nan_b8 = __riscv_vmor_mm_b8(nan_low_b8, nan_high_b8, vector_length);
2952
+ vuint8m1_t data_min_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0xFF, is_nan_b8, vector_length);
2953
+ vuint8m1_t data_max_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0x00, is_nan_b8, vector_length);
2954
+
2955
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(data_min_u8m1, min_u8m1, vector_length);
2956
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, data_min_u8m1, less_b8, vector_length);
2957
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
2958
+ vector_length);
2959
+
2960
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, data_max_u8m1, vector_length);
2961
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, data_max_u8m1, greater_b8, vector_length);
2962
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
2963
+ vector_length);
2964
+ }
2965
+
2966
+ // Horizontal reduction + convert back
2967
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, 1);
2968
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
2969
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
2970
+
2971
+ // All-NaN case
2972
+ if (min_comparable == 0xFF) {
2973
+ *min_value_ptr = (nk_e5m2_t)NK_E5M2_MAX, *min_index_ptr = NK_SIZE_MAX;
2974
+ *max_value_ptr = (nk_e5m2_t)NK_E5M2_MIN, *max_index_ptr = NK_SIZE_MAX;
2975
+ return;
2976
+ }
2977
+
2978
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
2979
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
2980
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
2981
+ max_vector_length);
2982
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
2983
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2984
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
2985
+
2986
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
2987
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(min_vec_u8m1, 1);
2988
+ *min_value_ptr = (nk_e5m2_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
2989
+
2990
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
2991
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
2992
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
2993
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
2994
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
2995
+ max_vector_length);
2996
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
2997
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
2998
+
2999
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
3000
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(max_vec_u8m1, 1);
3001
+ *max_value_ptr = (nk_e5m2_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
3002
+ }
3003
+
3004
+ NK_INTERNAL void nk_reduce_minmax_e5m2_rvv_strided_( //
3005
+ nk_e5m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3006
+ nk_e5m2_t *min_value_ptr, nk_size_t *min_index_ptr, //
3007
+ nk_e5m2_t *max_value_ptr, nk_size_t *max_index_ptr) {
3008
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
3009
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, max_vector_length);
3010
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length);
3011
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3012
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3013
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
3014
+
3015
+ nk_size_t offset = 0;
3016
+ for (nk_size_t vector_length; count > 0;
3017
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
3018
+ vector_length = __riscv_vsetvl_e8m1(count);
3019
+ vuint8m1_t raw_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
3020
+
3021
+ vuint8m1_t comparable_u8m1 = nk_fp8m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
3022
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
3023
+ vector_length);
3024
+
3025
+ // Detect E5M2 NaN: comparable <= 0x02 (neg NaN) or comparable >= 0xFD (pos NaN)
3026
+ vbool8_t nan_low_b8 = __riscv_vmsleu_vx_u8m1_b8(comparable_u8m1, 0x02, vector_length);
3027
+ vbool8_t nan_high_b8 = __riscv_vmsgeu_vx_u8m1_b8(comparable_u8m1, 0xFD, vector_length);
3028
+ vbool8_t is_nan_b8 = __riscv_vmor_mm_b8(nan_low_b8, nan_high_b8, vector_length);
3029
+ vuint8m1_t data_min_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0xFF, is_nan_b8, vector_length);
3030
+ vuint8m1_t data_max_u8m1 = __riscv_vmerge_vxm_u8m1(comparable_u8m1, 0x00, is_nan_b8, vector_length);
3031
+
3032
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(data_min_u8m1, min_u8m1, vector_length);
3033
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, data_min_u8m1, less_b8, vector_length);
3034
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
3035
+ vector_length);
3036
+
3037
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, data_max_u8m1, vector_length);
3038
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, data_max_u8m1, greater_b8, vector_length);
3039
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
3040
+ vector_length);
3041
+ }
3042
+
3043
+ // Horizontal reduction (same as contiguous)
3044
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0xFF, 1);
3045
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
3046
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
3047
+
3048
+ // All-NaN case
3049
+ if (min_comparable == 0xFF) {
3050
+ *min_value_ptr = (nk_e5m2_t)NK_E5M2_MAX, *min_index_ptr = NK_SIZE_MAX;
3051
+ *max_value_ptr = (nk_e5m2_t)NK_E5M2_MIN, *max_index_ptr = NK_SIZE_MAX;
3052
+ return;
3053
+ }
3054
+
3055
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
3056
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
3057
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
3058
+ max_vector_length);
3059
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
3060
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3061
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
3062
+
3063
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
3064
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(min_vec_u8m1, 1);
3065
+ *min_value_ptr = (nk_e5m2_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
3066
+
3067
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
3068
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
3069
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
3070
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
3071
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
3072
+ max_vector_length);
3073
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3074
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
3075
+
3076
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
3077
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp8m1_rvv_(max_vec_u8m1, 1);
3078
+ *max_value_ptr = (nk_e5m2_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
3079
+ }
3080
+
3081
+ NK_PUBLIC void nk_reduce_minmax_e5m2_rvv( //
3082
+ nk_e5m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3083
+ nk_e5m2_t *min_value_ptr, nk_size_t *min_index_ptr, //
3084
+ nk_e5m2_t *max_value_ptr, nk_size_t *max_index_ptr) {
3085
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e5m2_t);
3086
+ int aligned = (stride_bytes % sizeof(nk_e5m2_t) == 0);
3087
+
3088
+ if (count == 0)
3089
+ *min_value_ptr = NK_E5M2_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_E5M2_MIN,
3090
+ *max_index_ptr = NK_SIZE_MAX;
3091
+ else if (!aligned)
3092
+ nk_reduce_minmax_e5m2_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
3093
+ max_index_ptr);
3094
+ else if (stride_elements == 1)
3095
+ nk_reduce_minmax_e5m2_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
3096
+ max_index_ptr);
3097
+ else
3098
+ nk_reduce_minmax_e5m2_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
3099
+ max_index_ptr);
3100
+ }
3101
+
3102
+ NK_INTERNAL void nk_reduce_moments_e2m3_rvv_contiguous_( //
3103
+ nk_e2m3_t const *data_ptr, nk_size_t count, //
3104
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
3105
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
3106
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3107
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3108
+
3109
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
3110
+ vector_length = __riscv_vsetvl_e8m1(count);
3111
+ vuint8m1_t data_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
3112
+
3113
+ // Convert e2m3 → f32 (m1 → m4)
3114
+ vfloat32m4_t data_f32m4 = nk_e2m3m1_to_f32m4_rvv_(data_u8m1, vector_length);
3115
+
3116
+ // Accumulate at f32 precision
3117
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
3118
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
3119
+ }
3120
+
3121
+ // Horizontal reduction
3122
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
3123
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
3124
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
3125
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
3126
+ }
3127
+
3128
+ NK_INTERNAL void nk_reduce_moments_e2m3_rvv_strided_( //
3129
+ nk_e2m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3130
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
3131
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
3132
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3133
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3134
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
3135
+
3136
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
3137
+ vector_length = __riscv_vsetvl_e8m1(count);
3138
+ vuint8m1_t data_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
3139
+
3140
+ // Convert e2m3 → f32 (m1 → m4)
3141
+ vfloat32m4_t data_f32m4 = nk_e2m3m1_to_f32m4_rvv_(data_u8m1, vector_length);
3142
+
3143
+ // Accumulate at f32 precision
3144
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
3145
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
3146
+ }
3147
+
3148
+ // Horizontal reduction
3149
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
3150
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
3151
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
3152
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
3153
+ }
3154
+
3155
+ NK_PUBLIC void nk_reduce_moments_e2m3_rvv( //
3156
+ nk_e2m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3157
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
3158
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e2m3_t);
3159
+ int aligned = (stride_bytes % sizeof(nk_e2m3_t) == 0);
3160
+
3161
+ if (count == 0) *sum_ptr = 0.0f, *sumsq_ptr = 0.0f;
3162
+ else if (!aligned) nk_reduce_moments_e2m3_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
3163
+ else if (stride_elements == 1) nk_reduce_moments_e2m3_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
3164
+ else nk_reduce_moments_e2m3_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
3165
+ }
3166
+
3167
+ NK_INTERNAL void nk_reduce_minmax_e2m3_rvv_contiguous_( //
3168
+ nk_e2m3_t const *data_ptr, nk_size_t count, //
3169
+ nk_e2m3_t *min_value_ptr, nk_size_t *min_index_ptr, //
3170
+ nk_e2m3_t *max_value_ptr, nk_size_t *max_index_ptr) {
3171
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
3172
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, max_vector_length); // Largest FP6 comparable
3173
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length); // Smallest FP6 comparable
3174
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3175
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3176
+
3177
+ nk_size_t offset = 0;
3178
+ for (nk_size_t vector_length; count > 0;
3179
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
3180
+ vector_length = __riscv_vsetvl_e8m1(count);
3181
+ vuint8m1_t raw_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
3182
+
3183
+ // Convert to FP6 comparable form
3184
+ vuint8m1_t comparable_u8m1 = nk_fp6m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
3185
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
3186
+ vector_length);
3187
+
3188
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(comparable_u8m1, min_u8m1, vector_length);
3189
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, comparable_u8m1, less_b8, vector_length);
3190
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
3191
+ vector_length);
3192
+
3193
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, comparable_u8m1, vector_length);
3194
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, comparable_u8m1, greater_b8, vector_length);
3195
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
3196
+ vector_length);
3197
+ }
3198
+
3199
+ // Horizontal reduction + convert back
3200
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, 1);
3201
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
3202
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
3203
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
3204
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
3205
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
3206
+ max_vector_length);
3207
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
3208
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3209
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
3210
+
3211
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
3212
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(min_vec_u8m1, 1);
3213
+ *min_value_ptr = (nk_e2m3_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
3214
+
3215
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
3216
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
3217
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
3218
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
3219
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
3220
+ max_vector_length);
3221
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3222
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
3223
+
3224
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
3225
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(max_vec_u8m1, 1);
3226
+ *max_value_ptr = (nk_e2m3_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
3227
+ }
3228
+
3229
+ NK_INTERNAL void nk_reduce_minmax_e2m3_rvv_strided_( //
3230
+ nk_e2m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3231
+ nk_e2m3_t *min_value_ptr, nk_size_t *min_index_ptr, //
3232
+ nk_e2m3_t *max_value_ptr, nk_size_t *max_index_ptr) {
3233
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
3234
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, max_vector_length);
3235
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length);
3236
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3237
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3238
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
3239
+
3240
+ nk_size_t offset = 0;
3241
+ for (nk_size_t vector_length; count > 0;
3242
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
3243
+ vector_length = __riscv_vsetvl_e8m1(count);
3244
+ vuint8m1_t raw_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
3245
+
3246
+ vuint8m1_t comparable_u8m1 = nk_fp6m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
3247
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
3248
+ vector_length);
3249
+
3250
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(comparable_u8m1, min_u8m1, vector_length);
3251
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, comparable_u8m1, less_b8, vector_length);
3252
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
3253
+ vector_length);
3254
+
3255
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, comparable_u8m1, vector_length);
3256
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, comparable_u8m1, greater_b8, vector_length);
3257
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
3258
+ vector_length);
3259
+ }
3260
+
3261
+ // Horizontal reduction (same as contiguous)
3262
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, 1);
3263
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
3264
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
3265
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
3266
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
3267
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
3268
+ max_vector_length);
3269
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
3270
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3271
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
3272
+
3273
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
3274
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(min_vec_u8m1, 1);
3275
+ *min_value_ptr = (nk_e2m3_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
3276
+
3277
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
3278
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
3279
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
3280
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
3281
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
3282
+ max_vector_length);
3283
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3284
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
3285
+
3286
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
3287
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(max_vec_u8m1, 1);
3288
+ *max_value_ptr = (nk_e2m3_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
3289
+ }
3290
+
3291
+ NK_PUBLIC void nk_reduce_minmax_e2m3_rvv( //
3292
+ nk_e2m3_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3293
+ nk_e2m3_t *min_value_ptr, nk_size_t *min_index_ptr, //
3294
+ nk_e2m3_t *max_value_ptr, nk_size_t *max_index_ptr) {
3295
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e2m3_t);
3296
+ int aligned = (stride_bytes % sizeof(nk_e2m3_t) == 0);
3297
+
3298
+ if (count == 0)
3299
+ *min_value_ptr = NK_E2M3_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_E2M3_MIN,
3300
+ *max_index_ptr = NK_SIZE_MAX;
3301
+ else if (!aligned)
3302
+ nk_reduce_minmax_e2m3_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
3303
+ max_index_ptr);
3304
+ else if (stride_elements == 1)
3305
+ nk_reduce_minmax_e2m3_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
3306
+ max_index_ptr);
3307
+ else
3308
+ nk_reduce_minmax_e2m3_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
3309
+ max_index_ptr);
3310
+ }
3311
+
3312
+ NK_INTERNAL void nk_reduce_moments_e3m2_rvv_contiguous_( //
3313
+ nk_e3m2_t const *data_ptr, nk_size_t count, //
3314
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
3315
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
3316
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3317
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3318
+
3319
+ for (nk_size_t vector_length; count > 0; count -= vector_length, data_ptr += vector_length) {
3320
+ vector_length = __riscv_vsetvl_e8m1(count);
3321
+ vuint8m1_t data_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
3322
+
3323
+ // Convert e3m2 → f32 (m1 → m4)
3324
+ vfloat32m4_t data_f32m4 = nk_e3m2m1_to_f32m4_rvv_(data_u8m1, vector_length);
3325
+
3326
+ // Accumulate at f32 precision
3327
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
3328
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
3329
+ }
3330
+
3331
+ // Horizontal reduction
3332
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
3333
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
3334
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
3335
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
3336
+ }
3337
+
3338
+ NK_INTERNAL void nk_reduce_moments_e3m2_rvv_strided_( //
3339
+ nk_e3m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3340
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
3341
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
3342
+ vfloat32m4_t sum_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3343
+ vfloat32m4_t sumsq_f32m4 = __riscv_vfmv_v_f_f32m4(0.0f, max_vector_length);
3344
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
3345
+
3346
+ for (nk_size_t vector_length; count > 0; count -= vector_length, ptr += vector_length * stride_bytes) {
3347
+ vector_length = __riscv_vsetvl_e8m1(count);
3348
+ vuint8m1_t data_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
3349
+
3350
+ // Convert e3m2 → f32 (m1 → m4)
3351
+ vfloat32m4_t data_f32m4 = nk_e3m2m1_to_f32m4_rvv_(data_u8m1, vector_length);
3352
+
3353
+ // Accumulate at f32 precision
3354
+ sum_f32m4 = __riscv_vfadd_vv_f32m4_tu(sum_f32m4, sum_f32m4, data_f32m4, vector_length);
3355
+ sumsq_f32m4 = __riscv_vfmacc_vv_f32m4_tu(sumsq_f32m4, data_f32m4, data_f32m4, vector_length);
3356
+ }
3357
+
3358
+ // Horizontal reduction
3359
+ vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
3360
+ *sum_ptr = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m4_f32m1(sum_f32m4, zero_f32m1, max_vector_length)),
3361
+ *sumsq_ptr = __riscv_vfmv_f_s_f32m1_f32(
3362
+ __riscv_vfredusum_vs_f32m4_f32m1(sumsq_f32m4, zero_f32m1, max_vector_length));
3363
+ }
3364
+
3365
+ NK_PUBLIC void nk_reduce_moments_e3m2_rvv( //
3366
+ nk_e3m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3367
+ nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
3368
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e3m2_t);
3369
+ int aligned = (stride_bytes % sizeof(nk_e3m2_t) == 0);
3370
+
3371
+ if (count == 0) *sum_ptr = 0.0f, *sumsq_ptr = 0.0f;
3372
+ else if (!aligned) nk_reduce_moments_e3m2_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
3373
+ else if (stride_elements == 1) nk_reduce_moments_e3m2_rvv_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
3374
+ else nk_reduce_moments_e3m2_rvv_strided_(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
3375
+ }
3376
+
3377
+ NK_INTERNAL void nk_reduce_minmax_e3m2_rvv_contiguous_( //
3378
+ nk_e3m2_t const *data_ptr, nk_size_t count, //
3379
+ nk_e3m2_t *min_value_ptr, nk_size_t *min_index_ptr, //
3380
+ nk_e3m2_t *max_value_ptr, nk_size_t *max_index_ptr) {
3381
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
3382
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, max_vector_length);
3383
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length);
3384
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3385
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3386
+
3387
+ nk_size_t offset = 0;
3388
+ for (nk_size_t vector_length; count > 0;
3389
+ count -= vector_length, offset += vector_length, data_ptr += vector_length) {
3390
+ vector_length = __riscv_vsetvl_e8m1(count);
3391
+ vuint8m1_t raw_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)data_ptr, vector_length);
3392
+
3393
+ vuint8m1_t comparable_u8m1 = nk_fp6m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
3394
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
3395
+ vector_length);
3396
+
3397
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(comparable_u8m1, min_u8m1, vector_length);
3398
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, comparable_u8m1, less_b8, vector_length);
3399
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
3400
+ vector_length);
3401
+
3402
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, comparable_u8m1, vector_length);
3403
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, comparable_u8m1, greater_b8, vector_length);
3404
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
3405
+ vector_length);
3406
+ }
3407
+
3408
+ // Horizontal reduction + convert back
3409
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, 1);
3410
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
3411
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
3412
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
3413
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
3414
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
3415
+ max_vector_length);
3416
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
3417
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3418
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
3419
+
3420
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
3421
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(min_vec_u8m1, 1);
3422
+ *min_value_ptr = (nk_e3m2_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
3423
+
3424
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
3425
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
3426
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
3427
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
3428
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
3429
+ max_vector_length);
3430
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3431
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
3432
+
3433
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
3434
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(max_vec_u8m1, 1);
3435
+ *max_value_ptr = (nk_e3m2_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
3436
+ }
3437
+
3438
+ NK_INTERNAL void nk_reduce_minmax_e3m2_rvv_strided_( //
3439
+ nk_e3m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3440
+ nk_e3m2_t *min_value_ptr, nk_size_t *min_index_ptr, //
3441
+ nk_e3m2_t *max_value_ptr, nk_size_t *max_index_ptr) {
3442
+ nk_size_t max_vector_length = __riscv_vsetvlmax_e8m1();
3443
+ vuint8m1_t min_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, max_vector_length);
3444
+ vuint8m1_t max_u8m1 = __riscv_vmv_v_x_u8m1(0x00, max_vector_length);
3445
+ vuint64m8_t min_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3446
+ vuint64m8_t max_indices_u64m8 = __riscv_vmv_v_x_u64m8(0, max_vector_length);
3447
+ unsigned char const *ptr = (unsigned char const *)data_ptr;
3448
+
3449
+ nk_size_t offset = 0;
3450
+ for (nk_size_t vector_length; count > 0;
3451
+ count -= vector_length, offset += vector_length, ptr += vector_length * stride_bytes) {
3452
+ vector_length = __riscv_vsetvl_e8m1(count);
3453
+ vuint8m1_t raw_u8m1 = __riscv_vlse8_v_u8m1((nk_u8_t const *)ptr, (nk_ssize_t)stride_bytes, vector_length);
3454
+
3455
+ vuint8m1_t comparable_u8m1 = nk_fp6m1_to_comparable_u8m1_rvv_(raw_u8m1, vector_length);
3456
+ vuint64m8_t pos_u64m8 = __riscv_vadd_vx_u64m8(__riscv_vid_v_u64m8(vector_length), (nk_u64_t)offset,
3457
+ vector_length);
3458
+
3459
+ vbool8_t less_b8 = __riscv_vmsltu_vv_u8m1_b8(comparable_u8m1, min_u8m1, vector_length);
3460
+ min_u8m1 = __riscv_vmerge_vvm_u8m1_tu(min_u8m1, min_u8m1, comparable_u8m1, less_b8, vector_length);
3461
+ min_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(min_indices_u64m8, min_indices_u64m8, pos_u64m8, less_b8,
3462
+ vector_length);
3463
+
3464
+ vbool8_t greater_b8 = __riscv_vmsltu_vv_u8m1_b8(max_u8m1, comparable_u8m1, vector_length);
3465
+ max_u8m1 = __riscv_vmerge_vvm_u8m1_tu(max_u8m1, max_u8m1, comparable_u8m1, greater_b8, vector_length);
3466
+ max_indices_u64m8 = __riscv_vmerge_vvm_u64m8_tu(max_indices_u64m8, max_indices_u64m8, pos_u64m8, greater_b8,
3467
+ vector_length);
3468
+ }
3469
+
3470
+ // Horizontal reduction (same as contiguous)
3471
+ vuint8m1_t init_max_u8m1 = __riscv_vmv_v_x_u8m1(0x3F, 1);
3472
+ nk_u8_t min_comparable = __riscv_vmv_x_s_u8m1_u8(
3473
+ __riscv_vredminu_vs_u8m1_u8m1(min_u8m1, init_max_u8m1, max_vector_length));
3474
+ vbool8_t min_match_b8 = __riscv_vmseq_vx_u8m1_b8(min_u8m1, min_comparable, max_vector_length);
3475
+ vuint64m8_t sentinel_u64m8 = __riscv_vmv_v_x_u64m8(NK_U64_MAX, max_vector_length);
3476
+ vuint64m8_t min_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, min_indices_u64m8, min_match_b8,
3477
+ max_vector_length);
3478
+ vuint64m1_t init_umax_u64m1 = __riscv_vmv_v_x_u64m1(NK_U64_MAX, 1);
3479
+ *min_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3480
+ __riscv_vredminu_vs_u64m8_u64m1(min_cands_u64m8, init_umax_u64m1, max_vector_length));
3481
+
3482
+ vuint8m1_t min_vec_u8m1 = __riscv_vmv_v_x_u8m1(min_comparable, 1);
3483
+ vuint8m1_t min_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(min_vec_u8m1, 1);
3484
+ *min_value_ptr = (nk_e3m2_t)__riscv_vmv_x_s_u8m1_u8(min_raw_u8m1);
3485
+
3486
+ vuint8m1_t init_min_u8m1 = __riscv_vmv_v_x_u8m1(0x00, 1);
3487
+ nk_u8_t max_comparable = __riscv_vmv_x_s_u8m1_u8(
3488
+ __riscv_vredmaxu_vs_u8m1_u8m1(max_u8m1, init_min_u8m1, max_vector_length));
3489
+ vbool8_t max_match_b8 = __riscv_vmseq_vx_u8m1_b8(max_u8m1, max_comparable, max_vector_length);
3490
+ vuint64m8_t max_cands_u64m8 = __riscv_vmerge_vvm_u64m8(sentinel_u64m8, max_indices_u64m8, max_match_b8,
3491
+ max_vector_length);
3492
+ *max_index_ptr = (nk_size_t)__riscv_vmv_x_s_u64m1_u64(
3493
+ __riscv_vredminu_vs_u64m8_u64m1(max_cands_u64m8, init_umax_u64m1, max_vector_length));
3494
+
3495
+ vuint8m1_t max_vec_u8m1 = __riscv_vmv_v_x_u8m1(max_comparable, 1);
3496
+ vuint8m1_t max_raw_u8m1 = nk_comparable_to_fp6m1_rvv_(max_vec_u8m1, 1);
3497
+ *max_value_ptr = (nk_e3m2_t)__riscv_vmv_x_s_u8m1_u8(max_raw_u8m1);
3498
+ }
3499
+
3500
+ NK_PUBLIC void nk_reduce_minmax_e3m2_rvv( //
3501
+ nk_e3m2_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
3502
+ nk_e3m2_t *min_value_ptr, nk_size_t *min_index_ptr, //
3503
+ nk_e3m2_t *max_value_ptr, nk_size_t *max_index_ptr) {
3504
+ nk_size_t stride_elements = stride_bytes / sizeof(nk_e3m2_t);
3505
+ int aligned = (stride_bytes % sizeof(nk_e3m2_t) == 0);
3506
+
3507
+ if (count == 0)
3508
+ *min_value_ptr = NK_E3M2_MAX, *min_index_ptr = NK_SIZE_MAX, *max_value_ptr = NK_E3M2_MIN,
3509
+ *max_index_ptr = NK_SIZE_MAX;
3510
+ else if (!aligned)
3511
+ nk_reduce_minmax_e3m2_serial(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
3512
+ max_index_ptr);
3513
+ else if (stride_elements == 1)
3514
+ nk_reduce_minmax_e3m2_rvv_contiguous_(data_ptr, count, min_value_ptr, min_index_ptr, max_value_ptr,
3515
+ max_index_ptr);
3516
+ else
3517
+ nk_reduce_minmax_e3m2_rvv_strided_(data_ptr, count, stride_bytes, min_value_ptr, min_index_ptr, max_value_ptr,
3518
+ max_index_ptr);
3519
+ }
3520
+
3521
+ #if defined(__clang__)
3522
+ #pragma clang attribute pop
3523
+ #elif defined(__GNUC__)
3524
+ #pragma GCC pop_options
3525
+ #endif
3526
+
3527
+ #if defined(__cplusplus)
3528
+ } // extern "C"
3529
+ #endif
3530
+
3531
+ #endif // NK_TARGET_RVV
3532
+ #endif // NK_TARGET_RISCV64_
3533
+ #endif // NK_REDUCE_RVV_H