usearch 2.21.4 → 2.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (604) hide show
  1. package/README.md +65 -69
  2. package/binding.gyp +15 -37
  3. package/c/CMakeLists.txt +4 -5
  4. package/c/README.md +8 -8
  5. package/c/lib.cpp +32 -14
  6. package/c/test.c +124 -1
  7. package/c/usearch.h +24 -3
  8. package/cpp/CMakeLists.txt +4 -0
  9. package/cpp/README.md +5 -5
  10. package/cpp/bench.cpp +97 -116
  11. package/cpp/test.cpp +117 -20
  12. package/include/usearch/index.hpp +224 -48
  13. package/include/usearch/index_dense.hpp +105 -40
  14. package/include/usearch/index_plugins.hpp +1387 -204
  15. package/javascript/README.md +1 -1
  16. package/javascript/dist/cjs/usearch.d.ts +8 -3
  17. package/javascript/dist/cjs/usearch.js +12 -4
  18. package/javascript/dist/esm/usearch.d.ts +8 -3
  19. package/javascript/dist/esm/usearch.js +12 -4
  20. package/javascript/lib.cpp +17 -0
  21. package/javascript/usearch.test.js +117 -94
  22. package/javascript/usearch.ts +14 -6
  23. package/numkong/.clang-tidy +30 -0
  24. package/{simsimd → numkong}/.cmake-format.py +18 -0
  25. package/numkong/.editorconfig +12 -0
  26. package/numkong/.git-blame-ignore-revs +123 -0
  27. package/numkong/.mailmap +15 -0
  28. package/numkong/.spi.yml +11 -0
  29. package/numkong/.swift-format +14 -0
  30. package/numkong/CITATION.cff +81 -0
  31. package/numkong/CONTRIBUTING.md +500 -0
  32. package/numkong/Cargo.toml +54 -0
  33. package/numkong/MANIFEST.in +8 -0
  34. package/numkong/Package.swift +84 -0
  35. package/numkong/README.md +569 -0
  36. package/numkong/bench/README.md +217 -0
  37. package/numkong/bench/bench-browser.html +323 -0
  38. package/numkong/bench/bench.cpp +344 -0
  39. package/numkong/bench/bench.hpp +816 -0
  40. package/numkong/bench/bench.js +148 -0
  41. package/numkong/bench/bench.mjs +616 -0
  42. package/numkong/bench/bench_attention.cpp +113 -0
  43. package/numkong/bench/bench_cast.cpp +264 -0
  44. package/numkong/bench/bench_cross_amx.cpp +62 -0
  45. package/numkong/bench/bench_cross_arm.cpp +245 -0
  46. package/numkong/bench/bench_cross_blas.cpp +309 -0
  47. package/numkong/bench/bench_cross_loongarch.cpp +106 -0
  48. package/numkong/bench/bench_cross_power.cpp +84 -0
  49. package/numkong/bench/bench_cross_rvv.cpp +121 -0
  50. package/numkong/bench/bench_cross_serial.cpp +151 -0
  51. package/numkong/bench/bench_cross_sme.cpp +163 -0
  52. package/numkong/bench/bench_cross_wasm.cpp +142 -0
  53. package/numkong/bench/bench_cross_x86.cpp +455 -0
  54. package/numkong/bench/bench_curved.cpp +210 -0
  55. package/numkong/bench/bench_dot.cpp +315 -0
  56. package/numkong/bench/bench_each.cpp +503 -0
  57. package/numkong/bench/bench_geospatial.cpp +112 -0
  58. package/numkong/bench/bench_maxsim.cpp +154 -0
  59. package/numkong/bench/bench_mesh.cpp +148 -0
  60. package/numkong/bench/bench_probability.cpp +60 -0
  61. package/numkong/bench/bench_reduce.cpp +371 -0
  62. package/numkong/bench/bench_set.cpp +73 -0
  63. package/numkong/bench/bench_sparse.cpp +198 -0
  64. package/numkong/bench/bench_spatial.cpp +347 -0
  65. package/numkong/bench/bench_trigonometry.cpp +137 -0
  66. package/numkong/bench/swift/Bench.swift +506 -0
  67. package/numkong/binding.gyp +169 -0
  68. package/numkong/build.rs +484 -0
  69. package/numkong/c/dispatch.h +512 -0
  70. package/numkong/c/dispatch_bf16.c +447 -0
  71. package/numkong/c/dispatch_bf16c.c +52 -0
  72. package/numkong/c/dispatch_e2m3.c +296 -0
  73. package/numkong/c/dispatch_e3m2.c +284 -0
  74. package/numkong/c/dispatch_e4m3.c +318 -0
  75. package/numkong/c/dispatch_e5m2.c +316 -0
  76. package/numkong/c/dispatch_f16.c +452 -0
  77. package/numkong/c/dispatch_f16c.c +58 -0
  78. package/numkong/c/dispatch_f32.c +434 -0
  79. package/numkong/c/dispatch_f32c.c +99 -0
  80. package/numkong/c/dispatch_f64.c +348 -0
  81. package/numkong/c/dispatch_f64c.c +98 -0
  82. package/numkong/c/dispatch_i16.c +96 -0
  83. package/numkong/c/dispatch_i32.c +89 -0
  84. package/numkong/c/dispatch_i4.c +153 -0
  85. package/numkong/c/dispatch_i64.c +86 -0
  86. package/numkong/c/dispatch_i8.c +348 -0
  87. package/numkong/c/dispatch_other.c +348 -0
  88. package/numkong/c/dispatch_u1.c +193 -0
  89. package/numkong/c/dispatch_u16.c +124 -0
  90. package/numkong/c/dispatch_u32.c +118 -0
  91. package/numkong/c/dispatch_u4.c +153 -0
  92. package/numkong/c/dispatch_u64.c +102 -0
  93. package/numkong/c/dispatch_u8.c +364 -0
  94. package/numkong/c/numkong.c +940 -0
  95. package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
  96. package/numkong/cmake/nk_isa_probe.cmake +90 -0
  97. package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
  98. package/numkong/cmake/nk_math_lib.cmake +46 -0
  99. package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
  100. package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
  101. package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
  102. package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
  103. package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
  104. package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
  105. package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
  106. package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
  107. package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
  108. package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
  109. package/numkong/cmake/toolchain-wasi.cmake +77 -0
  110. package/numkong/cmake/toolchain-wasm.cmake +70 -0
  111. package/numkong/cmake/toolchain-wasm64.cmake +62 -0
  112. package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
  113. package/numkong/include/README.md +643 -0
  114. package/numkong/include/module.modulemap +129 -0
  115. package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
  116. package/numkong/include/numkong/attention/sme.h +2075 -0
  117. package/numkong/include/numkong/attention.h +49 -0
  118. package/numkong/include/numkong/capabilities.h +930 -0
  119. package/numkong/include/numkong/cast/README.md +262 -0
  120. package/numkong/include/numkong/cast/diamond.h +64 -0
  121. package/numkong/include/numkong/cast/haswell.h +823 -0
  122. package/numkong/include/numkong/cast/icelake.h +475 -0
  123. package/numkong/include/numkong/cast/loongsonasx.h +252 -0
  124. package/numkong/include/numkong/cast/neon.h +1174 -0
  125. package/numkong/include/numkong/cast/powervsx.h +449 -0
  126. package/numkong/include/numkong/cast/rvv.h +970 -0
  127. package/numkong/include/numkong/cast/sapphire.h +262 -0
  128. package/numkong/include/numkong/cast/serial.h +2336 -0
  129. package/numkong/include/numkong/cast/skylake.h +915 -0
  130. package/numkong/include/numkong/cast/v128relaxed.h +537 -0
  131. package/numkong/include/numkong/cast.h +254 -0
  132. package/numkong/include/numkong/cast.hpp +44 -0
  133. package/numkong/include/numkong/curved/README.md +223 -0
  134. package/numkong/include/numkong/curved/genoa.h +182 -0
  135. package/numkong/include/numkong/curved/haswell.h +276 -0
  136. package/numkong/include/numkong/curved/neon.h +329 -0
  137. package/numkong/include/numkong/curved/neonbfdot.h +211 -0
  138. package/numkong/include/numkong/curved/rvv.h +305 -0
  139. package/numkong/include/numkong/curved/serial.h +207 -0
  140. package/numkong/include/numkong/curved/skylake.h +457 -0
  141. package/numkong/include/numkong/curved/smef64.h +530 -0
  142. package/numkong/include/numkong/curved.h +513 -0
  143. package/numkong/include/numkong/curved.hpp +144 -0
  144. package/numkong/include/numkong/dot/README.md +442 -0
  145. package/numkong/include/numkong/dot/alder.h +563 -0
  146. package/numkong/include/numkong/dot/diamond.h +158 -0
  147. package/numkong/include/numkong/dot/genoa.h +289 -0
  148. package/numkong/include/numkong/dot/haswell.h +1723 -0
  149. package/numkong/include/numkong/dot/icelake.h +997 -0
  150. package/numkong/include/numkong/dot/loongsonasx.h +671 -0
  151. package/numkong/include/numkong/dot/neon.h +869 -0
  152. package/numkong/include/numkong/dot/neonbfdot.h +243 -0
  153. package/numkong/include/numkong/dot/neonfhm.h +358 -0
  154. package/numkong/include/numkong/dot/neonfp8.h +323 -0
  155. package/numkong/include/numkong/dot/neonsdot.h +622 -0
  156. package/numkong/include/numkong/dot/powervsx.h +752 -0
  157. package/numkong/include/numkong/dot/rvv.h +722 -0
  158. package/numkong/include/numkong/dot/rvvbb.h +72 -0
  159. package/numkong/include/numkong/dot/rvvbf16.h +123 -0
  160. package/numkong/include/numkong/dot/rvvhalf.h +129 -0
  161. package/numkong/include/numkong/dot/sapphire.h +141 -0
  162. package/numkong/include/numkong/dot/serial.h +874 -0
  163. package/numkong/include/numkong/dot/sierra.h +405 -0
  164. package/numkong/include/numkong/dot/skylake.h +1116 -0
  165. package/numkong/include/numkong/dot/sve.h +420 -0
  166. package/numkong/include/numkong/dot/svebfdot.h +75 -0
  167. package/numkong/include/numkong/dot/svehalf.h +168 -0
  168. package/numkong/include/numkong/dot/svesdot.h +90 -0
  169. package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
  170. package/numkong/include/numkong/dot.h +1136 -0
  171. package/numkong/include/numkong/dot.hpp +94 -0
  172. package/numkong/include/numkong/dots/README.md +496 -0
  173. package/numkong/include/numkong/dots/alder.h +117 -0
  174. package/numkong/include/numkong/dots/diamond.h +86 -0
  175. package/numkong/include/numkong/dots/genoa.h +100 -0
  176. package/numkong/include/numkong/dots/graniteamx.h +733 -0
  177. package/numkong/include/numkong/dots/haswell.h +310 -0
  178. package/numkong/include/numkong/dots/icelake.h +180 -0
  179. package/numkong/include/numkong/dots/loongsonasx.h +176 -0
  180. package/numkong/include/numkong/dots/neon.h +123 -0
  181. package/numkong/include/numkong/dots/neonbfdot.h +59 -0
  182. package/numkong/include/numkong/dots/neonfhm.h +96 -0
  183. package/numkong/include/numkong/dots/neonfp8.h +99 -0
  184. package/numkong/include/numkong/dots/neonsdot.h +144 -0
  185. package/numkong/include/numkong/dots/powervsx.h +194 -0
  186. package/numkong/include/numkong/dots/rvv.h +2593 -0
  187. package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
  188. package/numkong/include/numkong/dots/serial.h +2867 -0
  189. package/numkong/include/numkong/dots/sierra.h +100 -0
  190. package/numkong/include/numkong/dots/skylake.h +207 -0
  191. package/numkong/include/numkong/dots/sme.h +5041 -0
  192. package/numkong/include/numkong/dots/smebi32.h +477 -0
  193. package/numkong/include/numkong/dots/smef64.h +1328 -0
  194. package/numkong/include/numkong/dots/smehalf.h +47 -0
  195. package/numkong/include/numkong/dots/v128relaxed.h +317 -0
  196. package/numkong/include/numkong/dots.h +3110 -0
  197. package/numkong/include/numkong/dots.hpp +641 -0
  198. package/numkong/include/numkong/each/README.md +465 -0
  199. package/numkong/include/numkong/each/haswell.h +1658 -0
  200. package/numkong/include/numkong/each/icelake.h +272 -0
  201. package/numkong/include/numkong/each/neon.h +1138 -0
  202. package/numkong/include/numkong/each/neonbfdot.h +211 -0
  203. package/numkong/include/numkong/each/neonhalf.h +318 -0
  204. package/numkong/include/numkong/each/rvv.h +1121 -0
  205. package/numkong/include/numkong/each/sapphire.h +343 -0
  206. package/numkong/include/numkong/each/serial.h +282 -0
  207. package/numkong/include/numkong/each/skylake.h +1562 -0
  208. package/numkong/include/numkong/each/v128relaxed.h +562 -0
  209. package/numkong/include/numkong/each.h +2232 -0
  210. package/numkong/include/numkong/each.hpp +434 -0
  211. package/numkong/include/numkong/geospatial/README.md +147 -0
  212. package/numkong/include/numkong/geospatial/haswell.h +633 -0
  213. package/numkong/include/numkong/geospatial/neon.h +615 -0
  214. package/numkong/include/numkong/geospatial/rvv.h +701 -0
  215. package/numkong/include/numkong/geospatial/serial.h +309 -0
  216. package/numkong/include/numkong/geospatial/skylake.h +613 -0
  217. package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
  218. package/numkong/include/numkong/geospatial.h +453 -0
  219. package/numkong/include/numkong/geospatial.hpp +235 -0
  220. package/numkong/include/numkong/matrix.hpp +336 -0
  221. package/numkong/include/numkong/maxsim/README.md +187 -0
  222. package/numkong/include/numkong/maxsim/alder.h +511 -0
  223. package/numkong/include/numkong/maxsim/genoa.h +115 -0
  224. package/numkong/include/numkong/maxsim/haswell.h +553 -0
  225. package/numkong/include/numkong/maxsim/icelake.h +480 -0
  226. package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
  227. package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
  228. package/numkong/include/numkong/maxsim/serial.h +490 -0
  229. package/numkong/include/numkong/maxsim/sme.h +993 -0
  230. package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
  231. package/numkong/include/numkong/maxsim.h +571 -0
  232. package/numkong/include/numkong/maxsim.hpp +133 -0
  233. package/numkong/include/numkong/mesh/README.md +219 -0
  234. package/numkong/include/numkong/mesh/haswell.h +2013 -0
  235. package/numkong/include/numkong/mesh/neon.h +1821 -0
  236. package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
  237. package/numkong/include/numkong/mesh/rvv.h +936 -0
  238. package/numkong/include/numkong/mesh/serial.h +721 -0
  239. package/numkong/include/numkong/mesh/skylake.h +2344 -0
  240. package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
  241. package/numkong/include/numkong/mesh.h +666 -0
  242. package/numkong/include/numkong/mesh.hpp +701 -0
  243. package/numkong/include/numkong/numkong.h +78 -0
  244. package/numkong/include/numkong/numkong.hpp +58 -0
  245. package/numkong/include/numkong/probability/README.md +177 -0
  246. package/numkong/include/numkong/probability/haswell.h +267 -0
  247. package/numkong/include/numkong/probability/neon.h +255 -0
  248. package/numkong/include/numkong/probability/rvv.h +414 -0
  249. package/numkong/include/numkong/probability/serial.h +181 -0
  250. package/numkong/include/numkong/probability/skylake.h +324 -0
  251. package/numkong/include/numkong/probability.h +380 -0
  252. package/numkong/include/numkong/probability.hpp +120 -0
  253. package/numkong/include/numkong/random.h +50 -0
  254. package/numkong/include/numkong/random.hpp +285 -0
  255. package/numkong/include/numkong/reduce/README.md +552 -0
  256. package/numkong/include/numkong/reduce/alder.h +636 -0
  257. package/numkong/include/numkong/reduce/genoa.h +201 -0
  258. package/numkong/include/numkong/reduce/haswell.h +3785 -0
  259. package/numkong/include/numkong/reduce/icelake.h +549 -0
  260. package/numkong/include/numkong/reduce/neon.h +3980 -0
  261. package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
  262. package/numkong/include/numkong/reduce/neonfhm.h +293 -0
  263. package/numkong/include/numkong/reduce/neonsdot.h +357 -0
  264. package/numkong/include/numkong/reduce/rvv.h +3533 -0
  265. package/numkong/include/numkong/reduce/serial.h +753 -0
  266. package/numkong/include/numkong/reduce/sierra.h +338 -0
  267. package/numkong/include/numkong/reduce/skylake.h +3794 -0
  268. package/numkong/include/numkong/reduce/sve.h +52 -0
  269. package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
  270. package/numkong/include/numkong/reduce.h +1582 -0
  271. package/numkong/include/numkong/reduce.hpp +778 -0
  272. package/numkong/include/numkong/scalar/README.md +89 -0
  273. package/numkong/include/numkong/scalar/haswell.h +122 -0
  274. package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
  275. package/numkong/include/numkong/scalar/neon.h +122 -0
  276. package/numkong/include/numkong/scalar/neonhalf.h +70 -0
  277. package/numkong/include/numkong/scalar/powervsx.h +96 -0
  278. package/numkong/include/numkong/scalar/rvv.h +211 -0
  279. package/numkong/include/numkong/scalar/sapphire.h +74 -0
  280. package/numkong/include/numkong/scalar/serial.h +332 -0
  281. package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
  282. package/numkong/include/numkong/scalar.h +696 -0
  283. package/numkong/include/numkong/set/README.md +179 -0
  284. package/numkong/include/numkong/set/haswell.h +334 -0
  285. package/numkong/include/numkong/set/icelake.h +485 -0
  286. package/numkong/include/numkong/set/loongsonasx.h +181 -0
  287. package/numkong/include/numkong/set/neon.h +363 -0
  288. package/numkong/include/numkong/set/powervsx.h +326 -0
  289. package/numkong/include/numkong/set/rvv.h +226 -0
  290. package/numkong/include/numkong/set/rvvbb.h +117 -0
  291. package/numkong/include/numkong/set/serial.h +174 -0
  292. package/numkong/include/numkong/set/sve.h +187 -0
  293. package/numkong/include/numkong/set/v128relaxed.h +240 -0
  294. package/numkong/include/numkong/set.h +471 -0
  295. package/numkong/include/numkong/set.hpp +114 -0
  296. package/numkong/include/numkong/sets/README.md +149 -0
  297. package/numkong/include/numkong/sets/haswell.h +63 -0
  298. package/numkong/include/numkong/sets/icelake.h +66 -0
  299. package/numkong/include/numkong/sets/loongsonasx.h +52 -0
  300. package/numkong/include/numkong/sets/neon.h +61 -0
  301. package/numkong/include/numkong/sets/powervsx.h +65 -0
  302. package/numkong/include/numkong/sets/serial.h +43 -0
  303. package/numkong/include/numkong/sets/smebi32.h +1135 -0
  304. package/numkong/include/numkong/sets/v128relaxed.h +58 -0
  305. package/numkong/include/numkong/sets.h +382 -0
  306. package/numkong/include/numkong/sparse/README.md +156 -0
  307. package/numkong/include/numkong/sparse/icelake.h +463 -0
  308. package/numkong/include/numkong/sparse/neon.h +288 -0
  309. package/numkong/include/numkong/sparse/serial.h +117 -0
  310. package/numkong/include/numkong/sparse/sve2.h +504 -0
  311. package/numkong/include/numkong/sparse/turin.h +322 -0
  312. package/numkong/include/numkong/sparse.h +363 -0
  313. package/numkong/include/numkong/sparse.hpp +113 -0
  314. package/numkong/include/numkong/spatial/README.md +439 -0
  315. package/numkong/include/numkong/spatial/alder.h +604 -0
  316. package/numkong/include/numkong/spatial/diamond.h +240 -0
  317. package/numkong/include/numkong/spatial/genoa.h +222 -0
  318. package/numkong/include/numkong/spatial/haswell.h +979 -0
  319. package/numkong/include/numkong/spatial/icelake.h +1067 -0
  320. package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
  321. package/numkong/include/numkong/spatial/neon.h +846 -0
  322. package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
  323. package/numkong/include/numkong/spatial/neonfp8.h +258 -0
  324. package/numkong/include/numkong/spatial/neonsdot.h +429 -0
  325. package/numkong/include/numkong/spatial/powervsx.h +738 -0
  326. package/numkong/include/numkong/spatial/rvv.h +991 -0
  327. package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
  328. package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
  329. package/numkong/include/numkong/spatial/serial.h +347 -0
  330. package/numkong/include/numkong/spatial/sierra.h +516 -0
  331. package/numkong/include/numkong/spatial/skylake.h +605 -0
  332. package/numkong/include/numkong/spatial/sve.h +246 -0
  333. package/numkong/include/numkong/spatial/svebfdot.h +125 -0
  334. package/numkong/include/numkong/spatial/svehalf.h +136 -0
  335. package/numkong/include/numkong/spatial/svesdot.h +143 -0
  336. package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
  337. package/numkong/include/numkong/spatial.h +1681 -0
  338. package/numkong/include/numkong/spatial.hpp +183 -0
  339. package/numkong/include/numkong/spatials/README.md +580 -0
  340. package/numkong/include/numkong/spatials/alder.h +94 -0
  341. package/numkong/include/numkong/spatials/diamond.h +82 -0
  342. package/numkong/include/numkong/spatials/genoa.h +94 -0
  343. package/numkong/include/numkong/spatials/graniteamx.h +173 -0
  344. package/numkong/include/numkong/spatials/haswell.h +219 -0
  345. package/numkong/include/numkong/spatials/icelake.h +113 -0
  346. package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
  347. package/numkong/include/numkong/spatials/neon.h +109 -0
  348. package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
  349. package/numkong/include/numkong/spatials/neonfhm.h +92 -0
  350. package/numkong/include/numkong/spatials/neonfp8.h +111 -0
  351. package/numkong/include/numkong/spatials/neonsdot.h +143 -0
  352. package/numkong/include/numkong/spatials/powervsx.h +153 -0
  353. package/numkong/include/numkong/spatials/rvv.h +1976 -0
  354. package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
  355. package/numkong/include/numkong/spatials/serial.h +248 -0
  356. package/numkong/include/numkong/spatials/sierra.h +96 -0
  357. package/numkong/include/numkong/spatials/skylake.h +184 -0
  358. package/numkong/include/numkong/spatials/sme.h +1927 -0
  359. package/numkong/include/numkong/spatials/smef64.h +483 -0
  360. package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
  361. package/numkong/include/numkong/spatials.h +3374 -0
  362. package/numkong/include/numkong/spatials.hpp +509 -0
  363. package/numkong/include/numkong/tensor.hpp +1981 -0
  364. package/numkong/include/numkong/trigonometry/README.md +184 -0
  365. package/numkong/include/numkong/trigonometry/haswell.h +653 -0
  366. package/numkong/include/numkong/trigonometry/neon.h +638 -0
  367. package/numkong/include/numkong/trigonometry/rvv.h +700 -0
  368. package/numkong/include/numkong/trigonometry/serial.h +703 -0
  369. package/numkong/include/numkong/trigonometry/skylake.h +725 -0
  370. package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
  371. package/numkong/include/numkong/trigonometry.h +467 -0
  372. package/numkong/include/numkong/trigonometry.hpp +166 -0
  373. package/numkong/include/numkong/types.h +1695 -0
  374. package/numkong/include/numkong/types.hpp +6027 -0
  375. package/numkong/include/numkong/vector.hpp +766 -0
  376. package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
  377. package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
  378. package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
  379. package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
  380. package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
  381. package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
  382. package/numkong/javascript/README.md +254 -0
  383. package/numkong/javascript/node-gyp-build.d.ts +4 -0
  384. package/numkong/javascript/numkong-browser.ts +40 -0
  385. package/numkong/javascript/numkong-wasm.ts +750 -0
  386. package/numkong/javascript/numkong.c +712 -0
  387. package/numkong/javascript/numkong.ts +613 -0
  388. package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
  389. package/numkong/javascript/tsconfig-cjs.json +13 -0
  390. package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
  391. package/numkong/javascript/types.ts +784 -0
  392. package/numkong/numkong.gypi +104 -0
  393. package/numkong/package.json +108 -0
  394. package/numkong/probes/arm_neon.c +8 -0
  395. package/numkong/probes/arm_neon_bfdot.c +9 -0
  396. package/numkong/probes/arm_neon_fhm.c +9 -0
  397. package/numkong/probes/arm_neon_half.c +8 -0
  398. package/numkong/probes/arm_neon_sdot.c +9 -0
  399. package/numkong/probes/arm_neonfp8.c +9 -0
  400. package/numkong/probes/arm_sme.c +16 -0
  401. package/numkong/probes/arm_sme2.c +16 -0
  402. package/numkong/probes/arm_sme2p1.c +16 -0
  403. package/numkong/probes/arm_sme_bf16.c +16 -0
  404. package/numkong/probes/arm_sme_bi32.c +16 -0
  405. package/numkong/probes/arm_sme_f64.c +16 -0
  406. package/numkong/probes/arm_sme_fa64.c +14 -0
  407. package/numkong/probes/arm_sme_half.c +16 -0
  408. package/numkong/probes/arm_sme_lut2.c +15 -0
  409. package/numkong/probes/arm_sve.c +18 -0
  410. package/numkong/probes/arm_sve2.c +20 -0
  411. package/numkong/probes/arm_sve2p1.c +18 -0
  412. package/numkong/probes/arm_sve_bfdot.c +20 -0
  413. package/numkong/probes/arm_sve_half.c +18 -0
  414. package/numkong/probes/arm_sve_sdot.c +21 -0
  415. package/numkong/probes/loongarch_lasx.c +12 -0
  416. package/numkong/probes/power_vsx.c +12 -0
  417. package/numkong/probes/probe.js +127 -0
  418. package/numkong/probes/riscv_rvv.c +14 -0
  419. package/numkong/probes/riscv_rvv_bb.c +15 -0
  420. package/numkong/probes/riscv_rvv_bf16.c +17 -0
  421. package/numkong/probes/riscv_rvv_half.c +14 -0
  422. package/numkong/probes/wasm_v128relaxed.c +11 -0
  423. package/numkong/probes/x86_alder.c +17 -0
  424. package/numkong/probes/x86_diamond.c +17 -0
  425. package/numkong/probes/x86_genoa.c +17 -0
  426. package/numkong/probes/x86_graniteamx.c +19 -0
  427. package/numkong/probes/x86_haswell.c +11 -0
  428. package/numkong/probes/x86_icelake.c +17 -0
  429. package/numkong/probes/x86_sapphire.c +16 -0
  430. package/numkong/probes/x86_sapphireamx.c +18 -0
  431. package/numkong/probes/x86_sierra.c +17 -0
  432. package/numkong/probes/x86_skylake.c +15 -0
  433. package/numkong/probes/x86_turin.c +17 -0
  434. package/numkong/rust/README.md +698 -0
  435. package/numkong/rust/capabilities.rs +96 -0
  436. package/numkong/rust/cast.rs +314 -0
  437. package/numkong/rust/curved.rs +391 -0
  438. package/numkong/rust/each.rs +2721 -0
  439. package/numkong/rust/geospatial.rs +343 -0
  440. package/numkong/rust/matrix.rs +5424 -0
  441. package/numkong/rust/maxsim.rs +540 -0
  442. package/numkong/rust/mesh.rs +837 -0
  443. package/numkong/rust/numkong.rs +409 -0
  444. package/numkong/rust/probability.rs +272 -0
  445. package/numkong/rust/reduce.rs +1249 -0
  446. package/numkong/rust/set.rs +194 -0
  447. package/numkong/rust/sparse.rs +513 -0
  448. package/numkong/rust/spatial.rs +1815 -0
  449. package/numkong/rust/tensor.rs +7565 -0
  450. package/numkong/rust/types.rs +3666 -0
  451. package/numkong/rust/vector.rs +1899 -0
  452. package/numkong/rustfmt.toml +2 -0
  453. package/numkong/swift/Matrix.swift +1388 -0
  454. package/numkong/swift/MaxSim.swift +151 -0
  455. package/numkong/swift/NumKong.swift +283 -0
  456. package/numkong/swift/Numerics.swift +329 -0
  457. package/numkong/swift/README.md +433 -0
  458. package/numkong/swift/Scalar.swift +741 -0
  459. package/numkong/swift/Tensor.swift +209 -0
  460. package/numkong/test/README.md +315 -0
  461. package/numkong/test/playwright.config.ts +16 -0
  462. package/numkong/test/swift/Test.swift +534 -0
  463. package/numkong/test/test-browser-bundle-runner.mjs +36 -0
  464. package/numkong/test/test-browser-bundle.html +188 -0
  465. package/numkong/test/test-browser-runner.mjs +43 -0
  466. package/numkong/test/test-browser.html +305 -0
  467. package/numkong/test/test-dtypes.mjs +453 -0
  468. package/numkong/test/test-wasm.mjs +267 -0
  469. package/numkong/test/test.cpp +442 -0
  470. package/numkong/test/test.hpp +580 -0
  471. package/numkong/test/test.mjs +363 -0
  472. package/numkong/test/test_base.py +1041 -0
  473. package/numkong/test/test_cast.cpp +242 -0
  474. package/numkong/test/test_cross.hpp +551 -0
  475. package/numkong/test/test_cross.py +467 -0
  476. package/numkong/test/test_cross_amx.cpp +50 -0
  477. package/numkong/test/test_cross_arm.cpp +225 -0
  478. package/numkong/test/test_cross_blas.cpp +354 -0
  479. package/numkong/test/test_cross_loongarch.cpp +90 -0
  480. package/numkong/test/test_cross_power.cpp +73 -0
  481. package/numkong/test/test_cross_rvv.cpp +109 -0
  482. package/numkong/test/test_cross_serial.cpp +181 -0
  483. package/numkong/test/test_cross_sme.cpp +148 -0
  484. package/numkong/test/test_cross_wasm.cpp +141 -0
  485. package/numkong/test/test_cross_x86.cpp +442 -0
  486. package/numkong/test/test_curved.cpp +182 -0
  487. package/numkong/test/test_curved.py +207 -0
  488. package/numkong/test/test_dot.cpp +353 -0
  489. package/numkong/test/test_dot.py +296 -0
  490. package/numkong/test/test_each.cpp +437 -0
  491. package/numkong/test/test_each.py +756 -0
  492. package/numkong/test/test_geospatial.cpp +155 -0
  493. package/numkong/test/test_geospatial.py +241 -0
  494. package/numkong/test/test_maxsim.cpp +138 -0
  495. package/numkong/test/test_maxsim.py +209 -0
  496. package/numkong/test/test_mesh.cpp +203 -0
  497. package/numkong/test/test_mesh.py +394 -0
  498. package/numkong/test/test_probability.cpp +136 -0
  499. package/numkong/test/test_probability.py +166 -0
  500. package/numkong/test/test_reduce.cpp +563 -0
  501. package/numkong/test/test_reduce.py +781 -0
  502. package/numkong/test/test_set.cpp +151 -0
  503. package/numkong/test/test_set.py +120 -0
  504. package/numkong/test/test_similarities.py +579 -0
  505. package/numkong/test/test_sparse.cpp +121 -0
  506. package/numkong/test/test_sparse.py +118 -0
  507. package/numkong/test/test_spatial.cpp +518 -0
  508. package/numkong/test/test_spatial.py +269 -0
  509. package/numkong/test/test_tensor.cpp +884 -0
  510. package/numkong/test/test_tensor.py +1417 -0
  511. package/numkong/test/test_trigonometry.cpp +156 -0
  512. package/numkong/test/test_trigonometry.py +180 -0
  513. package/package.json +7 -8
  514. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  515. package/prebuilds/linux-arm64/usearch.node +0 -0
  516. package/prebuilds/linux-x64/usearch.node +0 -0
  517. package/fp16/CMakeLists.txt +0 -176
  518. package/fp16/LICENSE +0 -11
  519. package/fp16/README.md +0 -20
  520. package/fp16/bench/alt-element.cc +0 -89
  521. package/fp16/bench/from-alt-array.cc +0 -175
  522. package/fp16/bench/from-ieee-array.cc +0 -399
  523. package/fp16/bench/ieee-element.cc +0 -272
  524. package/fp16/bench/to-alt-array.cc +0 -91
  525. package/fp16/bench/to-ieee-array.cc +0 -303
  526. package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
  527. package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
  528. package/fp16/cmake/DownloadPSimd.cmake +0 -15
  529. package/fp16/configure.py +0 -65
  530. package/fp16/confu.yaml +0 -8
  531. package/fp16/include/fp16/__init__.py +0 -0
  532. package/fp16/include/fp16/avx.py +0 -50
  533. package/fp16/include/fp16/avx2.py +0 -53
  534. package/fp16/include/fp16/bitcasts.h +0 -92
  535. package/fp16/include/fp16/fp16.h +0 -451
  536. package/fp16/include/fp16/psimd.h +0 -131
  537. package/fp16/include/fp16.h +0 -11
  538. package/fp16/jni/Android.mk +0 -6
  539. package/fp16/jni/Application.mk +0 -4
  540. package/fp16/test/alt-from-fp32-value.cc +0 -498
  541. package/fp16/test/alt-to-fp32-bits.cc +0 -263
  542. package/fp16/test/alt-to-fp32-psimd.cc +0 -145
  543. package/fp16/test/alt-to-fp32-value.cc +0 -348
  544. package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
  545. package/fp16/test/bitcasts.cc +0 -56
  546. package/fp16/test/ieee-from-fp32-value.cc +0 -523
  547. package/fp16/test/ieee-to-fp32-bits.cc +0 -333
  548. package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
  549. package/fp16/test/ieee-to-fp32-value.cc +0 -420
  550. package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
  551. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
  552. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
  553. package/fp16/test/peachpy/stubs.py +0 -38
  554. package/fp16/test/tables.cc +0 -530
  555. package/fp16/test/tables.h +0 -13
  556. package/fp16/third-party/THHalf.h +0 -92
  557. package/fp16/third-party/eigen-half.h +0 -170
  558. package/fp16/third-party/float16-compressor.h +0 -80
  559. package/fp16/third-party/half.hpp +0 -2916
  560. package/fp16/third-party/npy-halffloat.h +0 -328
  561. package/simsimd/.git-blame-ignore-revs +0 -2
  562. package/simsimd/CONTRIBUTING.md +0 -321
  563. package/simsimd/Cargo.toml +0 -49
  564. package/simsimd/MANIFEST.in +0 -4
  565. package/simsimd/Package.swift +0 -28
  566. package/simsimd/README.md +0 -1493
  567. package/simsimd/binding.gyp +0 -35
  568. package/simsimd/build.rs +0 -70
  569. package/simsimd/c/lib.c +0 -374
  570. package/simsimd/include/module.modulemap +0 -4
  571. package/simsimd/include/simsimd/binary.h +0 -486
  572. package/simsimd/include/simsimd/curved.h +0 -1541
  573. package/simsimd/include/simsimd/dot.h +0 -1853
  574. package/simsimd/include/simsimd/elementwise.h +0 -2465
  575. package/simsimd/include/simsimd/geospatial.h +0 -43
  576. package/simsimd/include/simsimd/mesh.h +0 -69
  577. package/simsimd/include/simsimd/probability.h +0 -606
  578. package/simsimd/include/simsimd/simsimd.h +0 -2567
  579. package/simsimd/include/simsimd/sparse.h +0 -1384
  580. package/simsimd/include/simsimd/spatial.h +0 -2335
  581. package/simsimd/include/simsimd/types.h +0 -666
  582. package/simsimd/javascript/fallback.ts +0 -233
  583. package/simsimd/javascript/lib.c +0 -125
  584. package/simsimd/javascript/node-gyp-build.d.ts +0 -1
  585. package/simsimd/javascript/simsimd.ts +0 -179
  586. package/simsimd/javascript/tsconfig-cjs.json +0 -8
  587. package/simsimd/package-lock.json +0 -1761
  588. package/simsimd/package.json +0 -61
  589. package/simsimd/rust/lib.rs +0 -1821
  590. package/simsimd/scripts/bench.cxx +0 -1170
  591. package/simsimd/scripts/bench.ipynb +0 -187
  592. package/simsimd/scripts/bench.js +0 -127
  593. package/simsimd/scripts/bench.rs +0 -876
  594. package/simsimd/scripts/bench_vectors.py +0 -734
  595. package/simsimd/scripts/bench_vectors_live.py +0 -213
  596. package/simsimd/scripts/test.c +0 -187
  597. package/simsimd/scripts/test.mjs +0 -220
  598. package/simsimd/scripts/test.py +0 -1505
  599. package/simsimd/swift/SimSIMD.swift +0 -149
  600. package/simsimd/swift/Test.swift +0 -128
  601. /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
  602. /package/{simsimd → numkong}/LICENSE +0 -0
  603. /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
  604. /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
@@ -0,0 +1,2013 @@
1
+ /**
2
+ * @brief SIMD-accelerated Point Cloud Alignment for Haswell.
3
+ * @file include/numkong/mesh/haswell.h
4
+ * @author Ash Vardanian
5
+ * @date December 27, 2025
6
+ *
7
+ * @sa include/numkong/mesh.h
8
+ *
9
+ * @section haswell_mesh_instructions Key AVX2 Mesh Instructions
10
+ *
11
+ * Intrinsic Instruction Haswell Genoa
12
+ * _mm256_fmadd_ps VFMADD (YMM, YMM, YMM) 5cy @ p01 4cy @ p01
13
+ * _mm256_hadd_ps VHADDPS (YMM, YMM, YMM) 7cy @ p1+p5 4cy @ p123+p23+p23
14
+ * _mm256_permute2f128_ps VPERM2F128 (YMM, YMM, YMM, I8) 3cy @ p5 2cy @ p12
15
+ * _mm256_extractf128_ps VEXTRACTF128 (XMM, YMM, I8) 3cy @ p5 1cy @ p0123
16
+ * _mm256_i32gather_ps VGATHERDPS (YMM, M, YMM, YMM) 22cy (34 uops) 19cy (17 uops)
17
+ *
18
+ * Point cloud operations (centroid, covariance, Kabsch alignment) use gather instructions for
19
+ * stride-3 xyz deinterleaving. Multiple FMA accumulators hide the 5-cycle FMA latency. VHADDPS
20
+ * interleaves results across lanes, requiring additional shuffles for final scalar reduction.
21
+ */
22
+ #ifndef NK_MESH_HASWELL_H
23
+ #define NK_MESH_HASWELL_H
24
+
25
+ #if NK_TARGET_X8664_
26
+ #if NK_TARGET_HASWELL
27
+
28
+ #include "numkong/types.h"
29
+ #include "numkong/dot/haswell.h"
30
+ #include "numkong/mesh/serial.h"
31
+ #include "numkong/reduce/haswell.h" // `nk_reduce_add_f32x8_haswell_`
32
+ #include "numkong/spatial/haswell.h" // `nk_f32_sqrt_haswell`, `nk_f64_sqrt_haswell`
33
+
34
+ #if defined(__cplusplus)
35
+ extern "C" {
36
+ #endif
37
+
38
+ #if defined(__clang__)
39
+ #pragma clang attribute push(__attribute__((target("avx2,f16c,fma,bmi,bmi2"))), apply_to = function)
40
+ #elif defined(__GNUC__)
41
+ #pragma GCC push_options
42
+ #pragma GCC target("avx2", "f16c", "fma", "bmi", "bmi2")
43
+ #endif
44
+
45
+ /* Deinterleave 24 floats (8 xyz triplets) into separate x, y, z vectors.
46
+ * Uses AVX2 gather instructions for clean stride-3 access.
47
+ *
48
+ * Input: 24 contiguous floats [x0,y0,z0, x1,y1,z1, ..., x7,y7,z7]
49
+ * Output: x[8], y[8], z[8] vectors
50
+ */
51
+ NK_INTERNAL void nk_deinterleave_f32x8_haswell_(nk_f32_t const *ptr, __m256 *x_out, __m256 *y_out, __m256 *z_out) {
52
+ // Gather indices: 0, 3, 6, 9, 12, 15, 18, 21 (stride 3)
53
+ __m256i idx_i32x8 = _mm256_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21);
54
+ *x_out = _mm256_i32gather_ps(ptr + 0, idx_i32x8, 4);
55
+ *y_out = _mm256_i32gather_ps(ptr + 1, idx_i32x8, 4);
56
+ *z_out = _mm256_i32gather_ps(ptr + 2, idx_i32x8, 4);
57
+ }
58
+
59
+ /* Deinterleave 12 f64 values (4 xyz triplets) into separate x, y, z vectors.
60
+ * Uses scalar extraction for simplicity as AVX2 lacks efficient stride-3 gather for f64.
61
+ *
62
+ * Input: 12 contiguous f64 [x0,y0,z0, x1,y1,z1, x2,y2,z2, x3,y3,z3]
63
+ * Output: x[4], y[4], z[4] vectors
64
+ */
65
+ NK_INTERNAL void nk_deinterleave_f64x4_haswell_(nk_f64_t const *ptr, __m256d *x_out, __m256d *y_out, __m256d *z_out) {
66
+ nk_f64_t x0 = ptr[0], x1 = ptr[3], x2 = ptr[6], x3 = ptr[9];
67
+ nk_f64_t y0 = ptr[1], y1 = ptr[4], y2 = ptr[7], y3 = ptr[10];
68
+ nk_f64_t z0 = ptr[2], z1 = ptr[5], z2 = ptr[8], z3 = ptr[11];
69
+
70
+ *x_out = _mm256_setr_pd(x0, x1, x2, x3);
71
+ *y_out = _mm256_setr_pd(y0, y1, y2, y3);
72
+ *z_out = _mm256_setr_pd(z0, z1, z2, z3);
73
+ }
74
+
75
+ /* Horizontal reduction helpers moved to reduce.h:
76
+ * - nk_reduce_add_f32x8_haswell_
77
+ * - nk_reduce_add_f64x4_haswell_
78
+ */
79
+
80
+ NK_INTERNAL nk_f64_t nk_reduce_stable_f64x4_haswell_(__m256d values_f64x4) {
81
+ nk_b256_vec_t values;
82
+ values.ymm_pd = values_f64x4;
83
+ nk_f64_t sum = 0.0, compensation = 0.0;
84
+ nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[0]);
85
+ nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[1]);
86
+ nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[2]);
87
+ nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[3]);
88
+ return sum + compensation;
89
+ }
90
+
91
+ NK_INTERNAL void nk_rotation_from_svd_f64_haswell_(nk_f64_t const *svd_u, nk_f64_t const *svd_v, nk_f64_t *rotation) {
92
+ nk_rotation_from_svd_f64_serial_(svd_u, svd_v, rotation);
93
+ }
94
+
95
+ NK_INTERNAL void nk_accumulate_square_f64x4_haswell_(__m256d *sum_f64x4, __m256d *compensation_f64x4,
96
+ __m256d values_f64x4) {
97
+ __m256d product_f64x4 = _mm256_mul_pd(values_f64x4, values_f64x4);
98
+ __m256d product_error_f64x4 = _mm256_fmsub_pd(values_f64x4, values_f64x4, product_f64x4);
99
+ __m256d tentative_sum_f64x4 = _mm256_add_pd(*sum_f64x4, product_f64x4);
100
+ __m256d virtual_addend_f64x4 = _mm256_sub_pd(tentative_sum_f64x4, *sum_f64x4);
101
+ __m256d sum_error_f64x4 = _mm256_add_pd(
102
+ _mm256_sub_pd(*sum_f64x4, _mm256_sub_pd(tentative_sum_f64x4, virtual_addend_f64x4)),
103
+ _mm256_sub_pd(product_f64x4, virtual_addend_f64x4));
104
+ *sum_f64x4 = tentative_sum_f64x4;
105
+ *compensation_f64x4 = _mm256_add_pd(*compensation_f64x4, _mm256_add_pd(sum_error_f64x4, product_error_f64x4));
106
+ }
107
+
108
+ /* Compute sum of squared distances after applying rotation (and optional scale).
109
+ * Used by kabsch (scale=1.0) and umeyama (scale=computed_scale).
110
+ * Returns sum_squared, caller computes sqrt(sum_squared / n).
111
+ */
112
+ NK_INTERNAL nk_f64_t nk_transformed_ssd_f32_haswell_(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n,
113
+ nk_f64_t const *r, nk_f64_t scale, nk_f64_t centroid_a_x,
114
+ nk_f64_t centroid_a_y, nk_f64_t centroid_a_z,
115
+ nk_f64_t centroid_b_x, nk_f64_t centroid_b_y,
116
+ nk_f64_t centroid_b_z) {
117
+ __m256d scaled_rotation_x_x_f64x4 = _mm256_set1_pd(scale * r[0]);
118
+ __m256d scaled_rotation_x_y_f64x4 = _mm256_set1_pd(scale * r[1]);
119
+ __m256d scaled_rotation_x_z_f64x4 = _mm256_set1_pd(scale * r[2]);
120
+ __m256d scaled_rotation_y_x_f64x4 = _mm256_set1_pd(scale * r[3]);
121
+ __m256d scaled_rotation_y_y_f64x4 = _mm256_set1_pd(scale * r[4]);
122
+ __m256d scaled_rotation_y_z_f64x4 = _mm256_set1_pd(scale * r[5]);
123
+ __m256d scaled_rotation_z_x_f64x4 = _mm256_set1_pd(scale * r[6]);
124
+ __m256d scaled_rotation_z_y_f64x4 = _mm256_set1_pd(scale * r[7]);
125
+ __m256d scaled_rotation_z_z_f64x4 = _mm256_set1_pd(scale * r[8]);
126
+ __m256d centroid_a_x_f64x4 = _mm256_set1_pd(centroid_a_x), centroid_a_y_f64x4 = _mm256_set1_pd(centroid_a_y);
127
+ __m256d centroid_a_z_f64x4 = _mm256_set1_pd(centroid_a_z), centroid_b_x_f64x4 = _mm256_set1_pd(centroid_b_x);
128
+ __m256d centroid_b_y_f64x4 = _mm256_set1_pd(centroid_b_y), centroid_b_z_f64x4 = _mm256_set1_pd(centroid_b_z);
129
+ __m256d sum_squared_f64x4 = _mm256_setzero_pd();
130
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
131
+ nk_size_t index = 0;
132
+
133
+ for (; index + 8 <= n; index += 8) {
134
+ nk_deinterleave_f32x8_haswell_(a + index * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8),
135
+ nk_deinterleave_f32x8_haswell_(b + index * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
136
+
137
+ __m256d a_x_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_x_f32x8));
138
+ __m256d a_x_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_x_f32x8, 1));
139
+ __m256d a_y_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_y_f32x8));
140
+ __m256d a_y_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_y_f32x8, 1));
141
+ __m256d a_z_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_z_f32x8));
142
+ __m256d a_z_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_z_f32x8, 1));
143
+ __m256d b_x_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_x_f32x8));
144
+ __m256d b_x_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_x_f32x8, 1));
145
+ __m256d b_y_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_y_f32x8));
146
+ __m256d b_y_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_y_f32x8, 1));
147
+ __m256d b_z_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_z_f32x8));
148
+ __m256d b_z_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_z_f32x8, 1));
149
+
150
+ __m256d centered_a_x_low_f64x4 = _mm256_sub_pd(a_x_low_f64x4, centroid_a_x_f64x4);
151
+ __m256d centered_a_x_high_f64x4 = _mm256_sub_pd(a_x_high_f64x4, centroid_a_x_f64x4);
152
+ __m256d centered_a_y_low_f64x4 = _mm256_sub_pd(a_y_low_f64x4, centroid_a_y_f64x4);
153
+ __m256d centered_a_y_high_f64x4 = _mm256_sub_pd(a_y_high_f64x4, centroid_a_y_f64x4);
154
+ __m256d centered_a_z_low_f64x4 = _mm256_sub_pd(a_z_low_f64x4, centroid_a_z_f64x4);
155
+ __m256d centered_a_z_high_f64x4 = _mm256_sub_pd(a_z_high_f64x4, centroid_a_z_f64x4);
156
+ __m256d centered_b_x_low_f64x4 = _mm256_sub_pd(b_x_low_f64x4, centroid_b_x_f64x4);
157
+ __m256d centered_b_x_high_f64x4 = _mm256_sub_pd(b_x_high_f64x4, centroid_b_x_f64x4);
158
+ __m256d centered_b_y_low_f64x4 = _mm256_sub_pd(b_y_low_f64x4, centroid_b_y_f64x4);
159
+ __m256d centered_b_y_high_f64x4 = _mm256_sub_pd(b_y_high_f64x4, centroid_b_y_f64x4);
160
+ __m256d centered_b_z_low_f64x4 = _mm256_sub_pd(b_z_low_f64x4, centroid_b_z_f64x4);
161
+ __m256d centered_b_z_high_f64x4 = _mm256_sub_pd(b_z_high_f64x4, centroid_b_z_f64x4);
162
+
163
+ __m256d rotated_a_x_low_f64x4 = _mm256_fmadd_pd(
164
+ scaled_rotation_x_z_f64x4, centered_a_z_low_f64x4,
165
+ _mm256_fmadd_pd(scaled_rotation_x_y_f64x4, centered_a_y_low_f64x4,
166
+ _mm256_mul_pd(scaled_rotation_x_x_f64x4, centered_a_x_low_f64x4)));
167
+ __m256d rotated_a_x_high_f64x4 = _mm256_fmadd_pd(
168
+ scaled_rotation_x_z_f64x4, centered_a_z_high_f64x4,
169
+ _mm256_fmadd_pd(scaled_rotation_x_y_f64x4, centered_a_y_high_f64x4,
170
+ _mm256_mul_pd(scaled_rotation_x_x_f64x4, centered_a_x_high_f64x4)));
171
+ __m256d rotated_a_y_low_f64x4 = _mm256_fmadd_pd(
172
+ scaled_rotation_y_z_f64x4, centered_a_z_low_f64x4,
173
+ _mm256_fmadd_pd(scaled_rotation_y_y_f64x4, centered_a_y_low_f64x4,
174
+ _mm256_mul_pd(scaled_rotation_y_x_f64x4, centered_a_x_low_f64x4)));
175
+ __m256d rotated_a_y_high_f64x4 = _mm256_fmadd_pd(
176
+ scaled_rotation_y_z_f64x4, centered_a_z_high_f64x4,
177
+ _mm256_fmadd_pd(scaled_rotation_y_y_f64x4, centered_a_y_high_f64x4,
178
+ _mm256_mul_pd(scaled_rotation_y_x_f64x4, centered_a_x_high_f64x4)));
179
+ __m256d rotated_a_z_low_f64x4 = _mm256_fmadd_pd(
180
+ scaled_rotation_z_z_f64x4, centered_a_z_low_f64x4,
181
+ _mm256_fmadd_pd(scaled_rotation_z_y_f64x4, centered_a_y_low_f64x4,
182
+ _mm256_mul_pd(scaled_rotation_z_x_f64x4, centered_a_x_low_f64x4)));
183
+ __m256d rotated_a_z_high_f64x4 = _mm256_fmadd_pd(
184
+ scaled_rotation_z_z_f64x4, centered_a_z_high_f64x4,
185
+ _mm256_fmadd_pd(scaled_rotation_z_y_f64x4, centered_a_y_high_f64x4,
186
+ _mm256_mul_pd(scaled_rotation_z_x_f64x4, centered_a_x_high_f64x4)));
187
+
188
+ __m256d delta_x_low_f64x4 = _mm256_sub_pd(rotated_a_x_low_f64x4, centered_b_x_low_f64x4);
189
+ __m256d delta_x_high_f64x4 = _mm256_sub_pd(rotated_a_x_high_f64x4, centered_b_x_high_f64x4);
190
+ __m256d delta_y_low_f64x4 = _mm256_sub_pd(rotated_a_y_low_f64x4, centered_b_y_low_f64x4);
191
+ __m256d delta_y_high_f64x4 = _mm256_sub_pd(rotated_a_y_high_f64x4, centered_b_y_high_f64x4);
192
+ __m256d delta_z_low_f64x4 = _mm256_sub_pd(rotated_a_z_low_f64x4, centered_b_z_low_f64x4);
193
+ __m256d delta_z_high_f64x4 = _mm256_sub_pd(rotated_a_z_high_f64x4, centered_b_z_high_f64x4);
194
+
195
+ __m256d batch_sum_squared_f64x4 = _mm256_add_pd(_mm256_mul_pd(delta_x_low_f64x4, delta_x_low_f64x4),
196
+ _mm256_mul_pd(delta_x_high_f64x4, delta_x_high_f64x4));
197
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_y_low_f64x4, delta_y_low_f64x4, batch_sum_squared_f64x4);
198
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_y_high_f64x4, delta_y_high_f64x4, batch_sum_squared_f64x4);
199
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_z_low_f64x4, delta_z_low_f64x4, batch_sum_squared_f64x4);
200
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_z_high_f64x4, delta_z_high_f64x4, batch_sum_squared_f64x4);
201
+ sum_squared_f64x4 = _mm256_add_pd(sum_squared_f64x4, batch_sum_squared_f64x4);
202
+ }
203
+
204
+ nk_f64_t sum_squared = nk_reduce_add_f64x4_haswell_(sum_squared_f64x4);
205
+ for (; index < n; ++index) {
206
+ nk_f64_t centered_a_x = (nk_f64_t)a[index * 3 + 0] - centroid_a_x,
207
+ centered_a_y = (nk_f64_t)a[index * 3 + 1] - centroid_a_y,
208
+ centered_a_z = (nk_f64_t)a[index * 3 + 2] - centroid_a_z;
209
+ nk_f64_t centered_b_x = (nk_f64_t)b[index * 3 + 0] - centroid_b_x,
210
+ centered_b_y = (nk_f64_t)b[index * 3 + 1] - centroid_b_y,
211
+ centered_b_z = (nk_f64_t)b[index * 3 + 2] - centroid_b_z;
212
+ nk_f64_t rotated_a_x = scale * (r[0] * centered_a_x + r[1] * centered_a_y + r[2] * centered_a_z),
213
+ rotated_a_y = scale * (r[3] * centered_a_x + r[4] * centered_a_y + r[5] * centered_a_z),
214
+ rotated_a_z = scale * (r[6] * centered_a_x + r[7] * centered_a_y + r[8] * centered_a_z);
215
+ nk_f64_t delta_x = rotated_a_x - centered_b_x, delta_y = rotated_a_y - centered_b_y,
216
+ delta_z = rotated_a_z - centered_b_z;
217
+ sum_squared += delta_x * delta_x + delta_y * delta_y + delta_z * delta_z;
218
+ }
219
+
220
+ return sum_squared;
221
+ }
222
+
223
+ /* Compute sum of squared distances for f64 after applying rotation (and optional scale).
224
+ * Rotation matrix, scale and data are all f64 for full precision.
225
+ */
226
+ NK_INTERNAL nk_f64_t nk_transformed_ssd_f64_haswell_(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n,
227
+ nk_f64_t const *r, nk_f64_t scale, nk_f64_t centroid_a_x,
228
+ nk_f64_t centroid_a_y, nk_f64_t centroid_a_z,
229
+ nk_f64_t centroid_b_x, nk_f64_t centroid_b_y,
230
+ nk_f64_t centroid_b_z) {
231
+ // Broadcast scaled rotation matrix elements
232
+ __m256d scaled_rotation_x_x_f64x4 = _mm256_set1_pd(scale * r[0]);
233
+ __m256d scaled_rotation_x_y_f64x4 = _mm256_set1_pd(scale * r[1]);
234
+ __m256d scaled_rotation_x_z_f64x4 = _mm256_set1_pd(scale * r[2]);
235
+ __m256d scaled_rotation_y_x_f64x4 = _mm256_set1_pd(scale * r[3]);
236
+ __m256d scaled_rotation_y_y_f64x4 = _mm256_set1_pd(scale * r[4]);
237
+ __m256d scaled_rotation_y_z_f64x4 = _mm256_set1_pd(scale * r[5]);
238
+ __m256d scaled_rotation_z_x_f64x4 = _mm256_set1_pd(scale * r[6]);
239
+ __m256d scaled_rotation_z_y_f64x4 = _mm256_set1_pd(scale * r[7]);
240
+ __m256d scaled_rotation_z_z_f64x4 = _mm256_set1_pd(scale * r[8]);
241
+
242
+ // Broadcast centroids
243
+ __m256d centroid_a_x_f64x4 = _mm256_set1_pd(centroid_a_x);
244
+ __m256d centroid_a_y_f64x4 = _mm256_set1_pd(centroid_a_y);
245
+ __m256d centroid_a_z_f64x4 = _mm256_set1_pd(centroid_a_z);
246
+ __m256d centroid_b_x_f64x4 = _mm256_set1_pd(centroid_b_x);
247
+ __m256d centroid_b_y_f64x4 = _mm256_set1_pd(centroid_b_y);
248
+ __m256d centroid_b_z_f64x4 = _mm256_set1_pd(centroid_b_z);
249
+
250
+ __m256d sum_squared_f64x4 = _mm256_setzero_pd();
251
+ __m256d sum_squared_compensation_f64x4 = _mm256_setzero_pd();
252
+ __m256d a_x_f64x4, a_y_f64x4, a_z_f64x4, b_x_f64x4, b_y_f64x4, b_z_f64x4;
253
+ nk_size_t j = 0;
254
+
255
+ for (; j + 4 <= n; j += 4) {
256
+ nk_deinterleave_f64x4_haswell_(a + j * 3, &a_x_f64x4, &a_y_f64x4, &a_z_f64x4);
257
+ nk_deinterleave_f64x4_haswell_(b + j * 3, &b_x_f64x4, &b_y_f64x4, &b_z_f64x4);
258
+
259
+ // Center points
260
+ __m256d pa_x_f64x4 = _mm256_sub_pd(a_x_f64x4, centroid_a_x_f64x4);
261
+ __m256d pa_y_f64x4 = _mm256_sub_pd(a_y_f64x4, centroid_a_y_f64x4);
262
+ __m256d pa_z_f64x4 = _mm256_sub_pd(a_z_f64x4, centroid_a_z_f64x4);
263
+ __m256d pb_x_f64x4 = _mm256_sub_pd(b_x_f64x4, centroid_b_x_f64x4);
264
+ __m256d pb_y_f64x4 = _mm256_sub_pd(b_y_f64x4, centroid_b_y_f64x4);
265
+ __m256d pb_z_f64x4 = _mm256_sub_pd(b_z_f64x4, centroid_b_z_f64x4);
266
+
267
+ // Rotate and scale: ra = scale * R * pa
268
+ __m256d ra_x_f64x4 = _mm256_fmadd_pd(scaled_rotation_x_z_f64x4, pa_z_f64x4,
269
+ _mm256_fmadd_pd(scaled_rotation_x_y_f64x4, pa_y_f64x4,
270
+ _mm256_mul_pd(scaled_rotation_x_x_f64x4, pa_x_f64x4)));
271
+ __m256d ra_y_f64x4 = _mm256_fmadd_pd(scaled_rotation_y_z_f64x4, pa_z_f64x4,
272
+ _mm256_fmadd_pd(scaled_rotation_y_y_f64x4, pa_y_f64x4,
273
+ _mm256_mul_pd(scaled_rotation_y_x_f64x4, pa_x_f64x4)));
274
+ __m256d ra_z_f64x4 = _mm256_fmadd_pd(scaled_rotation_z_z_f64x4, pa_z_f64x4,
275
+ _mm256_fmadd_pd(scaled_rotation_z_y_f64x4, pa_y_f64x4,
276
+ _mm256_mul_pd(scaled_rotation_z_x_f64x4, pa_x_f64x4)));
277
+
278
+ // Delta and accumulate
279
+ __m256d delta_x_f64x4 = _mm256_sub_pd(ra_x_f64x4, pb_x_f64x4);
280
+ __m256d delta_y_f64x4 = _mm256_sub_pd(ra_y_f64x4, pb_y_f64x4);
281
+ __m256d delta_z_f64x4 = _mm256_sub_pd(ra_z_f64x4, pb_z_f64x4);
282
+
283
+ nk_accumulate_square_f64x4_haswell_(&sum_squared_f64x4, &sum_squared_compensation_f64x4, delta_x_f64x4);
284
+ nk_accumulate_square_f64x4_haswell_(&sum_squared_f64x4, &sum_squared_compensation_f64x4, delta_y_f64x4);
285
+ nk_accumulate_square_f64x4_haswell_(&sum_squared_f64x4, &sum_squared_compensation_f64x4, delta_z_f64x4);
286
+ }
287
+
288
+ nk_f64_t sum_squared = nk_dot_stable_sum_f64x4_haswell_(sum_squared_f64x4, sum_squared_compensation_f64x4);
289
+ nk_f64_t sum_squared_compensation = 0.0;
290
+
291
+ // Scalar tail
292
+ for (; j < n; ++j) {
293
+ nk_f64_t pa_x = a[j * 3 + 0] - centroid_a_x, pa_y = a[j * 3 + 1] - centroid_a_y,
294
+ pa_z = a[j * 3 + 2] - centroid_a_z;
295
+ nk_f64_t pb_x = b[j * 3 + 0] - centroid_b_x, pb_y = b[j * 3 + 1] - centroid_b_y,
296
+ pb_z = b[j * 3 + 2] - centroid_b_z;
297
+ nk_f64_t ra_x = scale * (r[0] * pa_x + r[1] * pa_y + r[2] * pa_z),
298
+ ra_y = scale * (r[3] * pa_x + r[4] * pa_y + r[5] * pa_z),
299
+ ra_z = scale * (r[6] * pa_x + r[7] * pa_y + r[8] * pa_z);
300
+
301
+ nk_f64_t delta_x = ra_x - pb_x, delta_y = ra_y - pb_y, delta_z = ra_z - pb_z;
302
+ nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_x);
303
+ nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_y);
304
+ nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_z);
305
+ }
306
+
307
+ return sum_squared + sum_squared_compensation;
308
+ }
309
+
310
+ NK_PUBLIC void nk_rmsd_f32_haswell(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
311
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
312
+ if (a_centroid) a_centroid[0] = 0, a_centroid[1] = 0, a_centroid[2] = 0;
313
+ if (b_centroid) b_centroid[0] = 0, b_centroid[1] = 0, b_centroid[2] = 0;
314
+ if (rotation)
315
+ rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
316
+ rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
317
+ if (scale) *scale = 1.0f;
318
+
319
+ __m256d sum_squared_f64x4 = _mm256_setzero_pd();
320
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
321
+ nk_size_t index = 0;
322
+
323
+ for (; index + 8 <= n; index += 8) {
324
+ nk_deinterleave_f32x8_haswell_(a + index * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8),
325
+ nk_deinterleave_f32x8_haswell_(b + index * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
326
+
327
+ __m256d delta_x_low_f64x4 = _mm256_sub_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(a_x_f32x8)),
328
+ _mm256_cvtps_pd(_mm256_castps256_ps128(b_x_f32x8)));
329
+ __m256d delta_x_high_f64x4 = _mm256_sub_pd(_mm256_cvtps_pd(_mm256_extractf128_ps(a_x_f32x8, 1)),
330
+ _mm256_cvtps_pd(_mm256_extractf128_ps(b_x_f32x8, 1)));
331
+ __m256d delta_y_low_f64x4 = _mm256_sub_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(a_y_f32x8)),
332
+ _mm256_cvtps_pd(_mm256_castps256_ps128(b_y_f32x8)));
333
+ __m256d delta_y_high_f64x4 = _mm256_sub_pd(_mm256_cvtps_pd(_mm256_extractf128_ps(a_y_f32x8, 1)),
334
+ _mm256_cvtps_pd(_mm256_extractf128_ps(b_y_f32x8, 1)));
335
+ __m256d delta_z_low_f64x4 = _mm256_sub_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(a_z_f32x8)),
336
+ _mm256_cvtps_pd(_mm256_castps256_ps128(b_z_f32x8)));
337
+ __m256d delta_z_high_f64x4 = _mm256_sub_pd(_mm256_cvtps_pd(_mm256_extractf128_ps(a_z_f32x8, 1)),
338
+ _mm256_cvtps_pd(_mm256_extractf128_ps(b_z_f32x8, 1)));
339
+
340
+ __m256d batch_sum_squared_f64x4 = _mm256_add_pd(_mm256_mul_pd(delta_x_low_f64x4, delta_x_low_f64x4),
341
+ _mm256_mul_pd(delta_x_high_f64x4, delta_x_high_f64x4));
342
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_y_low_f64x4, delta_y_low_f64x4, batch_sum_squared_f64x4);
343
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_y_high_f64x4, delta_y_high_f64x4, batch_sum_squared_f64x4);
344
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_z_low_f64x4, delta_z_low_f64x4, batch_sum_squared_f64x4);
345
+ batch_sum_squared_f64x4 = _mm256_fmadd_pd(delta_z_high_f64x4, delta_z_high_f64x4, batch_sum_squared_f64x4);
346
+ sum_squared_f64x4 = _mm256_add_pd(sum_squared_f64x4, batch_sum_squared_f64x4);
347
+ }
348
+
349
+ nk_f64_t sum_squared = nk_reduce_add_f64x4_haswell_(sum_squared_f64x4);
350
+
351
+ for (; index < n; ++index) {
352
+ nk_f64_t delta_x = (nk_f64_t)a[index * 3 + 0] - (nk_f64_t)b[index * 3 + 0];
353
+ nk_f64_t delta_y = (nk_f64_t)a[index * 3 + 1] - (nk_f64_t)b[index * 3 + 1];
354
+ nk_f64_t delta_z = (nk_f64_t)a[index * 3 + 2] - (nk_f64_t)b[index * 3 + 2];
355
+ sum_squared += delta_x * delta_x + delta_y * delta_y + delta_z * delta_z;
356
+ }
357
+
358
+ *result = nk_f64_sqrt_haswell(sum_squared / (nk_f64_t)n);
359
+ }
360
+
361
+ NK_PUBLIC void nk_rmsd_f64_haswell(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
362
+ nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
363
+ if (a_centroid) a_centroid[0] = 0, a_centroid[1] = 0, a_centroid[2] = 0;
364
+ if (b_centroid) b_centroid[0] = 0, b_centroid[1] = 0, b_centroid[2] = 0;
365
+ if (rotation)
366
+ rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
367
+ rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
368
+ if (scale) *scale = 1.0;
369
+ __m256d const zeros_f64x4 = _mm256_setzero_pd();
370
+
371
+ __m256d sum_squared_x_f64x4 = zeros_f64x4, sum_squared_y_f64x4 = zeros_f64x4, sum_squared_z_f64x4 = zeros_f64x4;
372
+
373
+ __m256d a_x_f64x4, a_y_f64x4, a_z_f64x4, b_x_f64x4, b_y_f64x4, b_z_f64x4;
374
+ nk_size_t i = 0;
375
+
376
+ // Main loop with 2× unrolling
377
+ for (; i + 8 <= n; i += 8) {
378
+ nk_deinterleave_f64x4_haswell_(a + i * 3, &a_x_f64x4, &a_y_f64x4, &a_z_f64x4);
379
+ nk_deinterleave_f64x4_haswell_(b + i * 3, &b_x_f64x4, &b_y_f64x4, &b_z_f64x4);
380
+
381
+ __m256d delta_x_f64x4 = _mm256_sub_pd(a_x_f64x4, b_x_f64x4);
382
+ __m256d delta_y_f64x4 = _mm256_sub_pd(a_y_f64x4, b_y_f64x4);
383
+ __m256d delta_z_f64x4 = _mm256_sub_pd(a_z_f64x4, b_z_f64x4);
384
+
385
+ sum_squared_x_f64x4 = _mm256_fmadd_pd(delta_x_f64x4, delta_x_f64x4, sum_squared_x_f64x4);
386
+ sum_squared_y_f64x4 = _mm256_fmadd_pd(delta_y_f64x4, delta_y_f64x4, sum_squared_y_f64x4);
387
+ sum_squared_z_f64x4 = _mm256_fmadd_pd(delta_z_f64x4, delta_z_f64x4, sum_squared_z_f64x4);
388
+
389
+ __m256d a_x1_f64x4, a_y1_f64x4, a_z1_f64x4, b_x1_f64x4, b_y1_f64x4, b_z1_f64x4;
390
+ nk_deinterleave_f64x4_haswell_(a + (i + 4) * 3, &a_x1_f64x4, &a_y1_f64x4, &a_z1_f64x4);
391
+ nk_deinterleave_f64x4_haswell_(b + (i + 4) * 3, &b_x1_f64x4, &b_y1_f64x4, &b_z1_f64x4);
392
+
393
+ __m256d delta_x1_f64x4 = _mm256_sub_pd(a_x1_f64x4, b_x1_f64x4);
394
+ __m256d delta_y1_f64x4 = _mm256_sub_pd(a_y1_f64x4, b_y1_f64x4);
395
+ __m256d delta_z1_f64x4 = _mm256_sub_pd(a_z1_f64x4, b_z1_f64x4);
396
+
397
+ sum_squared_x_f64x4 = _mm256_fmadd_pd(delta_x1_f64x4, delta_x1_f64x4, sum_squared_x_f64x4);
398
+ sum_squared_y_f64x4 = _mm256_fmadd_pd(delta_y1_f64x4, delta_y1_f64x4, sum_squared_y_f64x4);
399
+ sum_squared_z_f64x4 = _mm256_fmadd_pd(delta_z1_f64x4, delta_z1_f64x4, sum_squared_z_f64x4);
400
+ }
401
+
402
+ for (; i + 4 <= n; i += 4) {
403
+ nk_deinterleave_f64x4_haswell_(a + i * 3, &a_x_f64x4, &a_y_f64x4, &a_z_f64x4);
404
+ nk_deinterleave_f64x4_haswell_(b + i * 3, &b_x_f64x4, &b_y_f64x4, &b_z_f64x4);
405
+
406
+ __m256d delta_x_f64x4 = _mm256_sub_pd(a_x_f64x4, b_x_f64x4);
407
+ __m256d delta_y_f64x4 = _mm256_sub_pd(a_y_f64x4, b_y_f64x4);
408
+ __m256d delta_z_f64x4 = _mm256_sub_pd(a_z_f64x4, b_z_f64x4);
409
+
410
+ sum_squared_x_f64x4 = _mm256_fmadd_pd(delta_x_f64x4, delta_x_f64x4, sum_squared_x_f64x4);
411
+ sum_squared_y_f64x4 = _mm256_fmadd_pd(delta_y_f64x4, delta_y_f64x4, sum_squared_y_f64x4);
412
+ sum_squared_z_f64x4 = _mm256_fmadd_pd(delta_z_f64x4, delta_z_f64x4, sum_squared_z_f64x4);
413
+ }
414
+
415
+ nk_f64_t total_sq_x = nk_reduce_stable_f64x4_haswell_(sum_squared_x_f64x4), total_sq_x_compensation = 0.0;
416
+ nk_f64_t total_sq_y = nk_reduce_stable_f64x4_haswell_(sum_squared_y_f64x4), total_sq_y_compensation = 0.0;
417
+ nk_f64_t total_sq_z = nk_reduce_stable_f64x4_haswell_(sum_squared_z_f64x4), total_sq_z_compensation = 0.0;
418
+
419
+ for (; i < n; ++i) {
420
+ nk_f64_t delta_x = a[i * 3 + 0] - b[i * 3 + 0];
421
+ nk_f64_t delta_y = a[i * 3 + 1] - b[i * 3 + 1];
422
+ nk_f64_t delta_z = a[i * 3 + 2] - b[i * 3 + 2];
423
+ nk_accumulate_square_f64_(&total_sq_x, &total_sq_x_compensation, delta_x);
424
+ nk_accumulate_square_f64_(&total_sq_y, &total_sq_y_compensation, delta_y);
425
+ nk_accumulate_square_f64_(&total_sq_z, &total_sq_z_compensation, delta_z);
426
+ }
427
+
428
+ total_sq_x += total_sq_x_compensation, total_sq_y += total_sq_y_compensation, total_sq_z += total_sq_z_compensation;
429
+
430
+ *result = nk_f64_sqrt_haswell((total_sq_x + total_sq_y + total_sq_z) / (nk_f64_t)n);
431
+ }
432
+
433
+ NK_PUBLIC void nk_kabsch_f32_haswell(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
434
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
435
+ if (scale) *scale = 1.0f;
436
+ __m256d sum_a_x_f64x4 = _mm256_setzero_pd(), sum_a_y_f64x4 = _mm256_setzero_pd();
437
+ __m256d sum_a_z_f64x4 = _mm256_setzero_pd(), sum_b_x_f64x4 = _mm256_setzero_pd();
438
+ __m256d sum_b_y_f64x4 = _mm256_setzero_pd(), sum_b_z_f64x4 = _mm256_setzero_pd();
439
+ __m256d covariance_00_f64x4 = _mm256_setzero_pd(), covariance_01_f64x4 = _mm256_setzero_pd();
440
+ __m256d covariance_02_f64x4 = _mm256_setzero_pd(), covariance_10_f64x4 = _mm256_setzero_pd();
441
+ __m256d covariance_11_f64x4 = _mm256_setzero_pd(), covariance_12_f64x4 = _mm256_setzero_pd();
442
+ __m256d covariance_20_f64x4 = _mm256_setzero_pd(), covariance_21_f64x4 = _mm256_setzero_pd();
443
+ __m256d covariance_22_f64x4 = _mm256_setzero_pd();
444
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
445
+ nk_size_t index = 0;
446
+
447
+ for (; index + 8 <= n; index += 8) {
448
+ nk_deinterleave_f32x8_haswell_(a + index * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8),
449
+ nk_deinterleave_f32x8_haswell_(b + index * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
450
+ __m256d a_x_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_x_f32x8));
451
+ __m256d a_x_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_x_f32x8, 1));
452
+ __m256d a_y_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_y_f32x8));
453
+ __m256d a_y_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_y_f32x8, 1));
454
+ __m256d a_z_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_z_f32x8));
455
+ __m256d a_z_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_z_f32x8, 1));
456
+ __m256d b_x_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_x_f32x8));
457
+ __m256d b_x_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_x_f32x8, 1));
458
+ __m256d b_y_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_y_f32x8));
459
+ __m256d b_y_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_y_f32x8, 1));
460
+ __m256d b_z_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_z_f32x8));
461
+ __m256d b_z_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_z_f32x8, 1));
462
+
463
+ sum_a_x_f64x4 = _mm256_add_pd(sum_a_x_f64x4, _mm256_add_pd(a_x_low_f64x4, a_x_high_f64x4));
464
+ sum_a_y_f64x4 = _mm256_add_pd(sum_a_y_f64x4, _mm256_add_pd(a_y_low_f64x4, a_y_high_f64x4));
465
+ sum_a_z_f64x4 = _mm256_add_pd(sum_a_z_f64x4, _mm256_add_pd(a_z_low_f64x4, a_z_high_f64x4));
466
+ sum_b_x_f64x4 = _mm256_add_pd(sum_b_x_f64x4, _mm256_add_pd(b_x_low_f64x4, b_x_high_f64x4));
467
+ sum_b_y_f64x4 = _mm256_add_pd(sum_b_y_f64x4, _mm256_add_pd(b_y_low_f64x4, b_y_high_f64x4));
468
+ sum_b_z_f64x4 = _mm256_add_pd(sum_b_z_f64x4, _mm256_add_pd(b_z_low_f64x4, b_z_high_f64x4));
469
+
470
+ covariance_00_f64x4 = _mm256_add_pd(
471
+ covariance_00_f64x4,
472
+ _mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, b_x_low_f64x4), _mm256_mul_pd(a_x_high_f64x4, b_x_high_f64x4)));
473
+ covariance_01_f64x4 = _mm256_add_pd(
474
+ covariance_01_f64x4,
475
+ _mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, b_y_low_f64x4), _mm256_mul_pd(a_x_high_f64x4, b_y_high_f64x4)));
476
+ covariance_02_f64x4 = _mm256_add_pd(
477
+ covariance_02_f64x4,
478
+ _mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, b_z_low_f64x4), _mm256_mul_pd(a_x_high_f64x4, b_z_high_f64x4)));
479
+ covariance_10_f64x4 = _mm256_add_pd(
480
+ covariance_10_f64x4,
481
+ _mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, b_x_low_f64x4), _mm256_mul_pd(a_y_high_f64x4, b_x_high_f64x4)));
482
+ covariance_11_f64x4 = _mm256_add_pd(
483
+ covariance_11_f64x4,
484
+ _mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, b_y_low_f64x4), _mm256_mul_pd(a_y_high_f64x4, b_y_high_f64x4)));
485
+ covariance_12_f64x4 = _mm256_add_pd(
486
+ covariance_12_f64x4,
487
+ _mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, b_z_low_f64x4), _mm256_mul_pd(a_y_high_f64x4, b_z_high_f64x4)));
488
+ covariance_20_f64x4 = _mm256_add_pd(
489
+ covariance_20_f64x4,
490
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, b_x_low_f64x4), _mm256_mul_pd(a_z_high_f64x4, b_x_high_f64x4)));
491
+ covariance_21_f64x4 = _mm256_add_pd(
492
+ covariance_21_f64x4,
493
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, b_y_low_f64x4), _mm256_mul_pd(a_z_high_f64x4, b_y_high_f64x4)));
494
+ covariance_22_f64x4 = _mm256_add_pd(
495
+ covariance_22_f64x4,
496
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, b_z_low_f64x4), _mm256_mul_pd(a_z_high_f64x4, b_z_high_f64x4)));
497
+ }
498
+
499
+ nk_f64_t sum_a_x = nk_reduce_add_f64x4_haswell_(sum_a_x_f64x4);
500
+ nk_f64_t sum_a_y = nk_reduce_add_f64x4_haswell_(sum_a_y_f64x4);
501
+ nk_f64_t sum_a_z = nk_reduce_add_f64x4_haswell_(sum_a_z_f64x4);
502
+ nk_f64_t sum_b_x = nk_reduce_add_f64x4_haswell_(sum_b_x_f64x4);
503
+ nk_f64_t sum_b_y = nk_reduce_add_f64x4_haswell_(sum_b_y_f64x4);
504
+ nk_f64_t sum_b_z = nk_reduce_add_f64x4_haswell_(sum_b_z_f64x4);
505
+ nk_f64_t h[9] = {
506
+ nk_reduce_add_f64x4_haswell_(covariance_00_f64x4), nk_reduce_add_f64x4_haswell_(covariance_01_f64x4),
507
+ nk_reduce_add_f64x4_haswell_(covariance_02_f64x4), nk_reduce_add_f64x4_haswell_(covariance_10_f64x4),
508
+ nk_reduce_add_f64x4_haswell_(covariance_11_f64x4), nk_reduce_add_f64x4_haswell_(covariance_12_f64x4),
509
+ nk_reduce_add_f64x4_haswell_(covariance_20_f64x4), nk_reduce_add_f64x4_haswell_(covariance_21_f64x4),
510
+ nk_reduce_add_f64x4_haswell_(covariance_22_f64x4)};
511
+
512
+ for (; index < n; ++index) {
513
+ nk_f64_t a_x = a[index * 3 + 0], a_y = a[index * 3 + 1], a_z = a[index * 3 + 2];
514
+ nk_f64_t b_x = b[index * 3 + 0], b_y = b[index * 3 + 1], b_z = b[index * 3 + 2];
515
+ sum_a_x += a_x, sum_a_y += a_y, sum_a_z += a_z;
516
+ sum_b_x += b_x, sum_b_y += b_y, sum_b_z += b_z;
517
+ h[0] += a_x * b_x, h[1] += a_x * b_y, h[2] += a_x * b_z;
518
+ h[3] += a_y * b_x, h[4] += a_y * b_y, h[5] += a_y * b_z;
519
+ h[6] += a_z * b_x, h[7] += a_z * b_y, h[8] += a_z * b_z;
520
+ }
521
+
522
+ nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
523
+ nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
524
+ nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
525
+ if (a_centroid)
526
+ a_centroid[0] = (nk_f32_t)centroid_a_x, a_centroid[1] = (nk_f32_t)centroid_a_y,
527
+ a_centroid[2] = (nk_f32_t)centroid_a_z;
528
+ if (b_centroid)
529
+ b_centroid[0] = (nk_f32_t)centroid_b_x, b_centroid[1] = (nk_f32_t)centroid_b_y,
530
+ b_centroid[2] = (nk_f32_t)centroid_b_z;
531
+
532
+ h[0] -= (nk_f64_t)n * centroid_a_x * centroid_b_x, h[1] -= (nk_f64_t)n * centroid_a_x * centroid_b_y,
533
+ h[2] -= (nk_f64_t)n * centroid_a_x * centroid_b_z, h[3] -= (nk_f64_t)n * centroid_a_y * centroid_b_x,
534
+ h[4] -= (nk_f64_t)n * centroid_a_y * centroid_b_y, h[5] -= (nk_f64_t)n * centroid_a_y * centroid_b_z,
535
+ h[6] -= (nk_f64_t)n * centroid_a_z * centroid_b_x, h[7] -= (nk_f64_t)n * centroid_a_z * centroid_b_y,
536
+ h[8] -= (nk_f64_t)n * centroid_a_z * centroid_b_z;
537
+
538
+ nk_f64_t cross_covariance[9] = {h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], h[8]};
539
+ nk_f64_t svd_u[9], svd_s[9], svd_v[9], r[9];
540
+ nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
541
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
542
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
543
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
544
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
545
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
546
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
547
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
548
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
549
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
550
+ if (nk_det3x3_f64_(r) < 0) {
551
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
552
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
553
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
554
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
555
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
556
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
557
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
558
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
559
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
560
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
561
+ }
562
+
563
+ if (rotation)
564
+ for (int j = 0; j != 9; ++j) rotation[j] = (nk_f32_t)r[j];
565
+ nk_f64_t sum_squared = nk_transformed_ssd_f32_haswell_(a, b, n, r, 1.0, centroid_a_x, centroid_a_y, centroid_a_z,
566
+ centroid_b_x, centroid_b_y, centroid_b_z);
567
+ *result = nk_f64_sqrt_haswell(sum_squared / (nk_f64_t)n);
568
+ }
569
+
570
+ NK_PUBLIC void nk_kabsch_f64_haswell(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
571
+ nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
572
+ __m256d const zeros_f64x4 = _mm256_setzero_pd();
573
+
574
+ // Accumulators for centroids
575
+ __m256d sum_a_x_f64x4 = zeros_f64x4, sum_a_y_f64x4 = zeros_f64x4, sum_a_z_f64x4 = zeros_f64x4;
576
+ __m256d sum_b_x_f64x4 = zeros_f64x4, sum_b_y_f64x4 = zeros_f64x4, sum_b_z_f64x4 = zeros_f64x4;
577
+
578
+ // Accumulators for covariance matrix (sum of outer products)
579
+ __m256d cov_xx_f64x4 = zeros_f64x4, cov_xy_f64x4 = zeros_f64x4, cov_xz_f64x4 = zeros_f64x4;
580
+ __m256d cov_yx_f64x4 = zeros_f64x4, cov_yy_f64x4 = zeros_f64x4, cov_yz_f64x4 = zeros_f64x4;
581
+ __m256d cov_zx_f64x4 = zeros_f64x4, cov_zy_f64x4 = zeros_f64x4, cov_zz_f64x4 = zeros_f64x4;
582
+
583
+ nk_size_t i = 0;
584
+ __m256d a_x_f64x4, a_y_f64x4, a_z_f64x4, b_x_f64x4, b_y_f64x4, b_z_f64x4;
585
+
586
+ // Fused single-pass
587
+ for (; i + 4 <= n; i += 4) {
588
+ nk_deinterleave_f64x4_haswell_(a + i * 3, &a_x_f64x4, &a_y_f64x4, &a_z_f64x4);
589
+ nk_deinterleave_f64x4_haswell_(b + i * 3, &b_x_f64x4, &b_y_f64x4, &b_z_f64x4);
590
+
591
+ sum_a_x_f64x4 = _mm256_add_pd(sum_a_x_f64x4, a_x_f64x4);
592
+ sum_a_y_f64x4 = _mm256_add_pd(sum_a_y_f64x4, a_y_f64x4);
593
+ sum_a_z_f64x4 = _mm256_add_pd(sum_a_z_f64x4, a_z_f64x4);
594
+ sum_b_x_f64x4 = _mm256_add_pd(sum_b_x_f64x4, b_x_f64x4);
595
+ sum_b_y_f64x4 = _mm256_add_pd(sum_b_y_f64x4, b_y_f64x4);
596
+ sum_b_z_f64x4 = _mm256_add_pd(sum_b_z_f64x4, b_z_f64x4);
597
+
598
+ cov_xx_f64x4 = _mm256_fmadd_pd(a_x_f64x4, b_x_f64x4, cov_xx_f64x4);
599
+ cov_xy_f64x4 = _mm256_fmadd_pd(a_x_f64x4, b_y_f64x4, cov_xy_f64x4);
600
+ cov_xz_f64x4 = _mm256_fmadd_pd(a_x_f64x4, b_z_f64x4, cov_xz_f64x4);
601
+ cov_yx_f64x4 = _mm256_fmadd_pd(a_y_f64x4, b_x_f64x4, cov_yx_f64x4);
602
+ cov_yy_f64x4 = _mm256_fmadd_pd(a_y_f64x4, b_y_f64x4, cov_yy_f64x4);
603
+ cov_yz_f64x4 = _mm256_fmadd_pd(a_y_f64x4, b_z_f64x4, cov_yz_f64x4);
604
+ cov_zx_f64x4 = _mm256_fmadd_pd(a_z_f64x4, b_x_f64x4, cov_zx_f64x4);
605
+ cov_zy_f64x4 = _mm256_fmadd_pd(a_z_f64x4, b_y_f64x4, cov_zy_f64x4);
606
+ cov_zz_f64x4 = _mm256_fmadd_pd(a_z_f64x4, b_z_f64x4, cov_zz_f64x4);
607
+ }
608
+
609
+ // Reduce vector accumulators
610
+ nk_f64_t sum_a_x = nk_reduce_stable_f64x4_haswell_(sum_a_x_f64x4), sum_a_x_compensation = 0.0;
611
+ nk_f64_t sum_a_y = nk_reduce_stable_f64x4_haswell_(sum_a_y_f64x4), sum_a_y_compensation = 0.0;
612
+ nk_f64_t sum_a_z = nk_reduce_stable_f64x4_haswell_(sum_a_z_f64x4), sum_a_z_compensation = 0.0;
613
+ nk_f64_t sum_b_x = nk_reduce_stable_f64x4_haswell_(sum_b_x_f64x4), sum_b_x_compensation = 0.0;
614
+ nk_f64_t sum_b_y = nk_reduce_stable_f64x4_haswell_(sum_b_y_f64x4), sum_b_y_compensation = 0.0;
615
+ nk_f64_t sum_b_z = nk_reduce_stable_f64x4_haswell_(sum_b_z_f64x4), sum_b_z_compensation = 0.0;
616
+
617
+ nk_f64_t covariance_x_x = nk_reduce_stable_f64x4_haswell_(cov_xx_f64x4), covariance_x_x_compensation = 0.0;
618
+ nk_f64_t covariance_x_y = nk_reduce_stable_f64x4_haswell_(cov_xy_f64x4), covariance_x_y_compensation = 0.0;
619
+ nk_f64_t covariance_x_z = nk_reduce_stable_f64x4_haswell_(cov_xz_f64x4), covariance_x_z_compensation = 0.0;
620
+ nk_f64_t covariance_y_x = nk_reduce_stable_f64x4_haswell_(cov_yx_f64x4), covariance_y_x_compensation = 0.0;
621
+ nk_f64_t covariance_y_y = nk_reduce_stable_f64x4_haswell_(cov_yy_f64x4), covariance_y_y_compensation = 0.0;
622
+ nk_f64_t covariance_y_z = nk_reduce_stable_f64x4_haswell_(cov_yz_f64x4), covariance_y_z_compensation = 0.0;
623
+ nk_f64_t covariance_z_x = nk_reduce_stable_f64x4_haswell_(cov_zx_f64x4), covariance_z_x_compensation = 0.0;
624
+ nk_f64_t covariance_z_y = nk_reduce_stable_f64x4_haswell_(cov_zy_f64x4), covariance_z_y_compensation = 0.0;
625
+ nk_f64_t covariance_z_z = nk_reduce_stable_f64x4_haswell_(cov_zz_f64x4), covariance_z_z_compensation = 0.0;
626
+
627
+ // Scalar tail
628
+ for (; i < n; ++i) {
629
+ nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
630
+ nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
631
+ nk_accumulate_sum_f64_(&sum_a_x, &sum_a_x_compensation, ax);
632
+ nk_accumulate_sum_f64_(&sum_a_y, &sum_a_y_compensation, ay);
633
+ nk_accumulate_sum_f64_(&sum_a_z, &sum_a_z_compensation, az);
634
+ nk_accumulate_sum_f64_(&sum_b_x, &sum_b_x_compensation, bx);
635
+ nk_accumulate_sum_f64_(&sum_b_y, &sum_b_y_compensation, by);
636
+ nk_accumulate_sum_f64_(&sum_b_z, &sum_b_z_compensation, bz);
637
+ nk_accumulate_product_f64_(&covariance_x_x, &covariance_x_x_compensation, ax, bx);
638
+ nk_accumulate_product_f64_(&covariance_x_y, &covariance_x_y_compensation, ax, by);
639
+ nk_accumulate_product_f64_(&covariance_x_z, &covariance_x_z_compensation, ax, bz);
640
+ nk_accumulate_product_f64_(&covariance_y_x, &covariance_y_x_compensation, ay, bx);
641
+ nk_accumulate_product_f64_(&covariance_y_y, &covariance_y_y_compensation, ay, by);
642
+ nk_accumulate_product_f64_(&covariance_y_z, &covariance_y_z_compensation, ay, bz);
643
+ nk_accumulate_product_f64_(&covariance_z_x, &covariance_z_x_compensation, az, bx);
644
+ nk_accumulate_product_f64_(&covariance_z_y, &covariance_z_y_compensation, az, by);
645
+ nk_accumulate_product_f64_(&covariance_z_z, &covariance_z_z_compensation, az, bz);
646
+ }
647
+
648
+ sum_a_x += sum_a_x_compensation, sum_a_y += sum_a_y_compensation, sum_a_z += sum_a_z_compensation;
649
+ sum_b_x += sum_b_x_compensation, sum_b_y += sum_b_y_compensation, sum_b_z += sum_b_z_compensation;
650
+ covariance_x_x += covariance_x_x_compensation, covariance_x_y += covariance_x_y_compensation,
651
+ covariance_x_z += covariance_x_z_compensation;
652
+ covariance_y_x += covariance_y_x_compensation, covariance_y_y += covariance_y_y_compensation,
653
+ covariance_y_z += covariance_y_z_compensation;
654
+ covariance_z_x += covariance_z_x_compensation, covariance_z_y += covariance_z_y_compensation,
655
+ covariance_z_z += covariance_z_z_compensation;
656
+
657
+ // Compute centroids
658
+ nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
659
+ nk_f64_t centroid_a_x = sum_a_x * inv_n;
660
+ nk_f64_t centroid_a_y = sum_a_y * inv_n;
661
+ nk_f64_t centroid_a_z = sum_a_z * inv_n;
662
+ nk_f64_t centroid_b_x = sum_b_x * inv_n;
663
+ nk_f64_t centroid_b_y = sum_b_y * inv_n;
664
+ nk_f64_t centroid_b_z = sum_b_z * inv_n;
665
+
666
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
667
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
668
+
669
+ // Apply centering correction: H_centered = H - n * centroid_a * centroid_bᵀ
670
+ covariance_x_x -= (nk_f64_t)n * centroid_a_x * centroid_b_x;
671
+ covariance_x_y -= (nk_f64_t)n * centroid_a_x * centroid_b_y;
672
+ covariance_x_z -= (nk_f64_t)n * centroid_a_x * centroid_b_z;
673
+ covariance_y_x -= (nk_f64_t)n * centroid_a_y * centroid_b_x;
674
+ covariance_y_y -= (nk_f64_t)n * centroid_a_y * centroid_b_y;
675
+ covariance_y_z -= (nk_f64_t)n * centroid_a_y * centroid_b_z;
676
+ covariance_z_x -= (nk_f64_t)n * centroid_a_z * centroid_b_x;
677
+ covariance_z_y -= (nk_f64_t)n * centroid_a_z * centroid_b_y;
678
+ covariance_z_z -= (nk_f64_t)n * centroid_a_z * centroid_b_z;
679
+
680
+ // Compute SVD and optimal rotation using f64 precision (svd_s is 9-element diagonal matrix)
681
+ nk_f64_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
682
+ covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
683
+ nk_f64_t svd_u[9], svd_s[9], svd_v[9];
684
+ nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
685
+
686
+ nk_f64_t r[9];
687
+ nk_rotation_from_svd_f64_haswell_(svd_u, svd_v, r);
688
+
689
+ // Handle reflection: if det(R) < 0, negate third column of V and recompute R
690
+ if (nk_det3x3_f64_(r) < 0) {
691
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
692
+ nk_rotation_from_svd_f64_haswell_(svd_u, svd_v, r);
693
+ }
694
+
695
+ // Output rotation matrix and scale=1.0
696
+ if (rotation)
697
+ for (int j = 0; j < 9; ++j) rotation[j] = r[j];
698
+ if (scale) *scale = 1.0;
699
+
700
+ // Compute RMSD after optimal rotation
701
+ nk_f64_t sum_squared = nk_transformed_ssd_f64_haswell_(a, b, n, r, 1.0, centroid_a_x, centroid_a_y, centroid_a_z,
702
+ centroid_b_x, centroid_b_y, centroid_b_z);
703
+ *result = nk_f64_sqrt_haswell(sum_squared * inv_n);
704
+ }
705
+
706
+ NK_PUBLIC void nk_umeyama_f32_haswell(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
707
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
708
+ __m256d sum_a_x_f64x4 = _mm256_setzero_pd(), sum_a_y_f64x4 = _mm256_setzero_pd();
709
+ __m256d sum_a_z_f64x4 = _mm256_setzero_pd(), sum_b_x_f64x4 = _mm256_setzero_pd();
710
+ __m256d sum_b_y_f64x4 = _mm256_setzero_pd(), sum_b_z_f64x4 = _mm256_setzero_pd();
711
+ __m256d covariance_00_f64x4 = _mm256_setzero_pd(), covariance_01_f64x4 = _mm256_setzero_pd();
712
+ __m256d covariance_02_f64x4 = _mm256_setzero_pd(), covariance_10_f64x4 = _mm256_setzero_pd();
713
+ __m256d covariance_11_f64x4 = _mm256_setzero_pd(), covariance_12_f64x4 = _mm256_setzero_pd();
714
+ __m256d covariance_20_f64x4 = _mm256_setzero_pd(), covariance_21_f64x4 = _mm256_setzero_pd();
715
+ __m256d covariance_22_f64x4 = _mm256_setzero_pd(), variance_a_f64x4 = _mm256_setzero_pd();
716
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
717
+ nk_size_t index = 0;
718
+
719
+ for (; index + 8 <= n; index += 8) {
720
+ nk_deinterleave_f32x8_haswell_(a + index * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8),
721
+ nk_deinterleave_f32x8_haswell_(b + index * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
722
+ __m256d a_x_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_x_f32x8));
723
+ __m256d a_x_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_x_f32x8, 1));
724
+ __m256d a_y_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_y_f32x8));
725
+ __m256d a_y_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_y_f32x8, 1));
726
+ __m256d a_z_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(a_z_f32x8));
727
+ __m256d a_z_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(a_z_f32x8, 1));
728
+ __m256d b_x_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_x_f32x8));
729
+ __m256d b_x_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_x_f32x8, 1));
730
+ __m256d b_y_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_y_f32x8));
731
+ __m256d b_y_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_y_f32x8, 1));
732
+ __m256d b_z_low_f64x4 = _mm256_cvtps_pd(_mm256_castps256_ps128(b_z_f32x8));
733
+ __m256d b_z_high_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps(b_z_f32x8, 1));
734
+
735
+ sum_a_x_f64x4 = _mm256_add_pd(sum_a_x_f64x4, _mm256_add_pd(a_x_low_f64x4, a_x_high_f64x4));
736
+ sum_a_y_f64x4 = _mm256_add_pd(sum_a_y_f64x4, _mm256_add_pd(a_y_low_f64x4, a_y_high_f64x4));
737
+ sum_a_z_f64x4 = _mm256_add_pd(sum_a_z_f64x4, _mm256_add_pd(a_z_low_f64x4, a_z_high_f64x4));
738
+ sum_b_x_f64x4 = _mm256_add_pd(sum_b_x_f64x4, _mm256_add_pd(b_x_low_f64x4, b_x_high_f64x4));
739
+ sum_b_y_f64x4 = _mm256_add_pd(sum_b_y_f64x4, _mm256_add_pd(b_y_low_f64x4, b_y_high_f64x4));
740
+ sum_b_z_f64x4 = _mm256_add_pd(sum_b_z_f64x4, _mm256_add_pd(b_z_low_f64x4, b_z_high_f64x4));
741
+ covariance_00_f64x4 = _mm256_add_pd(
742
+ covariance_00_f64x4,
743
+ _mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, b_x_low_f64x4), _mm256_mul_pd(a_x_high_f64x4, b_x_high_f64x4)));
744
+ covariance_01_f64x4 = _mm256_add_pd(
745
+ covariance_01_f64x4,
746
+ _mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, b_y_low_f64x4), _mm256_mul_pd(a_x_high_f64x4, b_y_high_f64x4)));
747
+ covariance_02_f64x4 = _mm256_add_pd(
748
+ covariance_02_f64x4,
749
+ _mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, b_z_low_f64x4), _mm256_mul_pd(a_x_high_f64x4, b_z_high_f64x4)));
750
+ covariance_10_f64x4 = _mm256_add_pd(
751
+ covariance_10_f64x4,
752
+ _mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, b_x_low_f64x4), _mm256_mul_pd(a_y_high_f64x4, b_x_high_f64x4)));
753
+ covariance_11_f64x4 = _mm256_add_pd(
754
+ covariance_11_f64x4,
755
+ _mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, b_y_low_f64x4), _mm256_mul_pd(a_y_high_f64x4, b_y_high_f64x4)));
756
+ covariance_12_f64x4 = _mm256_add_pd(
757
+ covariance_12_f64x4,
758
+ _mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, b_z_low_f64x4), _mm256_mul_pd(a_y_high_f64x4, b_z_high_f64x4)));
759
+ covariance_20_f64x4 = _mm256_add_pd(
760
+ covariance_20_f64x4,
761
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, b_x_low_f64x4), _mm256_mul_pd(a_z_high_f64x4, b_x_high_f64x4)));
762
+ covariance_21_f64x4 = _mm256_add_pd(
763
+ covariance_21_f64x4,
764
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, b_y_low_f64x4), _mm256_mul_pd(a_z_high_f64x4, b_y_high_f64x4)));
765
+ covariance_22_f64x4 = _mm256_add_pd(
766
+ covariance_22_f64x4,
767
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, b_z_low_f64x4), _mm256_mul_pd(a_z_high_f64x4, b_z_high_f64x4)));
768
+ variance_a_f64x4 = _mm256_add_pd(
769
+ variance_a_f64x4,
770
+ _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(a_x_low_f64x4, a_x_low_f64x4),
771
+ _mm256_mul_pd(a_x_high_f64x4, a_x_high_f64x4)),
772
+ _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(a_y_low_f64x4, a_y_low_f64x4),
773
+ _mm256_mul_pd(a_y_high_f64x4, a_y_high_f64x4)),
774
+ _mm256_add_pd(_mm256_mul_pd(a_z_low_f64x4, a_z_low_f64x4),
775
+ _mm256_mul_pd(a_z_high_f64x4, a_z_high_f64x4)))));
776
+ }
777
+
778
+ nk_f64_t sum_a_x = nk_reduce_add_f64x4_haswell_(sum_a_x_f64x4);
779
+ nk_f64_t sum_a_y = nk_reduce_add_f64x4_haswell_(sum_a_y_f64x4);
780
+ nk_f64_t sum_a_z = nk_reduce_add_f64x4_haswell_(sum_a_z_f64x4);
781
+ nk_f64_t sum_b_x = nk_reduce_add_f64x4_haswell_(sum_b_x_f64x4);
782
+ nk_f64_t sum_b_y = nk_reduce_add_f64x4_haswell_(sum_b_y_f64x4);
783
+ nk_f64_t sum_b_z = nk_reduce_add_f64x4_haswell_(sum_b_z_f64x4);
784
+ nk_f64_t h[9] = {
785
+ nk_reduce_add_f64x4_haswell_(covariance_00_f64x4), nk_reduce_add_f64x4_haswell_(covariance_01_f64x4),
786
+ nk_reduce_add_f64x4_haswell_(covariance_02_f64x4), nk_reduce_add_f64x4_haswell_(covariance_10_f64x4),
787
+ nk_reduce_add_f64x4_haswell_(covariance_11_f64x4), nk_reduce_add_f64x4_haswell_(covariance_12_f64x4),
788
+ nk_reduce_add_f64x4_haswell_(covariance_20_f64x4), nk_reduce_add_f64x4_haswell_(covariance_21_f64x4),
789
+ nk_reduce_add_f64x4_haswell_(covariance_22_f64x4)};
790
+ nk_f64_t variance_a = nk_reduce_add_f64x4_haswell_(variance_a_f64x4);
791
+
792
+ for (; index < n; ++index) {
793
+ nk_f64_t a_x = a[index * 3 + 0], a_y = a[index * 3 + 1], a_z = a[index * 3 + 2];
794
+ nk_f64_t b_x = b[index * 3 + 0], b_y = b[index * 3 + 1], b_z = b[index * 3 + 2];
795
+ sum_a_x += a_x, sum_a_y += a_y, sum_a_z += a_z;
796
+ sum_b_x += b_x, sum_b_y += b_y, sum_b_z += b_z;
797
+ h[0] += a_x * b_x, h[1] += a_x * b_y, h[2] += a_x * b_z;
798
+ h[3] += a_y * b_x, h[4] += a_y * b_y, h[5] += a_y * b_z;
799
+ h[6] += a_z * b_x, h[7] += a_z * b_y, h[8] += a_z * b_z;
800
+ variance_a += a_x * a_x + a_y * a_y + a_z * a_z;
801
+ }
802
+
803
+ nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
804
+ nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
805
+ nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
806
+ if (a_centroid)
807
+ a_centroid[0] = (nk_f32_t)centroid_a_x, a_centroid[1] = (nk_f32_t)centroid_a_y,
808
+ a_centroid[2] = (nk_f32_t)centroid_a_z;
809
+ if (b_centroid)
810
+ b_centroid[0] = (nk_f32_t)centroid_b_x, b_centroid[1] = (nk_f32_t)centroid_b_y,
811
+ b_centroid[2] = (nk_f32_t)centroid_b_z;
812
+
813
+ variance_a = variance_a * inv_n -
814
+ (centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
815
+ h[0] -= (nk_f64_t)n * centroid_a_x * centroid_b_x, h[1] -= (nk_f64_t)n * centroid_a_x * centroid_b_y,
816
+ h[2] -= (nk_f64_t)n * centroid_a_x * centroid_b_z, h[3] -= (nk_f64_t)n * centroid_a_y * centroid_b_x,
817
+ h[4] -= (nk_f64_t)n * centroid_a_y * centroid_b_y, h[5] -= (nk_f64_t)n * centroid_a_y * centroid_b_z,
818
+ h[6] -= (nk_f64_t)n * centroid_a_z * centroid_b_x, h[7] -= (nk_f64_t)n * centroid_a_z * centroid_b_y,
819
+ h[8] -= (nk_f64_t)n * centroid_a_z * centroid_b_z;
820
+
821
+ nk_f64_t cross_covariance[9] = {h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], h[8]};
822
+ nk_f64_t svd_u[9], svd_s[9], svd_v[9], r[9];
823
+ nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
824
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
825
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
826
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
827
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
828
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
829
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
830
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
831
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
832
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
833
+
834
+ nk_f64_t det = nk_det3x3_f64_(r), sign_correction = det < 0 ? -1.0 : 1.0;
835
+ if (det < 0) {
836
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
837
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
838
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
839
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
840
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
841
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
842
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
843
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
844
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
845
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
846
+ }
847
+
848
+ nk_f64_t applied_scale = (svd_s[0] + svd_s[4] + sign_correction * svd_s[8]) / ((nk_f64_t)n * variance_a);
849
+ if (rotation)
850
+ for (int j = 0; j != 9; ++j) rotation[j] = (nk_f32_t)r[j];
851
+ if (scale) *scale = (nk_f32_t)applied_scale;
852
+ *result = nk_f64_sqrt_haswell(nk_transformed_ssd_f32_haswell_(a, b, n, r, applied_scale, centroid_a_x, centroid_a_y,
853
+ centroid_a_z, centroid_b_x, centroid_b_y,
854
+ centroid_b_z) /
855
+ (nk_f64_t)n);
856
+ }
857
+
858
+ NK_PUBLIC void nk_umeyama_f64_haswell(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
859
+ nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
860
+ // Fused single-pass: centroids, covariance, and variance of A
861
+ __m256d const zeros_f64x4 = _mm256_setzero_pd();
862
+
863
+ __m256d sum_a_x_f64x4 = zeros_f64x4, sum_a_y_f64x4 = zeros_f64x4, sum_a_z_f64x4 = zeros_f64x4;
864
+ __m256d sum_b_x_f64x4 = zeros_f64x4, sum_b_y_f64x4 = zeros_f64x4, sum_b_z_f64x4 = zeros_f64x4;
865
+ __m256d cov_xx_f64x4 = zeros_f64x4, cov_xy_f64x4 = zeros_f64x4, cov_xz_f64x4 = zeros_f64x4;
866
+ __m256d cov_yx_f64x4 = zeros_f64x4, cov_yy_f64x4 = zeros_f64x4, cov_yz_f64x4 = zeros_f64x4;
867
+ __m256d cov_zx_f64x4 = zeros_f64x4, cov_zy_f64x4 = zeros_f64x4, cov_zz_f64x4 = zeros_f64x4;
868
+ __m256d variance_a_f64x4 = zeros_f64x4;
869
+
870
+ nk_size_t i = 0;
871
+ __m256d a_x_f64x4, a_y_f64x4, a_z_f64x4, b_x_f64x4, b_y_f64x4, b_z_f64x4;
872
+
873
+ for (; i + 4 <= n; i += 4) {
874
+ nk_deinterleave_f64x4_haswell_(a + i * 3, &a_x_f64x4, &a_y_f64x4, &a_z_f64x4);
875
+ nk_deinterleave_f64x4_haswell_(b + i * 3, &b_x_f64x4, &b_y_f64x4, &b_z_f64x4);
876
+
877
+ sum_a_x_f64x4 = _mm256_add_pd(sum_a_x_f64x4, a_x_f64x4),
878
+ sum_a_y_f64x4 = _mm256_add_pd(sum_a_y_f64x4, a_y_f64x4);
879
+ sum_a_z_f64x4 = _mm256_add_pd(sum_a_z_f64x4, a_z_f64x4);
880
+ sum_b_x_f64x4 = _mm256_add_pd(sum_b_x_f64x4, b_x_f64x4),
881
+ sum_b_y_f64x4 = _mm256_add_pd(sum_b_y_f64x4, b_y_f64x4);
882
+ sum_b_z_f64x4 = _mm256_add_pd(sum_b_z_f64x4, b_z_f64x4);
883
+
884
+ cov_xx_f64x4 = _mm256_fmadd_pd(a_x_f64x4, b_x_f64x4, cov_xx_f64x4),
885
+ cov_xy_f64x4 = _mm256_fmadd_pd(a_x_f64x4, b_y_f64x4, cov_xy_f64x4);
886
+ cov_xz_f64x4 = _mm256_fmadd_pd(a_x_f64x4, b_z_f64x4, cov_xz_f64x4);
887
+ cov_yx_f64x4 = _mm256_fmadd_pd(a_y_f64x4, b_x_f64x4, cov_yx_f64x4),
888
+ cov_yy_f64x4 = _mm256_fmadd_pd(a_y_f64x4, b_y_f64x4, cov_yy_f64x4);
889
+ cov_yz_f64x4 = _mm256_fmadd_pd(a_y_f64x4, b_z_f64x4, cov_yz_f64x4);
890
+ cov_zx_f64x4 = _mm256_fmadd_pd(a_z_f64x4, b_x_f64x4, cov_zx_f64x4),
891
+ cov_zy_f64x4 = _mm256_fmadd_pd(a_z_f64x4, b_y_f64x4, cov_zy_f64x4);
892
+ cov_zz_f64x4 = _mm256_fmadd_pd(a_z_f64x4, b_z_f64x4, cov_zz_f64x4);
893
+ variance_a_f64x4 = _mm256_fmadd_pd(a_x_f64x4, a_x_f64x4, variance_a_f64x4);
894
+ variance_a_f64x4 = _mm256_fmadd_pd(a_y_f64x4, a_y_f64x4, variance_a_f64x4);
895
+ variance_a_f64x4 = _mm256_fmadd_pd(a_z_f64x4, a_z_f64x4, variance_a_f64x4);
896
+ }
897
+
898
+ // Reduce vector accumulators
899
+ nk_f64_t sum_a_x = nk_reduce_stable_f64x4_haswell_(sum_a_x_f64x4), sum_a_x_compensation = 0.0;
900
+ nk_f64_t sum_a_y = nk_reduce_stable_f64x4_haswell_(sum_a_y_f64x4), sum_a_y_compensation = 0.0;
901
+ nk_f64_t sum_a_z = nk_reduce_stable_f64x4_haswell_(sum_a_z_f64x4), sum_a_z_compensation = 0.0;
902
+ nk_f64_t sum_b_x = nk_reduce_stable_f64x4_haswell_(sum_b_x_f64x4), sum_b_x_compensation = 0.0;
903
+ nk_f64_t sum_b_y = nk_reduce_stable_f64x4_haswell_(sum_b_y_f64x4), sum_b_y_compensation = 0.0;
904
+ nk_f64_t sum_b_z = nk_reduce_stable_f64x4_haswell_(sum_b_z_f64x4), sum_b_z_compensation = 0.0;
905
+ nk_f64_t covariance_x_x = nk_reduce_stable_f64x4_haswell_(cov_xx_f64x4), covariance_x_x_compensation = 0.0;
906
+ nk_f64_t covariance_x_y = nk_reduce_stable_f64x4_haswell_(cov_xy_f64x4), covariance_x_y_compensation = 0.0;
907
+ nk_f64_t covariance_x_z = nk_reduce_stable_f64x4_haswell_(cov_xz_f64x4), covariance_x_z_compensation = 0.0;
908
+ nk_f64_t covariance_y_x = nk_reduce_stable_f64x4_haswell_(cov_yx_f64x4), covariance_y_x_compensation = 0.0;
909
+ nk_f64_t covariance_y_y = nk_reduce_stable_f64x4_haswell_(cov_yy_f64x4), covariance_y_y_compensation = 0.0;
910
+ nk_f64_t covariance_y_z = nk_reduce_stable_f64x4_haswell_(cov_yz_f64x4), covariance_y_z_compensation = 0.0;
911
+ nk_f64_t covariance_z_x = nk_reduce_stable_f64x4_haswell_(cov_zx_f64x4), covariance_z_x_compensation = 0.0;
912
+ nk_f64_t covariance_z_y = nk_reduce_stable_f64x4_haswell_(cov_zy_f64x4), covariance_z_y_compensation = 0.0;
913
+ nk_f64_t covariance_z_z = nk_reduce_stable_f64x4_haswell_(cov_zz_f64x4), covariance_z_z_compensation = 0.0;
914
+ nk_f64_t variance_a_sum = nk_reduce_stable_f64x4_haswell_(variance_a_f64x4), variance_a_compensation = 0.0;
915
+
916
+ // Scalar tail loop for remaining points
917
+ for (; i < n; i++) {
918
+ nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
919
+ nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
920
+ nk_accumulate_sum_f64_(&sum_a_x, &sum_a_x_compensation, ax);
921
+ nk_accumulate_sum_f64_(&sum_a_y, &sum_a_y_compensation, ay);
922
+ nk_accumulate_sum_f64_(&sum_a_z, &sum_a_z_compensation, az);
923
+ nk_accumulate_sum_f64_(&sum_b_x, &sum_b_x_compensation, bx);
924
+ nk_accumulate_sum_f64_(&sum_b_y, &sum_b_y_compensation, by);
925
+ nk_accumulate_sum_f64_(&sum_b_z, &sum_b_z_compensation, bz);
926
+ nk_accumulate_product_f64_(&covariance_x_x, &covariance_x_x_compensation, ax, bx);
927
+ nk_accumulate_product_f64_(&covariance_x_y, &covariance_x_y_compensation, ax, by);
928
+ nk_accumulate_product_f64_(&covariance_x_z, &covariance_x_z_compensation, ax, bz);
929
+ nk_accumulate_product_f64_(&covariance_y_x, &covariance_y_x_compensation, ay, bx);
930
+ nk_accumulate_product_f64_(&covariance_y_y, &covariance_y_y_compensation, ay, by);
931
+ nk_accumulate_product_f64_(&covariance_y_z, &covariance_y_z_compensation, ay, bz);
932
+ nk_accumulate_product_f64_(&covariance_z_x, &covariance_z_x_compensation, az, bx);
933
+ nk_accumulate_product_f64_(&covariance_z_y, &covariance_z_y_compensation, az, by);
934
+ nk_accumulate_product_f64_(&covariance_z_z, &covariance_z_z_compensation, az, bz);
935
+ nk_accumulate_square_f64_(&variance_a_sum, &variance_a_compensation, ax);
936
+ nk_accumulate_square_f64_(&variance_a_sum, &variance_a_compensation, ay);
937
+ nk_accumulate_square_f64_(&variance_a_sum, &variance_a_compensation, az);
938
+ }
939
+
940
+ sum_a_x += sum_a_x_compensation, sum_a_y += sum_a_y_compensation, sum_a_z += sum_a_z_compensation;
941
+ sum_b_x += sum_b_x_compensation, sum_b_y += sum_b_y_compensation, sum_b_z += sum_b_z_compensation;
942
+ covariance_x_x += covariance_x_x_compensation, covariance_x_y += covariance_x_y_compensation,
943
+ covariance_x_z += covariance_x_z_compensation;
944
+ covariance_y_x += covariance_y_x_compensation, covariance_y_y += covariance_y_y_compensation,
945
+ covariance_y_z += covariance_y_z_compensation;
946
+ covariance_z_x += covariance_z_x_compensation, covariance_z_y += covariance_z_y_compensation,
947
+ covariance_z_z += covariance_z_z_compensation;
948
+ variance_a_sum += variance_a_compensation;
949
+
950
+ // Compute centroids
951
+ nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
952
+
953
+ nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
954
+ nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
955
+
956
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
957
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
958
+
959
+ // Compute centered covariance and variance
960
+ nk_f64_t variance_a = variance_a_sum * inv_n -
961
+ (centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
962
+
963
+ nk_f64_t cross_covariance[9];
964
+ cross_covariance[0] = covariance_x_x - sum_a_x * sum_b_x * inv_n;
965
+ cross_covariance[1] = covariance_x_y - sum_a_x * sum_b_y * inv_n;
966
+ cross_covariance[2] = covariance_x_z - sum_a_x * sum_b_z * inv_n;
967
+ cross_covariance[3] = covariance_y_x - sum_a_y * sum_b_x * inv_n;
968
+ cross_covariance[4] = covariance_y_y - sum_a_y * sum_b_y * inv_n;
969
+ cross_covariance[5] = covariance_y_z - sum_a_y * sum_b_z * inv_n;
970
+ cross_covariance[6] = covariance_z_x - sum_a_z * sum_b_x * inv_n;
971
+ cross_covariance[7] = covariance_z_y - sum_a_z * sum_b_y * inv_n;
972
+ cross_covariance[8] = covariance_z_z - sum_a_z * sum_b_z * inv_n;
973
+
974
+ // SVD using f64 for full precision (svd_s is 9-element diagonal matrix)
975
+ nk_f64_t svd_u[9], svd_s[9], svd_v[9];
976
+ nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
977
+
978
+ nk_f64_t r[9];
979
+ nk_rotation_from_svd_f64_haswell_(svd_u, svd_v, r);
980
+
981
+ // Scale factor: c = trace(D × S) / (n × variance(a))
982
+ // svd_s diagonal: [0], [4], [8]
983
+ nk_f64_t det = nk_det3x3_f64_(r);
984
+ nk_f64_t d3 = det < 0 ? -1.0 : 1.0;
985
+ nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_s[0], 1.0, svd_s[4], 1.0, svd_s[8], d3);
986
+ nk_f64_t c = trace_ds / ((nk_f64_t)n * variance_a);
987
+ if (scale) *scale = c;
988
+
989
+ // Handle reflection
990
+ if (det < 0) {
991
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
992
+ nk_rotation_from_svd_f64_haswell_(svd_u, svd_v, r);
993
+ }
994
+
995
+ // Output rotation matrix
996
+ if (rotation)
997
+ for (int j = 0; j < 9; ++j) rotation[j] = r[j];
998
+
999
+ // Compute RMSD with scaling
1000
+ nk_f64_t sum_squared = nk_transformed_ssd_f64_haswell_(a, b, n, r, c, centroid_a_x, centroid_a_y, centroid_a_z,
1001
+ centroid_b_x, centroid_b_y, centroid_b_z);
1002
+ *result = nk_f64_sqrt_haswell(sum_squared * inv_n);
1003
+ }
1004
+
1005
+ /* Deinterleave 8 f16 xyz triplets (24 f16 values) and convert to 3 x __m256 f32.
1006
+ * Uses scalar extraction for clean stride-3 access, then F16C conversion.
1007
+ *
1008
+ * Input: 24 contiguous f16 [x0,y0,z0, x1,y1,z1, ..., x7,y7,z7]
1009
+ * Output: x[8], y[8], z[8] vectors in f32
1010
+ */
1011
+ NK_INTERNAL void nk_deinterleave_f16x8_to_f32x8_haswell_(nk_f16_t const *ptr, __m256 *x_out, __m256 *y_out,
1012
+ __m256 *z_out) {
1013
+ // Extract x, y, z components with stride-3 access
1014
+ nk_b256_vec_t x_vec, y_vec, z_vec;
1015
+ x_vec.f16s[0] = ptr[0], x_vec.f16s[1] = ptr[3], x_vec.f16s[2] = ptr[6], x_vec.f16s[3] = ptr[9];
1016
+ x_vec.f16s[4] = ptr[12], x_vec.f16s[5] = ptr[15], x_vec.f16s[6] = ptr[18], x_vec.f16s[7] = ptr[21];
1017
+ y_vec.f16s[0] = ptr[1], y_vec.f16s[1] = ptr[4], y_vec.f16s[2] = ptr[7], y_vec.f16s[3] = ptr[10];
1018
+ y_vec.f16s[4] = ptr[13], y_vec.f16s[5] = ptr[16], y_vec.f16s[6] = ptr[19], y_vec.f16s[7] = ptr[22];
1019
+ z_vec.f16s[0] = ptr[2], z_vec.f16s[1] = ptr[5], z_vec.f16s[2] = ptr[8], z_vec.f16s[3] = ptr[11];
1020
+ z_vec.f16s[4] = ptr[14], z_vec.f16s[5] = ptr[17], z_vec.f16s[6] = ptr[20], z_vec.f16s[7] = ptr[23];
1021
+ // Convert f16 to f32 using F16C
1022
+ *x_out = _mm256_cvtph_ps(x_vec.xmms[0]);
1023
+ *y_out = _mm256_cvtph_ps(y_vec.xmms[0]);
1024
+ *z_out = _mm256_cvtph_ps(z_vec.xmms[0]);
1025
+ }
1026
+
1027
+ /* Deinterleave 8 bf16 xyz triplets (24 bf16 values) and convert to 3 x __m256 f32.
1028
+ * Uses scalar extraction for clean stride-3 access, then bit-shift conversion.
1029
+ *
1030
+ * Input: 24 contiguous bf16 [x0,y0,z0, x1,y1,z1, ..., x7,y7,z7]
1031
+ * Output: x[8], y[8], z[8] vectors in f32
1032
+ */
1033
+ NK_INTERNAL void nk_deinterleave_bf16x8_to_f32x8_haswell_(nk_bf16_t const *ptr, __m256 *x_out, __m256 *y_out,
1034
+ __m256 *z_out) {
1035
+ // Extract x, y, z components with stride-3 access
1036
+ nk_b256_vec_t x_vec, y_vec, z_vec;
1037
+ x_vec.bf16s[0] = ptr[0], x_vec.bf16s[1] = ptr[3], x_vec.bf16s[2] = ptr[6], x_vec.bf16s[3] = ptr[9];
1038
+ x_vec.bf16s[4] = ptr[12], x_vec.bf16s[5] = ptr[15], x_vec.bf16s[6] = ptr[18], x_vec.bf16s[7] = ptr[21];
1039
+ y_vec.bf16s[0] = ptr[1], y_vec.bf16s[1] = ptr[4], y_vec.bf16s[2] = ptr[7], y_vec.bf16s[3] = ptr[10];
1040
+ y_vec.bf16s[4] = ptr[13], y_vec.bf16s[5] = ptr[16], y_vec.bf16s[6] = ptr[19], y_vec.bf16s[7] = ptr[22];
1041
+ z_vec.bf16s[0] = ptr[2], z_vec.bf16s[1] = ptr[5], z_vec.bf16s[2] = ptr[8], z_vec.bf16s[3] = ptr[11];
1042
+ z_vec.bf16s[4] = ptr[14], z_vec.bf16s[5] = ptr[17], z_vec.bf16s[6] = ptr[20], z_vec.bf16s[7] = ptr[23];
1043
+ // Convert bf16 to f32 by left-shifting 16 bits
1044
+ *x_out = nk_bf16x8_to_f32x8_haswell_(x_vec.xmms[0]);
1045
+ *y_out = nk_bf16x8_to_f32x8_haswell_(y_vec.xmms[0]);
1046
+ *z_out = nk_bf16x8_to_f32x8_haswell_(z_vec.xmms[0]);
1047
+ }
1048
+
1049
+ /* Compute sum of squared distances for f16 data after applying rotation (and optional scale).
1050
+ * Loads f16 data, converts to f32 during processing.
1051
+ * Note: rotation matrix r is f32 (from SVD), scale and computation done in f32.
1052
+ */
1053
+ NK_INTERNAL nk_f32_t nk_transformed_ssd_f16_haswell_(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n,
1054
+ nk_f32_t const *r, nk_f32_t scale, nk_f32_t centroid_a_x,
1055
+ nk_f32_t centroid_a_y, nk_f32_t centroid_a_z,
1056
+ nk_f32_t centroid_b_x, nk_f32_t centroid_b_y,
1057
+ nk_f32_t centroid_b_z) {
1058
+ // Broadcast scaled rotation matrix elements
1059
+ __m256 scaled_rotation_x_x_f32x8 = _mm256_set1_ps(scale * r[0]);
1060
+ __m256 scaled_rotation_x_y_f32x8 = _mm256_set1_ps(scale * r[1]);
1061
+ __m256 scaled_rotation_x_z_f32x8 = _mm256_set1_ps(scale * r[2]);
1062
+ __m256 scaled_rotation_y_x_f32x8 = _mm256_set1_ps(scale * r[3]);
1063
+ __m256 scaled_rotation_y_y_f32x8 = _mm256_set1_ps(scale * r[4]);
1064
+ __m256 scaled_rotation_y_z_f32x8 = _mm256_set1_ps(scale * r[5]);
1065
+ __m256 scaled_rotation_z_x_f32x8 = _mm256_set1_ps(scale * r[6]);
1066
+ __m256 scaled_rotation_z_y_f32x8 = _mm256_set1_ps(scale * r[7]);
1067
+ __m256 scaled_rotation_z_z_f32x8 = _mm256_set1_ps(scale * r[8]);
1068
+
1069
+ // Broadcast centroids
1070
+ __m256 centroid_a_x_f32x8 = _mm256_set1_ps(centroid_a_x);
1071
+ __m256 centroid_a_y_f32x8 = _mm256_set1_ps(centroid_a_y);
1072
+ __m256 centroid_a_z_f32x8 = _mm256_set1_ps(centroid_a_z);
1073
+ __m256 centroid_b_x_f32x8 = _mm256_set1_ps(centroid_b_x);
1074
+ __m256 centroid_b_y_f32x8 = _mm256_set1_ps(centroid_b_y);
1075
+ __m256 centroid_b_z_f32x8 = _mm256_set1_ps(centroid_b_z);
1076
+
1077
+ __m256 sum_squared_f32x8 = _mm256_setzero_ps();
1078
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1079
+ nk_size_t j = 0;
1080
+
1081
+ for (; j + 8 <= n; j += 8) {
1082
+ nk_deinterleave_f16x8_to_f32x8_haswell_(a + j * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1083
+ nk_deinterleave_f16x8_to_f32x8_haswell_(b + j * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1084
+
1085
+ // Center points
1086
+ __m256 pa_x_f32x8 = _mm256_sub_ps(a_x_f32x8, centroid_a_x_f32x8);
1087
+ __m256 pa_y_f32x8 = _mm256_sub_ps(a_y_f32x8, centroid_a_y_f32x8);
1088
+ __m256 pa_z_f32x8 = _mm256_sub_ps(a_z_f32x8, centroid_a_z_f32x8);
1089
+ __m256 pb_x_f32x8 = _mm256_sub_ps(b_x_f32x8, centroid_b_x_f32x8);
1090
+ __m256 pb_y_f32x8 = _mm256_sub_ps(b_y_f32x8, centroid_b_y_f32x8);
1091
+ __m256 pb_z_f32x8 = _mm256_sub_ps(b_z_f32x8, centroid_b_z_f32x8);
1092
+
1093
+ // Rotate and scale: ra = scale * R * pa
1094
+ __m256 ra_x_f32x8 = _mm256_fmadd_ps(scaled_rotation_x_z_f32x8, pa_z_f32x8,
1095
+ _mm256_fmadd_ps(scaled_rotation_x_y_f32x8, pa_y_f32x8,
1096
+ _mm256_mul_ps(scaled_rotation_x_x_f32x8, pa_x_f32x8)));
1097
+ __m256 ra_y_f32x8 = _mm256_fmadd_ps(scaled_rotation_y_z_f32x8, pa_z_f32x8,
1098
+ _mm256_fmadd_ps(scaled_rotation_y_y_f32x8, pa_y_f32x8,
1099
+ _mm256_mul_ps(scaled_rotation_y_x_f32x8, pa_x_f32x8)));
1100
+ __m256 ra_z_f32x8 = _mm256_fmadd_ps(scaled_rotation_z_z_f32x8, pa_z_f32x8,
1101
+ _mm256_fmadd_ps(scaled_rotation_z_y_f32x8, pa_y_f32x8,
1102
+ _mm256_mul_ps(scaled_rotation_z_x_f32x8, pa_x_f32x8)));
1103
+
1104
+ // Delta and accumulate
1105
+ __m256 delta_x_f32x8 = _mm256_sub_ps(ra_x_f32x8, pb_x_f32x8);
1106
+ __m256 delta_y_f32x8 = _mm256_sub_ps(ra_y_f32x8, pb_y_f32x8);
1107
+ __m256 delta_z_f32x8 = _mm256_sub_ps(ra_z_f32x8, pb_z_f32x8);
1108
+
1109
+ sum_squared_f32x8 = _mm256_fmadd_ps(delta_x_f32x8, delta_x_f32x8, sum_squared_f32x8);
1110
+ sum_squared_f32x8 = _mm256_fmadd_ps(delta_y_f32x8, delta_y_f32x8, sum_squared_f32x8);
1111
+ sum_squared_f32x8 = _mm256_fmadd_ps(delta_z_f32x8, delta_z_f32x8, sum_squared_f32x8);
1112
+ }
1113
+
1114
+ nk_f32_t sum_squared = nk_reduce_add_f32x8_haswell_(sum_squared_f32x8);
1115
+
1116
+ // Scalar tail
1117
+ for (; j < n; ++j) {
1118
+ nk_f32_t a_x_f32, a_y_f32, a_z_f32, b_x_f32, b_y_f32, b_z_f32;
1119
+ nk_f16_to_f32_haswell(&a[j * 3 + 0], &a_x_f32);
1120
+ nk_f16_to_f32_haswell(&a[j * 3 + 1], &a_y_f32);
1121
+ nk_f16_to_f32_haswell(&a[j * 3 + 2], &a_z_f32);
1122
+ nk_f16_to_f32_haswell(&b[j * 3 + 0], &b_x_f32);
1123
+ nk_f16_to_f32_haswell(&b[j * 3 + 1], &b_y_f32);
1124
+ nk_f16_to_f32_haswell(&b[j * 3 + 2], &b_z_f32);
1125
+
1126
+ nk_f32_t pa_x = a_x_f32 - centroid_a_x, pa_y = a_y_f32 - centroid_a_y, pa_z = a_z_f32 - centroid_a_z;
1127
+ nk_f32_t pb_x = b_x_f32 - centroid_b_x, pb_y = b_y_f32 - centroid_b_y, pb_z = b_z_f32 - centroid_b_z;
1128
+ nk_f32_t ra_x = scale * (r[0] * pa_x + r[1] * pa_y + r[2] * pa_z),
1129
+ ra_y = scale * (r[3] * pa_x + r[4] * pa_y + r[5] * pa_z),
1130
+ ra_z = scale * (r[6] * pa_x + r[7] * pa_y + r[8] * pa_z);
1131
+
1132
+ nk_f32_t delta_x = ra_x - pb_x, delta_y = ra_y - pb_y, delta_z = ra_z - pb_z;
1133
+ sum_squared += delta_x * delta_x + delta_y * delta_y + delta_z * delta_z;
1134
+ }
1135
+
1136
+ return sum_squared;
1137
+ }
1138
+
1139
+ /* Compute sum of squared distances for bf16 data after applying rotation (and optional scale).
1140
+ * Loads bf16 data, converts to f32 during processing.
1141
+ * Note: rotation matrix r is f32 (from SVD), scale and computation done in f32.
1142
+ */
1143
+ NK_INTERNAL nk_f32_t nk_transformed_ssd_bf16_haswell_(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n,
1144
+ nk_f32_t const *r, nk_f32_t scale, nk_f32_t centroid_a_x,
1145
+ nk_f32_t centroid_a_y, nk_f32_t centroid_a_z,
1146
+ nk_f32_t centroid_b_x, nk_f32_t centroid_b_y,
1147
+ nk_f32_t centroid_b_z) {
1148
+ // Broadcast scaled rotation matrix elements
1149
+ __m256 scaled_rotation_x_x_f32x8 = _mm256_set1_ps(scale * r[0]);
1150
+ __m256 scaled_rotation_x_y_f32x8 = _mm256_set1_ps(scale * r[1]);
1151
+ __m256 scaled_rotation_x_z_f32x8 = _mm256_set1_ps(scale * r[2]);
1152
+ __m256 scaled_rotation_y_x_f32x8 = _mm256_set1_ps(scale * r[3]);
1153
+ __m256 scaled_rotation_y_y_f32x8 = _mm256_set1_ps(scale * r[4]);
1154
+ __m256 scaled_rotation_y_z_f32x8 = _mm256_set1_ps(scale * r[5]);
1155
+ __m256 scaled_rotation_z_x_f32x8 = _mm256_set1_ps(scale * r[6]);
1156
+ __m256 scaled_rotation_z_y_f32x8 = _mm256_set1_ps(scale * r[7]);
1157
+ __m256 scaled_rotation_z_z_f32x8 = _mm256_set1_ps(scale * r[8]);
1158
+
1159
+ // Broadcast centroids
1160
+ __m256 centroid_a_x_f32x8 = _mm256_set1_ps(centroid_a_x);
1161
+ __m256 centroid_a_y_f32x8 = _mm256_set1_ps(centroid_a_y);
1162
+ __m256 centroid_a_z_f32x8 = _mm256_set1_ps(centroid_a_z);
1163
+ __m256 centroid_b_x_f32x8 = _mm256_set1_ps(centroid_b_x);
1164
+ __m256 centroid_b_y_f32x8 = _mm256_set1_ps(centroid_b_y);
1165
+ __m256 centroid_b_z_f32x8 = _mm256_set1_ps(centroid_b_z);
1166
+
1167
+ __m256 sum_squared_f32x8 = _mm256_setzero_ps();
1168
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1169
+ nk_size_t j = 0;
1170
+
1171
+ for (; j + 8 <= n; j += 8) {
1172
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(a + j * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1173
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(b + j * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1174
+
1175
+ // Center points
1176
+ __m256 pa_x_f32x8 = _mm256_sub_ps(a_x_f32x8, centroid_a_x_f32x8);
1177
+ __m256 pa_y_f32x8 = _mm256_sub_ps(a_y_f32x8, centroid_a_y_f32x8);
1178
+ __m256 pa_z_f32x8 = _mm256_sub_ps(a_z_f32x8, centroid_a_z_f32x8);
1179
+ __m256 pb_x_f32x8 = _mm256_sub_ps(b_x_f32x8, centroid_b_x_f32x8);
1180
+ __m256 pb_y_f32x8 = _mm256_sub_ps(b_y_f32x8, centroid_b_y_f32x8);
1181
+ __m256 pb_z_f32x8 = _mm256_sub_ps(b_z_f32x8, centroid_b_z_f32x8);
1182
+
1183
+ // Rotate and scale: ra = scale * R * pa
1184
+ __m256 ra_x_f32x8 = _mm256_fmadd_ps(scaled_rotation_x_z_f32x8, pa_z_f32x8,
1185
+ _mm256_fmadd_ps(scaled_rotation_x_y_f32x8, pa_y_f32x8,
1186
+ _mm256_mul_ps(scaled_rotation_x_x_f32x8, pa_x_f32x8)));
1187
+ __m256 ra_y_f32x8 = _mm256_fmadd_ps(scaled_rotation_y_z_f32x8, pa_z_f32x8,
1188
+ _mm256_fmadd_ps(scaled_rotation_y_y_f32x8, pa_y_f32x8,
1189
+ _mm256_mul_ps(scaled_rotation_y_x_f32x8, pa_x_f32x8)));
1190
+ __m256 ra_z_f32x8 = _mm256_fmadd_ps(scaled_rotation_z_z_f32x8, pa_z_f32x8,
1191
+ _mm256_fmadd_ps(scaled_rotation_z_y_f32x8, pa_y_f32x8,
1192
+ _mm256_mul_ps(scaled_rotation_z_x_f32x8, pa_x_f32x8)));
1193
+
1194
+ // Delta and accumulate
1195
+ __m256 delta_x_f32x8 = _mm256_sub_ps(ra_x_f32x8, pb_x_f32x8);
1196
+ __m256 delta_y_f32x8 = _mm256_sub_ps(ra_y_f32x8, pb_y_f32x8);
1197
+ __m256 delta_z_f32x8 = _mm256_sub_ps(ra_z_f32x8, pb_z_f32x8);
1198
+
1199
+ sum_squared_f32x8 = _mm256_fmadd_ps(delta_x_f32x8, delta_x_f32x8, sum_squared_f32x8);
1200
+ sum_squared_f32x8 = _mm256_fmadd_ps(delta_y_f32x8, delta_y_f32x8, sum_squared_f32x8);
1201
+ sum_squared_f32x8 = _mm256_fmadd_ps(delta_z_f32x8, delta_z_f32x8, sum_squared_f32x8);
1202
+ }
1203
+
1204
+ nk_f32_t sum_squared = nk_reduce_add_f32x8_haswell_(sum_squared_f32x8);
1205
+
1206
+ // Scalar tail
1207
+ for (; j < n; ++j) {
1208
+ nk_f32_t a_x_f32, a_y_f32, a_z_f32, b_x_f32, b_y_f32, b_z_f32;
1209
+ nk_bf16_to_f32_serial(&a[j * 3 + 0], &a_x_f32);
1210
+ nk_bf16_to_f32_serial(&a[j * 3 + 1], &a_y_f32);
1211
+ nk_bf16_to_f32_serial(&a[j * 3 + 2], &a_z_f32);
1212
+ nk_bf16_to_f32_serial(&b[j * 3 + 0], &b_x_f32);
1213
+ nk_bf16_to_f32_serial(&b[j * 3 + 1], &b_y_f32);
1214
+ nk_bf16_to_f32_serial(&b[j * 3 + 2], &b_z_f32);
1215
+
1216
+ nk_f32_t pa_x = a_x_f32 - centroid_a_x, pa_y = a_y_f32 - centroid_a_y, pa_z = a_z_f32 - centroid_a_z;
1217
+ nk_f32_t pb_x = b_x_f32 - centroid_b_x, pb_y = b_y_f32 - centroid_b_y, pb_z = b_z_f32 - centroid_b_z;
1218
+ nk_f32_t ra_x = scale * (r[0] * pa_x + r[1] * pa_y + r[2] * pa_z),
1219
+ ra_y = scale * (r[3] * pa_x + r[4] * pa_y + r[5] * pa_z),
1220
+ ra_z = scale * (r[6] * pa_x + r[7] * pa_y + r[8] * pa_z);
1221
+
1222
+ nk_f32_t delta_x = ra_x - pb_x, delta_y = ra_y - pb_y, delta_z = ra_z - pb_z;
1223
+ sum_squared += delta_x * delta_x + delta_y * delta_y + delta_z * delta_z;
1224
+ }
1225
+
1226
+ return sum_squared;
1227
+ }
1228
+
1229
+ NK_PUBLIC void nk_rmsd_f16_haswell(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
1230
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
1231
+ // RMSD uses identity rotation and scale=1.0
1232
+ if (rotation)
1233
+ rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
1234
+ rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
1235
+ if (scale) *scale = 1.0f;
1236
+
1237
+ __m256 const zeros_f32x8 = _mm256_setzero_ps();
1238
+
1239
+ // Accumulators for centroids and squared differences (all in f32)
1240
+ __m256 sum_a_x_f32x8 = zeros_f32x8, sum_a_y_f32x8 = zeros_f32x8, sum_a_z_f32x8 = zeros_f32x8;
1241
+ __m256 sum_b_x_f32x8 = zeros_f32x8, sum_b_y_f32x8 = zeros_f32x8, sum_b_z_f32x8 = zeros_f32x8;
1242
+ __m256 sum_squared_x_f32x8 = zeros_f32x8, sum_squared_y_f32x8 = zeros_f32x8, sum_squared_z_f32x8 = zeros_f32x8;
1243
+
1244
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1245
+ nk_size_t i = 0;
1246
+
1247
+ // Main loop processing 8 points at a time
1248
+ for (; i + 8 <= n; i += 8) {
1249
+ nk_deinterleave_f16x8_to_f32x8_haswell_(a + i * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1250
+ nk_deinterleave_f16x8_to_f32x8_haswell_(b + i * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1251
+
1252
+ sum_a_x_f32x8 = _mm256_add_ps(sum_a_x_f32x8, a_x_f32x8);
1253
+ sum_a_y_f32x8 = _mm256_add_ps(sum_a_y_f32x8, a_y_f32x8);
1254
+ sum_a_z_f32x8 = _mm256_add_ps(sum_a_z_f32x8, a_z_f32x8);
1255
+ sum_b_x_f32x8 = _mm256_add_ps(sum_b_x_f32x8, b_x_f32x8);
1256
+ sum_b_y_f32x8 = _mm256_add_ps(sum_b_y_f32x8, b_y_f32x8);
1257
+ sum_b_z_f32x8 = _mm256_add_ps(sum_b_z_f32x8, b_z_f32x8);
1258
+
1259
+ __m256 delta_x_f32x8 = _mm256_sub_ps(a_x_f32x8, b_x_f32x8);
1260
+ __m256 delta_y_f32x8 = _mm256_sub_ps(a_y_f32x8, b_y_f32x8);
1261
+ __m256 delta_z_f32x8 = _mm256_sub_ps(a_z_f32x8, b_z_f32x8);
1262
+
1263
+ sum_squared_x_f32x8 = _mm256_fmadd_ps(delta_x_f32x8, delta_x_f32x8, sum_squared_x_f32x8);
1264
+ sum_squared_y_f32x8 = _mm256_fmadd_ps(delta_y_f32x8, delta_y_f32x8, sum_squared_y_f32x8);
1265
+ sum_squared_z_f32x8 = _mm256_fmadd_ps(delta_z_f32x8, delta_z_f32x8, sum_squared_z_f32x8);
1266
+ }
1267
+
1268
+ // Reduce vectors to scalars
1269
+ nk_f32_t total_ax = nk_reduce_add_f32x8_haswell_(sum_a_x_f32x8);
1270
+ nk_f32_t total_ay = nk_reduce_add_f32x8_haswell_(sum_a_y_f32x8);
1271
+ nk_f32_t total_az = nk_reduce_add_f32x8_haswell_(sum_a_z_f32x8);
1272
+ nk_f32_t total_bx = nk_reduce_add_f32x8_haswell_(sum_b_x_f32x8);
1273
+ nk_f32_t total_by = nk_reduce_add_f32x8_haswell_(sum_b_y_f32x8);
1274
+ nk_f32_t total_bz = nk_reduce_add_f32x8_haswell_(sum_b_z_f32x8);
1275
+ nk_f32_t total_sq_x = nk_reduce_add_f32x8_haswell_(sum_squared_x_f32x8);
1276
+ nk_f32_t total_sq_y = nk_reduce_add_f32x8_haswell_(sum_squared_y_f32x8);
1277
+ nk_f32_t total_sq_z = nk_reduce_add_f32x8_haswell_(sum_squared_z_f32x8);
1278
+
1279
+ // Scalar tail
1280
+ for (; i < n; ++i) {
1281
+ nk_f32_t ax, ay, az, bx, by, bz;
1282
+ nk_f16_to_f32_haswell(&a[i * 3 + 0], &ax);
1283
+ nk_f16_to_f32_haswell(&a[i * 3 + 1], &ay);
1284
+ nk_f16_to_f32_haswell(&a[i * 3 + 2], &az);
1285
+ nk_f16_to_f32_haswell(&b[i * 3 + 0], &bx);
1286
+ nk_f16_to_f32_haswell(&b[i * 3 + 1], &by);
1287
+ nk_f16_to_f32_haswell(&b[i * 3 + 2], &bz);
1288
+ total_ax += ax;
1289
+ total_ay += ay;
1290
+ total_az += az;
1291
+ total_bx += bx;
1292
+ total_by += by;
1293
+ total_bz += bz;
1294
+ nk_f32_t delta_x = ax - bx, delta_y = ay - by, delta_z = az - bz;
1295
+ total_sq_x += delta_x * delta_x;
1296
+ total_sq_y += delta_y * delta_y;
1297
+ total_sq_z += delta_z * delta_z;
1298
+ }
1299
+
1300
+ // Compute centroids
1301
+ nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
1302
+ nk_f32_t centroid_a_x = total_ax * inv_n;
1303
+ nk_f32_t centroid_a_y = total_ay * inv_n;
1304
+ nk_f32_t centroid_a_z = total_az * inv_n;
1305
+ nk_f32_t centroid_b_x = total_bx * inv_n;
1306
+ nk_f32_t centroid_b_y = total_by * inv_n;
1307
+ nk_f32_t centroid_b_z = total_bz * inv_n;
1308
+
1309
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
1310
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
1311
+
1312
+ // Compute RMSD
1313
+ nk_f32_t mean_diff_x = centroid_a_x - centroid_b_x;
1314
+ nk_f32_t mean_diff_y = centroid_a_y - centroid_b_y;
1315
+ nk_f32_t mean_diff_z = centroid_a_z - centroid_b_z;
1316
+ nk_f32_t sum_squared = total_sq_x + total_sq_y + total_sq_z;
1317
+ nk_f32_t mean_diff_sq = mean_diff_x * mean_diff_x + mean_diff_y * mean_diff_y + mean_diff_z * mean_diff_z;
1318
+
1319
+ *result = nk_f32_sqrt_haswell(sum_squared * inv_n - mean_diff_sq);
1320
+ }
1321
+
1322
+ NK_PUBLIC void nk_rmsd_bf16_haswell(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
1323
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
1324
+ // RMSD uses identity rotation and scale=1.0
1325
+ if (rotation)
1326
+ rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
1327
+ rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
1328
+ if (scale) *scale = 1.0f;
1329
+
1330
+ __m256 const zeros_f32x8 = _mm256_setzero_ps();
1331
+
1332
+ // Accumulators for centroids and squared differences (all in f32)
1333
+ __m256 sum_a_x_f32x8 = zeros_f32x8, sum_a_y_f32x8 = zeros_f32x8, sum_a_z_f32x8 = zeros_f32x8;
1334
+ __m256 sum_b_x_f32x8 = zeros_f32x8, sum_b_y_f32x8 = zeros_f32x8, sum_b_z_f32x8 = zeros_f32x8;
1335
+ __m256 sum_squared_x_f32x8 = zeros_f32x8, sum_squared_y_f32x8 = zeros_f32x8, sum_squared_z_f32x8 = zeros_f32x8;
1336
+
1337
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1338
+ nk_size_t i = 0;
1339
+
1340
+ // Main loop processing 8 points at a time
1341
+ for (; i + 8 <= n; i += 8) {
1342
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(a + i * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1343
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(b + i * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1344
+
1345
+ sum_a_x_f32x8 = _mm256_add_ps(sum_a_x_f32x8, a_x_f32x8);
1346
+ sum_a_y_f32x8 = _mm256_add_ps(sum_a_y_f32x8, a_y_f32x8);
1347
+ sum_a_z_f32x8 = _mm256_add_ps(sum_a_z_f32x8, a_z_f32x8);
1348
+ sum_b_x_f32x8 = _mm256_add_ps(sum_b_x_f32x8, b_x_f32x8);
1349
+ sum_b_y_f32x8 = _mm256_add_ps(sum_b_y_f32x8, b_y_f32x8);
1350
+ sum_b_z_f32x8 = _mm256_add_ps(sum_b_z_f32x8, b_z_f32x8);
1351
+
1352
+ __m256 delta_x_f32x8 = _mm256_sub_ps(a_x_f32x8, b_x_f32x8);
1353
+ __m256 delta_y_f32x8 = _mm256_sub_ps(a_y_f32x8, b_y_f32x8);
1354
+ __m256 delta_z_f32x8 = _mm256_sub_ps(a_z_f32x8, b_z_f32x8);
1355
+
1356
+ sum_squared_x_f32x8 = _mm256_fmadd_ps(delta_x_f32x8, delta_x_f32x8, sum_squared_x_f32x8);
1357
+ sum_squared_y_f32x8 = _mm256_fmadd_ps(delta_y_f32x8, delta_y_f32x8, sum_squared_y_f32x8);
1358
+ sum_squared_z_f32x8 = _mm256_fmadd_ps(delta_z_f32x8, delta_z_f32x8, sum_squared_z_f32x8);
1359
+ }
1360
+
1361
+ // Reduce vectors to scalars
1362
+ nk_f32_t total_ax = nk_reduce_add_f32x8_haswell_(sum_a_x_f32x8);
1363
+ nk_f32_t total_ay = nk_reduce_add_f32x8_haswell_(sum_a_y_f32x8);
1364
+ nk_f32_t total_az = nk_reduce_add_f32x8_haswell_(sum_a_z_f32x8);
1365
+ nk_f32_t total_bx = nk_reduce_add_f32x8_haswell_(sum_b_x_f32x8);
1366
+ nk_f32_t total_by = nk_reduce_add_f32x8_haswell_(sum_b_y_f32x8);
1367
+ nk_f32_t total_bz = nk_reduce_add_f32x8_haswell_(sum_b_z_f32x8);
1368
+ nk_f32_t total_sq_x = nk_reduce_add_f32x8_haswell_(sum_squared_x_f32x8);
1369
+ nk_f32_t total_sq_y = nk_reduce_add_f32x8_haswell_(sum_squared_y_f32x8);
1370
+ nk_f32_t total_sq_z = nk_reduce_add_f32x8_haswell_(sum_squared_z_f32x8);
1371
+
1372
+ // Scalar tail
1373
+ for (; i < n; ++i) {
1374
+ nk_f32_t ax, ay, az, bx, by, bz;
1375
+ nk_bf16_to_f32_serial(&a[i * 3 + 0], &ax);
1376
+ nk_bf16_to_f32_serial(&a[i * 3 + 1], &ay);
1377
+ nk_bf16_to_f32_serial(&a[i * 3 + 2], &az);
1378
+ nk_bf16_to_f32_serial(&b[i * 3 + 0], &bx);
1379
+ nk_bf16_to_f32_serial(&b[i * 3 + 1], &by);
1380
+ nk_bf16_to_f32_serial(&b[i * 3 + 2], &bz);
1381
+ total_ax += ax;
1382
+ total_ay += ay;
1383
+ total_az += az;
1384
+ total_bx += bx;
1385
+ total_by += by;
1386
+ total_bz += bz;
1387
+ nk_f32_t delta_x = ax - bx, delta_y = ay - by, delta_z = az - bz;
1388
+ total_sq_x += delta_x * delta_x;
1389
+ total_sq_y += delta_y * delta_y;
1390
+ total_sq_z += delta_z * delta_z;
1391
+ }
1392
+
1393
+ // Compute centroids
1394
+ nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
1395
+ nk_f32_t centroid_a_x = total_ax * inv_n;
1396
+ nk_f32_t centroid_a_y = total_ay * inv_n;
1397
+ nk_f32_t centroid_a_z = total_az * inv_n;
1398
+ nk_f32_t centroid_b_x = total_bx * inv_n;
1399
+ nk_f32_t centroid_b_y = total_by * inv_n;
1400
+ nk_f32_t centroid_b_z = total_bz * inv_n;
1401
+
1402
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
1403
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
1404
+
1405
+ // Compute RMSD
1406
+ nk_f32_t mean_diff_x = centroid_a_x - centroid_b_x;
1407
+ nk_f32_t mean_diff_y = centroid_a_y - centroid_b_y;
1408
+ nk_f32_t mean_diff_z = centroid_a_z - centroid_b_z;
1409
+ nk_f32_t sum_squared = total_sq_x + total_sq_y + total_sq_z;
1410
+ nk_f32_t mean_diff_sq = mean_diff_x * mean_diff_x + mean_diff_y * mean_diff_y + mean_diff_z * mean_diff_z;
1411
+
1412
+ *result = nk_f32_sqrt_haswell(sum_squared * inv_n - mean_diff_sq);
1413
+ }
1414
+
1415
+ NK_PUBLIC void nk_kabsch_f16_haswell(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
1416
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
1417
+ // Fused single-pass: load f16, convert to f32, compute centroids and covariance
1418
+ __m256 const zeros_f32x8 = _mm256_setzero_ps();
1419
+
1420
+ // Accumulators for centroids (f32)
1421
+ __m256 sum_a_x_f32x8 = zeros_f32x8, sum_a_y_f32x8 = zeros_f32x8, sum_a_z_f32x8 = zeros_f32x8;
1422
+ __m256 sum_b_x_f32x8 = zeros_f32x8, sum_b_y_f32x8 = zeros_f32x8, sum_b_z_f32x8 = zeros_f32x8;
1423
+
1424
+ // Accumulators for covariance matrix (sum of outer products)
1425
+ __m256 cov_xx_f32x8 = zeros_f32x8, cov_xy_f32x8 = zeros_f32x8, cov_xz_f32x8 = zeros_f32x8;
1426
+ __m256 cov_yx_f32x8 = zeros_f32x8, cov_yy_f32x8 = zeros_f32x8, cov_yz_f32x8 = zeros_f32x8;
1427
+ __m256 cov_zx_f32x8 = zeros_f32x8, cov_zy_f32x8 = zeros_f32x8, cov_zz_f32x8 = zeros_f32x8;
1428
+
1429
+ nk_size_t i = 0;
1430
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1431
+
1432
+ for (; i + 8 <= n; i += 8) {
1433
+ nk_deinterleave_f16x8_to_f32x8_haswell_(a + i * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1434
+ nk_deinterleave_f16x8_to_f32x8_haswell_(b + i * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1435
+
1436
+ // Accumulate centroids
1437
+ sum_a_x_f32x8 = _mm256_add_ps(sum_a_x_f32x8, a_x_f32x8);
1438
+ sum_a_y_f32x8 = _mm256_add_ps(sum_a_y_f32x8, a_y_f32x8);
1439
+ sum_a_z_f32x8 = _mm256_add_ps(sum_a_z_f32x8, a_z_f32x8);
1440
+ sum_b_x_f32x8 = _mm256_add_ps(sum_b_x_f32x8, b_x_f32x8);
1441
+ sum_b_y_f32x8 = _mm256_add_ps(sum_b_y_f32x8, b_y_f32x8);
1442
+ sum_b_z_f32x8 = _mm256_add_ps(sum_b_z_f32x8, b_z_f32x8);
1443
+
1444
+ // Accumulate outer products
1445
+ cov_xx_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_x_f32x8, cov_xx_f32x8);
1446
+ cov_xy_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_y_f32x8, cov_xy_f32x8);
1447
+ cov_xz_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_z_f32x8, cov_xz_f32x8);
1448
+ cov_yx_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_x_f32x8, cov_yx_f32x8);
1449
+ cov_yy_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_y_f32x8, cov_yy_f32x8);
1450
+ cov_yz_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_z_f32x8, cov_yz_f32x8);
1451
+ cov_zx_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_x_f32x8, cov_zx_f32x8);
1452
+ cov_zy_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_y_f32x8, cov_zy_f32x8);
1453
+ cov_zz_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_z_f32x8, cov_zz_f32x8);
1454
+ }
1455
+
1456
+ // Reduce vector accumulators
1457
+ nk_f32_t sum_a_x = nk_reduce_add_f32x8_haswell_(sum_a_x_f32x8);
1458
+ nk_f32_t sum_a_y = nk_reduce_add_f32x8_haswell_(sum_a_y_f32x8);
1459
+ nk_f32_t sum_a_z = nk_reduce_add_f32x8_haswell_(sum_a_z_f32x8);
1460
+ nk_f32_t sum_b_x = nk_reduce_add_f32x8_haswell_(sum_b_x_f32x8);
1461
+ nk_f32_t sum_b_y = nk_reduce_add_f32x8_haswell_(sum_b_y_f32x8);
1462
+ nk_f32_t sum_b_z = nk_reduce_add_f32x8_haswell_(sum_b_z_f32x8);
1463
+
1464
+ nk_f32_t covariance_x_x = nk_reduce_add_f32x8_haswell_(cov_xx_f32x8);
1465
+ nk_f32_t covariance_x_y = nk_reduce_add_f32x8_haswell_(cov_xy_f32x8);
1466
+ nk_f32_t covariance_x_z = nk_reduce_add_f32x8_haswell_(cov_xz_f32x8);
1467
+ nk_f32_t covariance_y_x = nk_reduce_add_f32x8_haswell_(cov_yx_f32x8);
1468
+ nk_f32_t covariance_y_y = nk_reduce_add_f32x8_haswell_(cov_yy_f32x8);
1469
+ nk_f32_t covariance_y_z = nk_reduce_add_f32x8_haswell_(cov_yz_f32x8);
1470
+ nk_f32_t covariance_z_x = nk_reduce_add_f32x8_haswell_(cov_zx_f32x8);
1471
+ nk_f32_t covariance_z_y = nk_reduce_add_f32x8_haswell_(cov_zy_f32x8);
1472
+ nk_f32_t covariance_z_z = nk_reduce_add_f32x8_haswell_(cov_zz_f32x8);
1473
+
1474
+ // Scalar tail
1475
+ for (; i < n; ++i) {
1476
+ nk_f32_t ax, ay, az, bx, by, bz;
1477
+ nk_f16_to_f32_haswell(&a[i * 3 + 0], &ax);
1478
+ nk_f16_to_f32_haswell(&a[i * 3 + 1], &ay);
1479
+ nk_f16_to_f32_haswell(&a[i * 3 + 2], &az);
1480
+ nk_f16_to_f32_haswell(&b[i * 3 + 0], &bx);
1481
+ nk_f16_to_f32_haswell(&b[i * 3 + 1], &by);
1482
+ nk_f16_to_f32_haswell(&b[i * 3 + 2], &bz);
1483
+ sum_a_x += ax, sum_a_y += ay, sum_a_z += az;
1484
+ sum_b_x += bx, sum_b_y += by, sum_b_z += bz;
1485
+ covariance_x_x += ax * bx, covariance_x_y += ax * by, covariance_x_z += ax * bz;
1486
+ covariance_y_x += ay * bx, covariance_y_y += ay * by, covariance_y_z += ay * bz;
1487
+ covariance_z_x += az * bx, covariance_z_y += az * by, covariance_z_z += az * bz;
1488
+ }
1489
+
1490
+ // Compute centroids
1491
+ nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
1492
+ nk_f32_t centroid_a_x = sum_a_x * inv_n;
1493
+ nk_f32_t centroid_a_y = sum_a_y * inv_n;
1494
+ nk_f32_t centroid_a_z = sum_a_z * inv_n;
1495
+ nk_f32_t centroid_b_x = sum_b_x * inv_n;
1496
+ nk_f32_t centroid_b_y = sum_b_y * inv_n;
1497
+ nk_f32_t centroid_b_z = sum_b_z * inv_n;
1498
+
1499
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
1500
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
1501
+
1502
+ // Apply centering correction: H_centered = H - n * centroid_a * centroid_bᵀ
1503
+ covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
1504
+ covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
1505
+ covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
1506
+ covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
1507
+ covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
1508
+ covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
1509
+ covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
1510
+ covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
1511
+ covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
1512
+
1513
+ // Compute SVD and optimal rotation
1514
+ nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
1515
+ covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
1516
+ nk_f32_t svd_u[9], svd_s[9], svd_v[9];
1517
+ nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
1518
+
1519
+ // R = V * Uᵀ
1520
+ nk_f32_t r[9];
1521
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1522
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1523
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1524
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1525
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1526
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1527
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1528
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1529
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1530
+
1531
+ // Handle reflection: if det(R) < 0, negate third column of V and recompute R
1532
+ if (nk_det3x3_f32_(r) < 0) {
1533
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
1534
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1535
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1536
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1537
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1538
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1539
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1540
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1541
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1542
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1543
+ }
1544
+
1545
+ // Output rotation matrix and scale=1.0
1546
+ if (rotation)
1547
+ for (int j = 0; j < 9; ++j) rotation[j] = r[j];
1548
+ if (scale) *scale = 1.0f;
1549
+
1550
+ // Compute RMSD after optimal rotation
1551
+ nk_f32_t sum_squared = nk_transformed_ssd_f16_haswell_(a, b, n, r, 1.0f, centroid_a_x, centroid_a_y, centroid_a_z,
1552
+ centroid_b_x, centroid_b_y, centroid_b_z);
1553
+ *result = nk_f32_sqrt_haswell(sum_squared * inv_n);
1554
+ }
1555
+
1556
+ NK_PUBLIC void nk_kabsch_bf16_haswell(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
1557
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
1558
+ // Fused single-pass: load bf16, convert to f32, compute centroids and covariance
1559
+ __m256 const zeros_f32x8 = _mm256_setzero_ps();
1560
+
1561
+ // Accumulators for centroids (f32)
1562
+ __m256 sum_a_x_f32x8 = zeros_f32x8, sum_a_y_f32x8 = zeros_f32x8, sum_a_z_f32x8 = zeros_f32x8;
1563
+ __m256 sum_b_x_f32x8 = zeros_f32x8, sum_b_y_f32x8 = zeros_f32x8, sum_b_z_f32x8 = zeros_f32x8;
1564
+
1565
+ // Accumulators for covariance matrix (sum of outer products)
1566
+ __m256 cov_xx_f32x8 = zeros_f32x8, cov_xy_f32x8 = zeros_f32x8, cov_xz_f32x8 = zeros_f32x8;
1567
+ __m256 cov_yx_f32x8 = zeros_f32x8, cov_yy_f32x8 = zeros_f32x8, cov_yz_f32x8 = zeros_f32x8;
1568
+ __m256 cov_zx_f32x8 = zeros_f32x8, cov_zy_f32x8 = zeros_f32x8, cov_zz_f32x8 = zeros_f32x8;
1569
+
1570
+ nk_size_t i = 0;
1571
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1572
+
1573
+ for (; i + 8 <= n; i += 8) {
1574
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(a + i * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1575
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(b + i * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1576
+
1577
+ // Accumulate centroids
1578
+ sum_a_x_f32x8 = _mm256_add_ps(sum_a_x_f32x8, a_x_f32x8);
1579
+ sum_a_y_f32x8 = _mm256_add_ps(sum_a_y_f32x8, a_y_f32x8);
1580
+ sum_a_z_f32x8 = _mm256_add_ps(sum_a_z_f32x8, a_z_f32x8);
1581
+ sum_b_x_f32x8 = _mm256_add_ps(sum_b_x_f32x8, b_x_f32x8);
1582
+ sum_b_y_f32x8 = _mm256_add_ps(sum_b_y_f32x8, b_y_f32x8);
1583
+ sum_b_z_f32x8 = _mm256_add_ps(sum_b_z_f32x8, b_z_f32x8);
1584
+
1585
+ // Accumulate outer products
1586
+ cov_xx_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_x_f32x8, cov_xx_f32x8);
1587
+ cov_xy_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_y_f32x8, cov_xy_f32x8);
1588
+ cov_xz_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_z_f32x8, cov_xz_f32x8);
1589
+ cov_yx_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_x_f32x8, cov_yx_f32x8);
1590
+ cov_yy_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_y_f32x8, cov_yy_f32x8);
1591
+ cov_yz_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_z_f32x8, cov_yz_f32x8);
1592
+ cov_zx_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_x_f32x8, cov_zx_f32x8);
1593
+ cov_zy_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_y_f32x8, cov_zy_f32x8);
1594
+ cov_zz_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_z_f32x8, cov_zz_f32x8);
1595
+ }
1596
+
1597
+ // Reduce vector accumulators
1598
+ nk_f32_t sum_a_x = nk_reduce_add_f32x8_haswell_(sum_a_x_f32x8);
1599
+ nk_f32_t sum_a_y = nk_reduce_add_f32x8_haswell_(sum_a_y_f32x8);
1600
+ nk_f32_t sum_a_z = nk_reduce_add_f32x8_haswell_(sum_a_z_f32x8);
1601
+ nk_f32_t sum_b_x = nk_reduce_add_f32x8_haswell_(sum_b_x_f32x8);
1602
+ nk_f32_t sum_b_y = nk_reduce_add_f32x8_haswell_(sum_b_y_f32x8);
1603
+ nk_f32_t sum_b_z = nk_reduce_add_f32x8_haswell_(sum_b_z_f32x8);
1604
+
1605
+ nk_f32_t covariance_x_x = nk_reduce_add_f32x8_haswell_(cov_xx_f32x8);
1606
+ nk_f32_t covariance_x_y = nk_reduce_add_f32x8_haswell_(cov_xy_f32x8);
1607
+ nk_f32_t covariance_x_z = nk_reduce_add_f32x8_haswell_(cov_xz_f32x8);
1608
+ nk_f32_t covariance_y_x = nk_reduce_add_f32x8_haswell_(cov_yx_f32x8);
1609
+ nk_f32_t covariance_y_y = nk_reduce_add_f32x8_haswell_(cov_yy_f32x8);
1610
+ nk_f32_t covariance_y_z = nk_reduce_add_f32x8_haswell_(cov_yz_f32x8);
1611
+ nk_f32_t covariance_z_x = nk_reduce_add_f32x8_haswell_(cov_zx_f32x8);
1612
+ nk_f32_t covariance_z_y = nk_reduce_add_f32x8_haswell_(cov_zy_f32x8);
1613
+ nk_f32_t covariance_z_z = nk_reduce_add_f32x8_haswell_(cov_zz_f32x8);
1614
+
1615
+ // Scalar tail
1616
+ for (; i < n; ++i) {
1617
+ nk_f32_t ax, ay, az, bx, by, bz;
1618
+ nk_bf16_to_f32_serial(&a[i * 3 + 0], &ax);
1619
+ nk_bf16_to_f32_serial(&a[i * 3 + 1], &ay);
1620
+ nk_bf16_to_f32_serial(&a[i * 3 + 2], &az);
1621
+ nk_bf16_to_f32_serial(&b[i * 3 + 0], &bx);
1622
+ nk_bf16_to_f32_serial(&b[i * 3 + 1], &by);
1623
+ nk_bf16_to_f32_serial(&b[i * 3 + 2], &bz);
1624
+ sum_a_x += ax, sum_a_y += ay, sum_a_z += az;
1625
+ sum_b_x += bx, sum_b_y += by, sum_b_z += bz;
1626
+ covariance_x_x += ax * bx, covariance_x_y += ax * by, covariance_x_z += ax * bz;
1627
+ covariance_y_x += ay * bx, covariance_y_y += ay * by, covariance_y_z += ay * bz;
1628
+ covariance_z_x += az * bx, covariance_z_y += az * by, covariance_z_z += az * bz;
1629
+ }
1630
+
1631
+ // Compute centroids
1632
+ nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
1633
+ nk_f32_t centroid_a_x = sum_a_x * inv_n;
1634
+ nk_f32_t centroid_a_y = sum_a_y * inv_n;
1635
+ nk_f32_t centroid_a_z = sum_a_z * inv_n;
1636
+ nk_f32_t centroid_b_x = sum_b_x * inv_n;
1637
+ nk_f32_t centroid_b_y = sum_b_y * inv_n;
1638
+ nk_f32_t centroid_b_z = sum_b_z * inv_n;
1639
+
1640
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
1641
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
1642
+
1643
+ // Apply centering correction: H_centered = H - n * centroid_a * centroid_bᵀ
1644
+ covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
1645
+ covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
1646
+ covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
1647
+ covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
1648
+ covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
1649
+ covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
1650
+ covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
1651
+ covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
1652
+ covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
1653
+
1654
+ // Compute SVD and optimal rotation
1655
+ nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
1656
+ covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
1657
+ nk_f32_t svd_u[9], svd_s[9], svd_v[9];
1658
+ nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
1659
+
1660
+ // R = V * Uᵀ
1661
+ nk_f32_t r[9];
1662
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1663
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1664
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1665
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1666
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1667
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1668
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1669
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1670
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1671
+
1672
+ // Handle reflection: if det(R) < 0, negate third column of V and recompute R
1673
+ if (nk_det3x3_f32_(r) < 0) {
1674
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
1675
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1676
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1677
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1678
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1679
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1680
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1681
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1682
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1683
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1684
+ }
1685
+
1686
+ // Output rotation matrix and scale=1.0
1687
+ if (rotation)
1688
+ for (int j = 0; j < 9; ++j) rotation[j] = r[j];
1689
+ if (scale) *scale = 1.0f;
1690
+
1691
+ // Compute RMSD after optimal rotation
1692
+ nk_f32_t sum_squared = nk_transformed_ssd_bf16_haswell_(a, b, n, r, 1.0f, centroid_a_x, centroid_a_y, centroid_a_z,
1693
+ centroid_b_x, centroid_b_y, centroid_b_z);
1694
+ *result = nk_f32_sqrt_haswell(sum_squared * inv_n);
1695
+ }
1696
+
1697
+ NK_PUBLIC void nk_umeyama_f16_haswell(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
1698
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
1699
+ // Fused single-pass: load f16, convert to f32, compute centroids, covariance, and variance
1700
+ __m256 const zeros_f32x8 = _mm256_setzero_ps();
1701
+
1702
+ __m256 sum_a_x_f32x8 = zeros_f32x8, sum_a_y_f32x8 = zeros_f32x8, sum_a_z_f32x8 = zeros_f32x8;
1703
+ __m256 sum_b_x_f32x8 = zeros_f32x8, sum_b_y_f32x8 = zeros_f32x8, sum_b_z_f32x8 = zeros_f32x8;
1704
+ __m256 cov_xx_f32x8 = zeros_f32x8, cov_xy_f32x8 = zeros_f32x8, cov_xz_f32x8 = zeros_f32x8;
1705
+ __m256 cov_yx_f32x8 = zeros_f32x8, cov_yy_f32x8 = zeros_f32x8, cov_yz_f32x8 = zeros_f32x8;
1706
+ __m256 cov_zx_f32x8 = zeros_f32x8, cov_zy_f32x8 = zeros_f32x8, cov_zz_f32x8 = zeros_f32x8;
1707
+ __m256 variance_a_f32x8 = zeros_f32x8;
1708
+
1709
+ nk_size_t i = 0;
1710
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1711
+
1712
+ for (; i + 8 <= n; i += 8) {
1713
+ nk_deinterleave_f16x8_to_f32x8_haswell_(a + i * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1714
+ nk_deinterleave_f16x8_to_f32x8_haswell_(b + i * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1715
+
1716
+ // Accumulate centroids
1717
+ sum_a_x_f32x8 = _mm256_add_ps(sum_a_x_f32x8, a_x_f32x8);
1718
+ sum_a_y_f32x8 = _mm256_add_ps(sum_a_y_f32x8, a_y_f32x8);
1719
+ sum_a_z_f32x8 = _mm256_add_ps(sum_a_z_f32x8, a_z_f32x8);
1720
+ sum_b_x_f32x8 = _mm256_add_ps(sum_b_x_f32x8, b_x_f32x8);
1721
+ sum_b_y_f32x8 = _mm256_add_ps(sum_b_y_f32x8, b_y_f32x8);
1722
+ sum_b_z_f32x8 = _mm256_add_ps(sum_b_z_f32x8, b_z_f32x8);
1723
+
1724
+ // Accumulate outer products
1725
+ cov_xx_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_x_f32x8, cov_xx_f32x8);
1726
+ cov_xy_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_y_f32x8, cov_xy_f32x8);
1727
+ cov_xz_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_z_f32x8, cov_xz_f32x8);
1728
+ cov_yx_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_x_f32x8, cov_yx_f32x8);
1729
+ cov_yy_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_y_f32x8, cov_yy_f32x8);
1730
+ cov_yz_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_z_f32x8, cov_yz_f32x8);
1731
+ cov_zx_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_x_f32x8, cov_zx_f32x8);
1732
+ cov_zy_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_y_f32x8, cov_zy_f32x8);
1733
+ cov_zz_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_z_f32x8, cov_zz_f32x8);
1734
+
1735
+ // Accumulate variance of A
1736
+ variance_a_f32x8 = _mm256_fmadd_ps(a_x_f32x8, a_x_f32x8, variance_a_f32x8);
1737
+ variance_a_f32x8 = _mm256_fmadd_ps(a_y_f32x8, a_y_f32x8, variance_a_f32x8);
1738
+ variance_a_f32x8 = _mm256_fmadd_ps(a_z_f32x8, a_z_f32x8, variance_a_f32x8);
1739
+ }
1740
+
1741
+ // Reduce vector accumulators
1742
+ nk_f32_t sum_a_x = nk_reduce_add_f32x8_haswell_(sum_a_x_f32x8);
1743
+ nk_f32_t sum_a_y = nk_reduce_add_f32x8_haswell_(sum_a_y_f32x8);
1744
+ nk_f32_t sum_a_z = nk_reduce_add_f32x8_haswell_(sum_a_z_f32x8);
1745
+ nk_f32_t sum_b_x = nk_reduce_add_f32x8_haswell_(sum_b_x_f32x8);
1746
+ nk_f32_t sum_b_y = nk_reduce_add_f32x8_haswell_(sum_b_y_f32x8);
1747
+ nk_f32_t sum_b_z = nk_reduce_add_f32x8_haswell_(sum_b_z_f32x8);
1748
+ nk_f32_t covariance_x_x = nk_reduce_add_f32x8_haswell_(cov_xx_f32x8);
1749
+ nk_f32_t covariance_x_y = nk_reduce_add_f32x8_haswell_(cov_xy_f32x8);
1750
+ nk_f32_t covariance_x_z = nk_reduce_add_f32x8_haswell_(cov_xz_f32x8);
1751
+ nk_f32_t covariance_y_x = nk_reduce_add_f32x8_haswell_(cov_yx_f32x8);
1752
+ nk_f32_t covariance_y_y = nk_reduce_add_f32x8_haswell_(cov_yy_f32x8);
1753
+ nk_f32_t covariance_y_z = nk_reduce_add_f32x8_haswell_(cov_yz_f32x8);
1754
+ nk_f32_t covariance_z_x = nk_reduce_add_f32x8_haswell_(cov_zx_f32x8);
1755
+ nk_f32_t covariance_z_y = nk_reduce_add_f32x8_haswell_(cov_zy_f32x8);
1756
+ nk_f32_t covariance_z_z = nk_reduce_add_f32x8_haswell_(cov_zz_f32x8);
1757
+ nk_f32_t variance_a_sum = nk_reduce_add_f32x8_haswell_(variance_a_f32x8);
1758
+
1759
+ // Scalar tail
1760
+ for (; i < n; ++i) {
1761
+ nk_f32_t ax, ay, az, bx, by, bz;
1762
+ nk_f16_to_f32_haswell(&a[i * 3 + 0], &ax);
1763
+ nk_f16_to_f32_haswell(&a[i * 3 + 1], &ay);
1764
+ nk_f16_to_f32_haswell(&a[i * 3 + 2], &az);
1765
+ nk_f16_to_f32_haswell(&b[i * 3 + 0], &bx);
1766
+ nk_f16_to_f32_haswell(&b[i * 3 + 1], &by);
1767
+ nk_f16_to_f32_haswell(&b[i * 3 + 2], &bz);
1768
+ sum_a_x += ax, sum_a_y += ay, sum_a_z += az;
1769
+ sum_b_x += bx, sum_b_y += by, sum_b_z += bz;
1770
+ covariance_x_x += ax * bx, covariance_x_y += ax * by, covariance_x_z += ax * bz;
1771
+ covariance_y_x += ay * bx, covariance_y_y += ay * by, covariance_y_z += ay * bz;
1772
+ covariance_z_x += az * bx, covariance_z_y += az * by, covariance_z_z += az * bz;
1773
+ variance_a_sum += ax * ax + ay * ay + az * az;
1774
+ }
1775
+
1776
+ // Compute centroids
1777
+ nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
1778
+ nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
1779
+ nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
1780
+
1781
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
1782
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
1783
+
1784
+ // Compute centered covariance and variance
1785
+ nk_f32_t variance_a = variance_a_sum * inv_n -
1786
+ (centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
1787
+
1788
+ // Apply centering correction to covariance matrix
1789
+ covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
1790
+ covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
1791
+ covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
1792
+ covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
1793
+ covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
1794
+ covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
1795
+ covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
1796
+ covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
1797
+ covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
1798
+
1799
+ nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
1800
+ covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
1801
+
1802
+ // SVD
1803
+ nk_f32_t svd_u[9], svd_s[9], svd_v[9];
1804
+ nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
1805
+
1806
+ // R = V * Uᵀ
1807
+ nk_f32_t r[9];
1808
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1809
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1810
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1811
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1812
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1813
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1814
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1815
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1816
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1817
+
1818
+ // Scale factor: c = trace(D × S) / (n × variance(a))
1819
+ nk_f32_t det = nk_det3x3_f32_(r);
1820
+ nk_f32_t d3 = det < 0 ? -1.0f : 1.0f;
1821
+ nk_f32_t trace_ds = svd_s[0] + svd_s[4] + d3 * svd_s[8];
1822
+ nk_f32_t c = trace_ds / ((nk_f32_t)n * variance_a);
1823
+ if (scale) *scale = c;
1824
+
1825
+ // Handle reflection
1826
+ if (det < 0) {
1827
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
1828
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1829
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1830
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1831
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1832
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1833
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1834
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1835
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1836
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1837
+ }
1838
+
1839
+ // Output rotation matrix
1840
+ if (rotation)
1841
+ for (int j = 0; j < 9; ++j) rotation[j] = r[j];
1842
+
1843
+ // Compute RMSD with scaling
1844
+ nk_f32_t sum_squared = nk_transformed_ssd_f16_haswell_(a, b, n, r, c, centroid_a_x, centroid_a_y, centroid_a_z,
1845
+ centroid_b_x, centroid_b_y, centroid_b_z);
1846
+ *result = nk_f32_sqrt_haswell(sum_squared * inv_n);
1847
+ }
1848
+
1849
+ NK_PUBLIC void nk_umeyama_bf16_haswell(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
1850
+ nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
1851
+ // Fused single-pass: load bf16, convert to f32, compute centroids, covariance, and variance
1852
+ __m256 const zeros_f32x8 = _mm256_setzero_ps();
1853
+
1854
+ __m256 sum_a_x_f32x8 = zeros_f32x8, sum_a_y_f32x8 = zeros_f32x8, sum_a_z_f32x8 = zeros_f32x8;
1855
+ __m256 sum_b_x_f32x8 = zeros_f32x8, sum_b_y_f32x8 = zeros_f32x8, sum_b_z_f32x8 = zeros_f32x8;
1856
+ __m256 cov_xx_f32x8 = zeros_f32x8, cov_xy_f32x8 = zeros_f32x8, cov_xz_f32x8 = zeros_f32x8;
1857
+ __m256 cov_yx_f32x8 = zeros_f32x8, cov_yy_f32x8 = zeros_f32x8, cov_yz_f32x8 = zeros_f32x8;
1858
+ __m256 cov_zx_f32x8 = zeros_f32x8, cov_zy_f32x8 = zeros_f32x8, cov_zz_f32x8 = zeros_f32x8;
1859
+ __m256 variance_a_f32x8 = zeros_f32x8;
1860
+
1861
+ nk_size_t i = 0;
1862
+ __m256 a_x_f32x8, a_y_f32x8, a_z_f32x8, b_x_f32x8, b_y_f32x8, b_z_f32x8;
1863
+
1864
+ for (; i + 8 <= n; i += 8) {
1865
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(a + i * 3, &a_x_f32x8, &a_y_f32x8, &a_z_f32x8);
1866
+ nk_deinterleave_bf16x8_to_f32x8_haswell_(b + i * 3, &b_x_f32x8, &b_y_f32x8, &b_z_f32x8);
1867
+
1868
+ // Accumulate centroids
1869
+ sum_a_x_f32x8 = _mm256_add_ps(sum_a_x_f32x8, a_x_f32x8);
1870
+ sum_a_y_f32x8 = _mm256_add_ps(sum_a_y_f32x8, a_y_f32x8);
1871
+ sum_a_z_f32x8 = _mm256_add_ps(sum_a_z_f32x8, a_z_f32x8);
1872
+ sum_b_x_f32x8 = _mm256_add_ps(sum_b_x_f32x8, b_x_f32x8);
1873
+ sum_b_y_f32x8 = _mm256_add_ps(sum_b_y_f32x8, b_y_f32x8);
1874
+ sum_b_z_f32x8 = _mm256_add_ps(sum_b_z_f32x8, b_z_f32x8);
1875
+
1876
+ // Accumulate outer products
1877
+ cov_xx_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_x_f32x8, cov_xx_f32x8);
1878
+ cov_xy_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_y_f32x8, cov_xy_f32x8);
1879
+ cov_xz_f32x8 = _mm256_fmadd_ps(a_x_f32x8, b_z_f32x8, cov_xz_f32x8);
1880
+ cov_yx_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_x_f32x8, cov_yx_f32x8);
1881
+ cov_yy_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_y_f32x8, cov_yy_f32x8);
1882
+ cov_yz_f32x8 = _mm256_fmadd_ps(a_y_f32x8, b_z_f32x8, cov_yz_f32x8);
1883
+ cov_zx_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_x_f32x8, cov_zx_f32x8);
1884
+ cov_zy_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_y_f32x8, cov_zy_f32x8);
1885
+ cov_zz_f32x8 = _mm256_fmadd_ps(a_z_f32x8, b_z_f32x8, cov_zz_f32x8);
1886
+
1887
+ // Accumulate variance of A
1888
+ variance_a_f32x8 = _mm256_fmadd_ps(a_x_f32x8, a_x_f32x8, variance_a_f32x8);
1889
+ variance_a_f32x8 = _mm256_fmadd_ps(a_y_f32x8, a_y_f32x8, variance_a_f32x8);
1890
+ variance_a_f32x8 = _mm256_fmadd_ps(a_z_f32x8, a_z_f32x8, variance_a_f32x8);
1891
+ }
1892
+
1893
+ // Reduce vector accumulators
1894
+ nk_f32_t sum_a_x = nk_reduce_add_f32x8_haswell_(sum_a_x_f32x8);
1895
+ nk_f32_t sum_a_y = nk_reduce_add_f32x8_haswell_(sum_a_y_f32x8);
1896
+ nk_f32_t sum_a_z = nk_reduce_add_f32x8_haswell_(sum_a_z_f32x8);
1897
+ nk_f32_t sum_b_x = nk_reduce_add_f32x8_haswell_(sum_b_x_f32x8);
1898
+ nk_f32_t sum_b_y = nk_reduce_add_f32x8_haswell_(sum_b_y_f32x8);
1899
+ nk_f32_t sum_b_z = nk_reduce_add_f32x8_haswell_(sum_b_z_f32x8);
1900
+ nk_f32_t covariance_x_x = nk_reduce_add_f32x8_haswell_(cov_xx_f32x8);
1901
+ nk_f32_t covariance_x_y = nk_reduce_add_f32x8_haswell_(cov_xy_f32x8);
1902
+ nk_f32_t covariance_x_z = nk_reduce_add_f32x8_haswell_(cov_xz_f32x8);
1903
+ nk_f32_t covariance_y_x = nk_reduce_add_f32x8_haswell_(cov_yx_f32x8);
1904
+ nk_f32_t covariance_y_y = nk_reduce_add_f32x8_haswell_(cov_yy_f32x8);
1905
+ nk_f32_t covariance_y_z = nk_reduce_add_f32x8_haswell_(cov_yz_f32x8);
1906
+ nk_f32_t covariance_z_x = nk_reduce_add_f32x8_haswell_(cov_zx_f32x8);
1907
+ nk_f32_t covariance_z_y = nk_reduce_add_f32x8_haswell_(cov_zy_f32x8);
1908
+ nk_f32_t covariance_z_z = nk_reduce_add_f32x8_haswell_(cov_zz_f32x8);
1909
+ nk_f32_t variance_a_sum = nk_reduce_add_f32x8_haswell_(variance_a_f32x8);
1910
+
1911
+ // Scalar tail
1912
+ for (; i < n; ++i) {
1913
+ nk_f32_t ax, ay, az, bx, by, bz;
1914
+ nk_bf16_to_f32_serial(&a[i * 3 + 0], &ax);
1915
+ nk_bf16_to_f32_serial(&a[i * 3 + 1], &ay);
1916
+ nk_bf16_to_f32_serial(&a[i * 3 + 2], &az);
1917
+ nk_bf16_to_f32_serial(&b[i * 3 + 0], &bx);
1918
+ nk_bf16_to_f32_serial(&b[i * 3 + 1], &by);
1919
+ nk_bf16_to_f32_serial(&b[i * 3 + 2], &bz);
1920
+ sum_a_x += ax, sum_a_y += ay, sum_a_z += az;
1921
+ sum_b_x += bx, sum_b_y += by, sum_b_z += bz;
1922
+ covariance_x_x += ax * bx, covariance_x_y += ax * by, covariance_x_z += ax * bz;
1923
+ covariance_y_x += ay * bx, covariance_y_y += ay * by, covariance_y_z += ay * bz;
1924
+ covariance_z_x += az * bx, covariance_z_y += az * by, covariance_z_z += az * bz;
1925
+ variance_a_sum += ax * ax + ay * ay + az * az;
1926
+ }
1927
+
1928
+ // Compute centroids
1929
+ nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
1930
+ nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
1931
+ nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
1932
+
1933
+ if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
1934
+ if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
1935
+
1936
+ // Compute centered covariance and variance
1937
+ nk_f32_t variance_a = variance_a_sum * inv_n -
1938
+ (centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
1939
+
1940
+ // Apply centering correction to covariance matrix
1941
+ covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
1942
+ covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
1943
+ covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
1944
+ covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
1945
+ covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
1946
+ covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
1947
+ covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
1948
+ covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
1949
+ covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
1950
+
1951
+ nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
1952
+ covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
1953
+
1954
+ // SVD
1955
+ nk_f32_t svd_u[9], svd_s[9], svd_v[9];
1956
+ nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
1957
+
1958
+ // R = V * Uᵀ
1959
+ nk_f32_t r[9];
1960
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1961
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1962
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1963
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1964
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1965
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1966
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1967
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1968
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1969
+
1970
+ // Scale factor: c = trace(D × S) / (n × variance(a))
1971
+ nk_f32_t det = nk_det3x3_f32_(r);
1972
+ nk_f32_t d3 = det < 0 ? -1.0f : 1.0f;
1973
+ nk_f32_t trace_ds = svd_s[0] + svd_s[4] + d3 * svd_s[8];
1974
+ nk_f32_t c = trace_ds / ((nk_f32_t)n * variance_a);
1975
+ if (scale) *scale = c;
1976
+
1977
+ // Handle reflection
1978
+ if (det < 0) {
1979
+ svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
1980
+ r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
1981
+ r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
1982
+ r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
1983
+ r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
1984
+ r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
1985
+ r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
1986
+ r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
1987
+ r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
1988
+ r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
1989
+ }
1990
+
1991
+ // Output rotation matrix
1992
+ if (rotation)
1993
+ for (int j = 0; j < 9; ++j) rotation[j] = r[j];
1994
+
1995
+ // Compute RMSD with scaling
1996
+ nk_f32_t sum_squared = nk_transformed_ssd_bf16_haswell_(a, b, n, r, c, centroid_a_x, centroid_a_y, centroid_a_z,
1997
+ centroid_b_x, centroid_b_y, centroid_b_z);
1998
+ *result = nk_f32_sqrt_haswell(sum_squared * inv_n);
1999
+ }
2000
+
2001
+ #if defined(__clang__)
2002
+ #pragma clang attribute pop
2003
+ #elif defined(__GNUC__)
2004
+ #pragma GCC pop_options
2005
+ #endif
2006
+
2007
+ #if defined(__cplusplus)
2008
+ } // extern "C"
2009
+ #endif
2010
+
2011
+ #endif // NK_TARGET_HASWELL
2012
+ #endif // NK_TARGET_X8664_
2013
+ #endif // NK_MESH_HASWELL_H