usearch 2.21.3 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -24
- package/binding.gyp +15 -37
- package/c/CMakeLists.txt +4 -5
- package/c/README.md +7 -7
- package/c/lib.cpp +32 -14
- package/c/test.c +83 -1
- package/c/usearch.h +24 -3
- package/cpp/CMakeLists.txt +4 -0
- package/cpp/README.md +5 -5
- package/cpp/bench.cpp +97 -116
- package/cpp/test.cpp +117 -20
- package/include/usearch/index.hpp +235 -52
- package/include/usearch/index_dense.hpp +105 -40
- package/include/usearch/index_plugins.hpp +1387 -204
- package/javascript/dist/cjs/usearch.d.ts +8 -3
- package/javascript/dist/cjs/usearch.js +12 -4
- package/javascript/dist/esm/usearch.d.ts +8 -3
- package/javascript/dist/esm/usearch.js +12 -4
- package/javascript/lib.cpp +17 -0
- package/javascript/usearch.ts +14 -6
- package/numkong/.clang-tidy +30 -0
- package/{simsimd → numkong}/.cmake-format.py +18 -0
- package/numkong/.editorconfig +12 -0
- package/numkong/.git-blame-ignore-revs +123 -0
- package/numkong/.mailmap +15 -0
- package/numkong/.spi.yml +11 -0
- package/numkong/.swift-format +14 -0
- package/numkong/CITATION.cff +81 -0
- package/numkong/CONTRIBUTING.md +500 -0
- package/numkong/Cargo.toml +54 -0
- package/numkong/MANIFEST.in +8 -0
- package/numkong/Package.swift +84 -0
- package/numkong/README.md +569 -0
- package/numkong/bench/README.md +217 -0
- package/numkong/bench/bench-browser.html +323 -0
- package/numkong/bench/bench.cpp +344 -0
- package/numkong/bench/bench.hpp +816 -0
- package/numkong/bench/bench.js +148 -0
- package/numkong/bench/bench.mjs +616 -0
- package/numkong/bench/bench_attention.cpp +113 -0
- package/numkong/bench/bench_cast.cpp +264 -0
- package/numkong/bench/bench_cross_amx.cpp +62 -0
- package/numkong/bench/bench_cross_arm.cpp +245 -0
- package/numkong/bench/bench_cross_blas.cpp +309 -0
- package/numkong/bench/bench_cross_loongarch.cpp +106 -0
- package/numkong/bench/bench_cross_power.cpp +84 -0
- package/numkong/bench/bench_cross_rvv.cpp +121 -0
- package/numkong/bench/bench_cross_serial.cpp +151 -0
- package/numkong/bench/bench_cross_sme.cpp +163 -0
- package/numkong/bench/bench_cross_wasm.cpp +142 -0
- package/numkong/bench/bench_cross_x86.cpp +455 -0
- package/numkong/bench/bench_curved.cpp +210 -0
- package/numkong/bench/bench_dot.cpp +315 -0
- package/numkong/bench/bench_each.cpp +503 -0
- package/numkong/bench/bench_geospatial.cpp +112 -0
- package/numkong/bench/bench_maxsim.cpp +154 -0
- package/numkong/bench/bench_mesh.cpp +148 -0
- package/numkong/bench/bench_probability.cpp +60 -0
- package/numkong/bench/bench_reduce.cpp +371 -0
- package/numkong/bench/bench_set.cpp +73 -0
- package/numkong/bench/bench_sparse.cpp +198 -0
- package/numkong/bench/bench_spatial.cpp +347 -0
- package/numkong/bench/bench_trigonometry.cpp +137 -0
- package/numkong/bench/swift/Bench.swift +506 -0
- package/numkong/binding.gyp +169 -0
- package/numkong/build.rs +484 -0
- package/numkong/c/dispatch.h +512 -0
- package/numkong/c/dispatch_bf16.c +447 -0
- package/numkong/c/dispatch_bf16c.c +52 -0
- package/numkong/c/dispatch_e2m3.c +296 -0
- package/numkong/c/dispatch_e3m2.c +284 -0
- package/numkong/c/dispatch_e4m3.c +318 -0
- package/numkong/c/dispatch_e5m2.c +316 -0
- package/numkong/c/dispatch_f16.c +452 -0
- package/numkong/c/dispatch_f16c.c +58 -0
- package/numkong/c/dispatch_f32.c +434 -0
- package/numkong/c/dispatch_f32c.c +99 -0
- package/numkong/c/dispatch_f64.c +348 -0
- package/numkong/c/dispatch_f64c.c +98 -0
- package/numkong/c/dispatch_i16.c +96 -0
- package/numkong/c/dispatch_i32.c +89 -0
- package/numkong/c/dispatch_i4.c +153 -0
- package/numkong/c/dispatch_i64.c +86 -0
- package/numkong/c/dispatch_i8.c +348 -0
- package/numkong/c/dispatch_other.c +348 -0
- package/numkong/c/dispatch_u1.c +193 -0
- package/numkong/c/dispatch_u16.c +124 -0
- package/numkong/c/dispatch_u32.c +118 -0
- package/numkong/c/dispatch_u4.c +153 -0
- package/numkong/c/dispatch_u64.c +102 -0
- package/numkong/c/dispatch_u8.c +364 -0
- package/numkong/c/numkong.c +940 -0
- package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
- package/numkong/cmake/nk_isa_probe.cmake +90 -0
- package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_math_lib.cmake +46 -0
- package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
- package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
- package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
- package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
- package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
- package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
- package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
- package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
- package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
- package/numkong/cmake/toolchain-wasi.cmake +77 -0
- package/numkong/cmake/toolchain-wasm.cmake +70 -0
- package/numkong/cmake/toolchain-wasm64.cmake +62 -0
- package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
- package/numkong/include/README.md +643 -0
- package/numkong/include/module.modulemap +129 -0
- package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
- package/numkong/include/numkong/attention/sme.h +2075 -0
- package/numkong/include/numkong/attention.h +49 -0
- package/numkong/include/numkong/capabilities.h +930 -0
- package/numkong/include/numkong/cast/README.md +262 -0
- package/numkong/include/numkong/cast/diamond.h +64 -0
- package/numkong/include/numkong/cast/haswell.h +823 -0
- package/numkong/include/numkong/cast/icelake.h +475 -0
- package/numkong/include/numkong/cast/loongsonasx.h +252 -0
- package/numkong/include/numkong/cast/neon.h +1174 -0
- package/numkong/include/numkong/cast/powervsx.h +449 -0
- package/numkong/include/numkong/cast/rvv.h +970 -0
- package/numkong/include/numkong/cast/sapphire.h +262 -0
- package/numkong/include/numkong/cast/serial.h +2336 -0
- package/numkong/include/numkong/cast/skylake.h +915 -0
- package/numkong/include/numkong/cast/v128relaxed.h +537 -0
- package/numkong/include/numkong/cast.h +254 -0
- package/numkong/include/numkong/cast.hpp +44 -0
- package/numkong/include/numkong/curved/README.md +223 -0
- package/numkong/include/numkong/curved/genoa.h +182 -0
- package/numkong/include/numkong/curved/haswell.h +276 -0
- package/numkong/include/numkong/curved/neon.h +329 -0
- package/numkong/include/numkong/curved/neonbfdot.h +211 -0
- package/numkong/include/numkong/curved/rvv.h +305 -0
- package/numkong/include/numkong/curved/serial.h +207 -0
- package/numkong/include/numkong/curved/skylake.h +457 -0
- package/numkong/include/numkong/curved/smef64.h +530 -0
- package/numkong/include/numkong/curved.h +513 -0
- package/numkong/include/numkong/curved.hpp +144 -0
- package/numkong/include/numkong/dot/README.md +442 -0
- package/numkong/include/numkong/dot/alder.h +563 -0
- package/numkong/include/numkong/dot/diamond.h +158 -0
- package/numkong/include/numkong/dot/genoa.h +289 -0
- package/numkong/include/numkong/dot/haswell.h +1723 -0
- package/numkong/include/numkong/dot/icelake.h +997 -0
- package/numkong/include/numkong/dot/loongsonasx.h +671 -0
- package/numkong/include/numkong/dot/neon.h +869 -0
- package/numkong/include/numkong/dot/neonbfdot.h +243 -0
- package/numkong/include/numkong/dot/neonfhm.h +358 -0
- package/numkong/include/numkong/dot/neonfp8.h +323 -0
- package/numkong/include/numkong/dot/neonsdot.h +622 -0
- package/numkong/include/numkong/dot/powervsx.h +752 -0
- package/numkong/include/numkong/dot/rvv.h +722 -0
- package/numkong/include/numkong/dot/rvvbb.h +72 -0
- package/numkong/include/numkong/dot/rvvbf16.h +123 -0
- package/numkong/include/numkong/dot/rvvhalf.h +129 -0
- package/numkong/include/numkong/dot/sapphire.h +141 -0
- package/numkong/include/numkong/dot/serial.h +874 -0
- package/numkong/include/numkong/dot/sierra.h +405 -0
- package/numkong/include/numkong/dot/skylake.h +1116 -0
- package/numkong/include/numkong/dot/sve.h +420 -0
- package/numkong/include/numkong/dot/svebfdot.h +75 -0
- package/numkong/include/numkong/dot/svehalf.h +168 -0
- package/numkong/include/numkong/dot/svesdot.h +90 -0
- package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
- package/numkong/include/numkong/dot.h +1136 -0
- package/numkong/include/numkong/dot.hpp +94 -0
- package/numkong/include/numkong/dots/README.md +496 -0
- package/numkong/include/numkong/dots/alder.h +117 -0
- package/numkong/include/numkong/dots/diamond.h +86 -0
- package/numkong/include/numkong/dots/genoa.h +100 -0
- package/numkong/include/numkong/dots/graniteamx.h +733 -0
- package/numkong/include/numkong/dots/haswell.h +310 -0
- package/numkong/include/numkong/dots/icelake.h +180 -0
- package/numkong/include/numkong/dots/loongsonasx.h +176 -0
- package/numkong/include/numkong/dots/neon.h +123 -0
- package/numkong/include/numkong/dots/neonbfdot.h +59 -0
- package/numkong/include/numkong/dots/neonfhm.h +96 -0
- package/numkong/include/numkong/dots/neonfp8.h +99 -0
- package/numkong/include/numkong/dots/neonsdot.h +144 -0
- package/numkong/include/numkong/dots/powervsx.h +194 -0
- package/numkong/include/numkong/dots/rvv.h +2593 -0
- package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
- package/numkong/include/numkong/dots/serial.h +2867 -0
- package/numkong/include/numkong/dots/sierra.h +100 -0
- package/numkong/include/numkong/dots/skylake.h +207 -0
- package/numkong/include/numkong/dots/sme.h +5041 -0
- package/numkong/include/numkong/dots/smebi32.h +477 -0
- package/numkong/include/numkong/dots/smef64.h +1328 -0
- package/numkong/include/numkong/dots/smehalf.h +47 -0
- package/numkong/include/numkong/dots/v128relaxed.h +317 -0
- package/numkong/include/numkong/dots.h +3110 -0
- package/numkong/include/numkong/dots.hpp +641 -0
- package/numkong/include/numkong/each/README.md +465 -0
- package/numkong/include/numkong/each/haswell.h +1658 -0
- package/numkong/include/numkong/each/icelake.h +272 -0
- package/numkong/include/numkong/each/neon.h +1138 -0
- package/numkong/include/numkong/each/neonbfdot.h +211 -0
- package/numkong/include/numkong/each/neonhalf.h +318 -0
- package/numkong/include/numkong/each/rvv.h +1121 -0
- package/numkong/include/numkong/each/sapphire.h +343 -0
- package/numkong/include/numkong/each/serial.h +282 -0
- package/numkong/include/numkong/each/skylake.h +1562 -0
- package/numkong/include/numkong/each/v128relaxed.h +562 -0
- package/numkong/include/numkong/each.h +2232 -0
- package/numkong/include/numkong/each.hpp +434 -0
- package/numkong/include/numkong/geospatial/README.md +147 -0
- package/numkong/include/numkong/geospatial/haswell.h +633 -0
- package/numkong/include/numkong/geospatial/neon.h +615 -0
- package/numkong/include/numkong/geospatial/rvv.h +701 -0
- package/numkong/include/numkong/geospatial/serial.h +309 -0
- package/numkong/include/numkong/geospatial/skylake.h +613 -0
- package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
- package/numkong/include/numkong/geospatial.h +453 -0
- package/numkong/include/numkong/geospatial.hpp +235 -0
- package/numkong/include/numkong/matrix.hpp +336 -0
- package/numkong/include/numkong/maxsim/README.md +187 -0
- package/numkong/include/numkong/maxsim/alder.h +511 -0
- package/numkong/include/numkong/maxsim/genoa.h +115 -0
- package/numkong/include/numkong/maxsim/haswell.h +553 -0
- package/numkong/include/numkong/maxsim/icelake.h +480 -0
- package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
- package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
- package/numkong/include/numkong/maxsim/serial.h +490 -0
- package/numkong/include/numkong/maxsim/sme.h +993 -0
- package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
- package/numkong/include/numkong/maxsim.h +571 -0
- package/numkong/include/numkong/maxsim.hpp +133 -0
- package/numkong/include/numkong/mesh/README.md +219 -0
- package/numkong/include/numkong/mesh/haswell.h +2013 -0
- package/numkong/include/numkong/mesh/neon.h +1821 -0
- package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
- package/numkong/include/numkong/mesh/rvv.h +936 -0
- package/numkong/include/numkong/mesh/serial.h +721 -0
- package/numkong/include/numkong/mesh/skylake.h +2344 -0
- package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
- package/numkong/include/numkong/mesh.h +666 -0
- package/numkong/include/numkong/mesh.hpp +701 -0
- package/numkong/include/numkong/numkong.h +78 -0
- package/numkong/include/numkong/numkong.hpp +58 -0
- package/numkong/include/numkong/probability/README.md +177 -0
- package/numkong/include/numkong/probability/haswell.h +267 -0
- package/numkong/include/numkong/probability/neon.h +255 -0
- package/numkong/include/numkong/probability/rvv.h +414 -0
- package/numkong/include/numkong/probability/serial.h +181 -0
- package/numkong/include/numkong/probability/skylake.h +324 -0
- package/numkong/include/numkong/probability.h +380 -0
- package/numkong/include/numkong/probability.hpp +120 -0
- package/numkong/include/numkong/random.h +50 -0
- package/numkong/include/numkong/random.hpp +285 -0
- package/numkong/include/numkong/reduce/README.md +552 -0
- package/numkong/include/numkong/reduce/alder.h +636 -0
- package/numkong/include/numkong/reduce/genoa.h +201 -0
- package/numkong/include/numkong/reduce/haswell.h +3785 -0
- package/numkong/include/numkong/reduce/icelake.h +549 -0
- package/numkong/include/numkong/reduce/neon.h +3980 -0
- package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
- package/numkong/include/numkong/reduce/neonfhm.h +293 -0
- package/numkong/include/numkong/reduce/neonsdot.h +357 -0
- package/numkong/include/numkong/reduce/rvv.h +3533 -0
- package/numkong/include/numkong/reduce/serial.h +753 -0
- package/numkong/include/numkong/reduce/sierra.h +338 -0
- package/numkong/include/numkong/reduce/skylake.h +3794 -0
- package/numkong/include/numkong/reduce/sve.h +52 -0
- package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
- package/numkong/include/numkong/reduce.h +1582 -0
- package/numkong/include/numkong/reduce.hpp +778 -0
- package/numkong/include/numkong/scalar/README.md +89 -0
- package/numkong/include/numkong/scalar/haswell.h +122 -0
- package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
- package/numkong/include/numkong/scalar/neon.h +122 -0
- package/numkong/include/numkong/scalar/neonhalf.h +70 -0
- package/numkong/include/numkong/scalar/powervsx.h +96 -0
- package/numkong/include/numkong/scalar/rvv.h +211 -0
- package/numkong/include/numkong/scalar/sapphire.h +74 -0
- package/numkong/include/numkong/scalar/serial.h +332 -0
- package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
- package/numkong/include/numkong/scalar.h +696 -0
- package/numkong/include/numkong/set/README.md +179 -0
- package/numkong/include/numkong/set/haswell.h +334 -0
- package/numkong/include/numkong/set/icelake.h +485 -0
- package/numkong/include/numkong/set/loongsonasx.h +181 -0
- package/numkong/include/numkong/set/neon.h +363 -0
- package/numkong/include/numkong/set/powervsx.h +326 -0
- package/numkong/include/numkong/set/rvv.h +226 -0
- package/numkong/include/numkong/set/rvvbb.h +117 -0
- package/numkong/include/numkong/set/serial.h +174 -0
- package/numkong/include/numkong/set/sve.h +187 -0
- package/numkong/include/numkong/set/v128relaxed.h +240 -0
- package/numkong/include/numkong/set.h +471 -0
- package/numkong/include/numkong/set.hpp +114 -0
- package/numkong/include/numkong/sets/README.md +149 -0
- package/numkong/include/numkong/sets/haswell.h +63 -0
- package/numkong/include/numkong/sets/icelake.h +66 -0
- package/numkong/include/numkong/sets/loongsonasx.h +52 -0
- package/numkong/include/numkong/sets/neon.h +61 -0
- package/numkong/include/numkong/sets/powervsx.h +65 -0
- package/numkong/include/numkong/sets/serial.h +43 -0
- package/numkong/include/numkong/sets/smebi32.h +1135 -0
- package/numkong/include/numkong/sets/v128relaxed.h +58 -0
- package/numkong/include/numkong/sets.h +382 -0
- package/numkong/include/numkong/sparse/README.md +156 -0
- package/numkong/include/numkong/sparse/icelake.h +463 -0
- package/numkong/include/numkong/sparse/neon.h +288 -0
- package/numkong/include/numkong/sparse/serial.h +117 -0
- package/numkong/include/numkong/sparse/sve2.h +504 -0
- package/numkong/include/numkong/sparse/turin.h +322 -0
- package/numkong/include/numkong/sparse.h +363 -0
- package/numkong/include/numkong/sparse.hpp +113 -0
- package/numkong/include/numkong/spatial/README.md +439 -0
- package/numkong/include/numkong/spatial/alder.h +604 -0
- package/numkong/include/numkong/spatial/diamond.h +240 -0
- package/numkong/include/numkong/spatial/genoa.h +222 -0
- package/numkong/include/numkong/spatial/haswell.h +979 -0
- package/numkong/include/numkong/spatial/icelake.h +1067 -0
- package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
- package/numkong/include/numkong/spatial/neon.h +846 -0
- package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
- package/numkong/include/numkong/spatial/neonfp8.h +258 -0
- package/numkong/include/numkong/spatial/neonsdot.h +429 -0
- package/numkong/include/numkong/spatial/powervsx.h +738 -0
- package/numkong/include/numkong/spatial/rvv.h +991 -0
- package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
- package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
- package/numkong/include/numkong/spatial/serial.h +347 -0
- package/numkong/include/numkong/spatial/sierra.h +516 -0
- package/numkong/include/numkong/spatial/skylake.h +605 -0
- package/numkong/include/numkong/spatial/sve.h +246 -0
- package/numkong/include/numkong/spatial/svebfdot.h +125 -0
- package/numkong/include/numkong/spatial/svehalf.h +136 -0
- package/numkong/include/numkong/spatial/svesdot.h +143 -0
- package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
- package/numkong/include/numkong/spatial.h +1681 -0
- package/numkong/include/numkong/spatial.hpp +183 -0
- package/numkong/include/numkong/spatials/README.md +580 -0
- package/numkong/include/numkong/spatials/alder.h +94 -0
- package/numkong/include/numkong/spatials/diamond.h +82 -0
- package/numkong/include/numkong/spatials/genoa.h +94 -0
- package/numkong/include/numkong/spatials/graniteamx.h +173 -0
- package/numkong/include/numkong/spatials/haswell.h +219 -0
- package/numkong/include/numkong/spatials/icelake.h +113 -0
- package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
- package/numkong/include/numkong/spatials/neon.h +109 -0
- package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
- package/numkong/include/numkong/spatials/neonfhm.h +92 -0
- package/numkong/include/numkong/spatials/neonfp8.h +111 -0
- package/numkong/include/numkong/spatials/neonsdot.h +143 -0
- package/numkong/include/numkong/spatials/powervsx.h +153 -0
- package/numkong/include/numkong/spatials/rvv.h +1976 -0
- package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
- package/numkong/include/numkong/spatials/serial.h +248 -0
- package/numkong/include/numkong/spatials/sierra.h +96 -0
- package/numkong/include/numkong/spatials/skylake.h +184 -0
- package/numkong/include/numkong/spatials/sme.h +1927 -0
- package/numkong/include/numkong/spatials/smef64.h +483 -0
- package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
- package/numkong/include/numkong/spatials.h +3374 -0
- package/numkong/include/numkong/spatials.hpp +509 -0
- package/numkong/include/numkong/tensor.hpp +1981 -0
- package/numkong/include/numkong/trigonometry/README.md +184 -0
- package/numkong/include/numkong/trigonometry/haswell.h +653 -0
- package/numkong/include/numkong/trigonometry/neon.h +638 -0
- package/numkong/include/numkong/trigonometry/rvv.h +700 -0
- package/numkong/include/numkong/trigonometry/serial.h +703 -0
- package/numkong/include/numkong/trigonometry/skylake.h +725 -0
- package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
- package/numkong/include/numkong/trigonometry.h +467 -0
- package/numkong/include/numkong/trigonometry.hpp +166 -0
- package/numkong/include/numkong/types.h +1695 -0
- package/numkong/include/numkong/types.hpp +6027 -0
- package/numkong/include/numkong/vector.hpp +766 -0
- package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
- package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
- package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
- package/numkong/javascript/README.md +254 -0
- package/numkong/javascript/node-gyp-build.d.ts +4 -0
- package/numkong/javascript/numkong-browser.ts +40 -0
- package/numkong/javascript/numkong-wasm.ts +750 -0
- package/numkong/javascript/numkong.c +712 -0
- package/numkong/javascript/numkong.ts +613 -0
- package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
- package/numkong/javascript/tsconfig-cjs.json +13 -0
- package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
- package/numkong/javascript/types.ts +784 -0
- package/numkong/numkong.gypi +104 -0
- package/numkong/package.json +108 -0
- package/numkong/probes/arm_neon.c +8 -0
- package/numkong/probes/arm_neon_bfdot.c +9 -0
- package/numkong/probes/arm_neon_fhm.c +9 -0
- package/numkong/probes/arm_neon_half.c +8 -0
- package/numkong/probes/arm_neon_sdot.c +9 -0
- package/numkong/probes/arm_neonfp8.c +9 -0
- package/numkong/probes/arm_sme.c +16 -0
- package/numkong/probes/arm_sme2.c +16 -0
- package/numkong/probes/arm_sme2p1.c +16 -0
- package/numkong/probes/arm_sme_bf16.c +16 -0
- package/numkong/probes/arm_sme_bi32.c +16 -0
- package/numkong/probes/arm_sme_f64.c +16 -0
- package/numkong/probes/arm_sme_fa64.c +14 -0
- package/numkong/probes/arm_sme_half.c +16 -0
- package/numkong/probes/arm_sme_lut2.c +15 -0
- package/numkong/probes/arm_sve.c +18 -0
- package/numkong/probes/arm_sve2.c +20 -0
- package/numkong/probes/arm_sve2p1.c +18 -0
- package/numkong/probes/arm_sve_bfdot.c +20 -0
- package/numkong/probes/arm_sve_half.c +18 -0
- package/numkong/probes/arm_sve_sdot.c +21 -0
- package/numkong/probes/loongarch_lasx.c +12 -0
- package/numkong/probes/power_vsx.c +12 -0
- package/numkong/probes/probe.js +127 -0
- package/numkong/probes/riscv_rvv.c +14 -0
- package/numkong/probes/riscv_rvv_bb.c +15 -0
- package/numkong/probes/riscv_rvv_bf16.c +17 -0
- package/numkong/probes/riscv_rvv_half.c +14 -0
- package/numkong/probes/wasm_v128relaxed.c +11 -0
- package/numkong/probes/x86_alder.c +17 -0
- package/numkong/probes/x86_diamond.c +17 -0
- package/numkong/probes/x86_genoa.c +17 -0
- package/numkong/probes/x86_graniteamx.c +19 -0
- package/numkong/probes/x86_haswell.c +11 -0
- package/numkong/probes/x86_icelake.c +17 -0
- package/numkong/probes/x86_sapphire.c +16 -0
- package/numkong/probes/x86_sapphireamx.c +18 -0
- package/numkong/probes/x86_sierra.c +17 -0
- package/numkong/probes/x86_skylake.c +15 -0
- package/numkong/probes/x86_turin.c +17 -0
- package/numkong/rust/README.md +698 -0
- package/numkong/rust/capabilities.rs +96 -0
- package/numkong/rust/cast.rs +314 -0
- package/numkong/rust/curved.rs +391 -0
- package/numkong/rust/each.rs +2721 -0
- package/numkong/rust/geospatial.rs +343 -0
- package/numkong/rust/matrix.rs +5424 -0
- package/numkong/rust/maxsim.rs +540 -0
- package/numkong/rust/mesh.rs +837 -0
- package/numkong/rust/numkong.rs +409 -0
- package/numkong/rust/probability.rs +272 -0
- package/numkong/rust/reduce.rs +1249 -0
- package/numkong/rust/set.rs +194 -0
- package/numkong/rust/sparse.rs +513 -0
- package/numkong/rust/spatial.rs +1815 -0
- package/numkong/rust/tensor.rs +7565 -0
- package/numkong/rust/types.rs +3666 -0
- package/numkong/rust/vector.rs +1899 -0
- package/numkong/rustfmt.toml +2 -0
- package/numkong/swift/Matrix.swift +1388 -0
- package/numkong/swift/MaxSim.swift +151 -0
- package/numkong/swift/NumKong.swift +283 -0
- package/numkong/swift/Numerics.swift +329 -0
- package/numkong/swift/README.md +433 -0
- package/numkong/swift/Scalar.swift +741 -0
- package/numkong/swift/Tensor.swift +209 -0
- package/numkong/test/README.md +315 -0
- package/numkong/test/playwright.config.ts +16 -0
- package/numkong/test/swift/Test.swift +534 -0
- package/numkong/test/test-browser-bundle-runner.mjs +36 -0
- package/numkong/test/test-browser-bundle.html +188 -0
- package/numkong/test/test-browser-runner.mjs +43 -0
- package/numkong/test/test-browser.html +305 -0
- package/numkong/test/test-dtypes.mjs +453 -0
- package/numkong/test/test-wasm.mjs +267 -0
- package/numkong/test/test.cpp +442 -0
- package/numkong/test/test.hpp +580 -0
- package/numkong/test/test.mjs +363 -0
- package/numkong/test/test_base.py +1041 -0
- package/numkong/test/test_cast.cpp +242 -0
- package/numkong/test/test_cross.hpp +551 -0
- package/numkong/test/test_cross.py +467 -0
- package/numkong/test/test_cross_amx.cpp +50 -0
- package/numkong/test/test_cross_arm.cpp +225 -0
- package/numkong/test/test_cross_blas.cpp +354 -0
- package/numkong/test/test_cross_loongarch.cpp +90 -0
- package/numkong/test/test_cross_power.cpp +73 -0
- package/numkong/test/test_cross_rvv.cpp +109 -0
- package/numkong/test/test_cross_serial.cpp +181 -0
- package/numkong/test/test_cross_sme.cpp +148 -0
- package/numkong/test/test_cross_wasm.cpp +141 -0
- package/numkong/test/test_cross_x86.cpp +442 -0
- package/numkong/test/test_curved.cpp +182 -0
- package/numkong/test/test_curved.py +207 -0
- package/numkong/test/test_dot.cpp +353 -0
- package/numkong/test/test_dot.py +296 -0
- package/numkong/test/test_each.cpp +437 -0
- package/numkong/test/test_each.py +756 -0
- package/numkong/test/test_geospatial.cpp +155 -0
- package/numkong/test/test_geospatial.py +241 -0
- package/numkong/test/test_maxsim.cpp +138 -0
- package/numkong/test/test_maxsim.py +209 -0
- package/numkong/test/test_mesh.cpp +203 -0
- package/numkong/test/test_mesh.py +394 -0
- package/numkong/test/test_probability.cpp +136 -0
- package/numkong/test/test_probability.py +166 -0
- package/numkong/test/test_reduce.cpp +563 -0
- package/numkong/test/test_reduce.py +781 -0
- package/numkong/test/test_set.cpp +151 -0
- package/numkong/test/test_set.py +120 -0
- package/numkong/test/test_similarities.py +579 -0
- package/numkong/test/test_sparse.cpp +121 -0
- package/numkong/test/test_sparse.py +118 -0
- package/numkong/test/test_spatial.cpp +518 -0
- package/numkong/test/test_spatial.py +269 -0
- package/numkong/test/test_tensor.cpp +884 -0
- package/numkong/test/test_tensor.py +1417 -0
- package/numkong/test/test_trigonometry.cpp +156 -0
- package/numkong/test/test_trigonometry.py +180 -0
- package/package.json +7 -8
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/simsimd/.git-blame-ignore-revs +0 -2
- package/simsimd/CONTRIBUTING.md +0 -321
- package/simsimd/Cargo.toml +0 -49
- package/simsimd/MANIFEST.in +0 -4
- package/simsimd/Package.swift +0 -28
- package/simsimd/README.md +0 -1493
- package/simsimd/binding.gyp +0 -35
- package/simsimd/build.rs +0 -70
- package/simsimd/c/lib.c +0 -374
- package/simsimd/include/module.modulemap +0 -4
- package/simsimd/include/simsimd/binary.h +0 -486
- package/simsimd/include/simsimd/curved.h +0 -1541
- package/simsimd/include/simsimd/dot.h +0 -1853
- package/simsimd/include/simsimd/elementwise.h +0 -2465
- package/simsimd/include/simsimd/geospatial.h +0 -43
- package/simsimd/include/simsimd/mesh.h +0 -69
- package/simsimd/include/simsimd/probability.h +0 -606
- package/simsimd/include/simsimd/simsimd.h +0 -2567
- package/simsimd/include/simsimd/sparse.h +0 -1384
- package/simsimd/include/simsimd/spatial.h +0 -2335
- package/simsimd/include/simsimd/types.h +0 -666
- package/simsimd/javascript/fallback.ts +0 -233
- package/simsimd/javascript/lib.c +0 -125
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -179
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/package-lock.json +0 -1761
- package/simsimd/package.json +0 -61
- package/simsimd/rust/lib.rs +0 -1821
- package/simsimd/scripts/bench.cxx +0 -1170
- package/simsimd/scripts/bench.ipynb +0 -187
- package/simsimd/scripts/bench.js +0 -127
- package/simsimd/scripts/bench.rs +0 -876
- package/simsimd/scripts/bench_vectors.py +0 -734
- package/simsimd/scripts/bench_vectors_live.py +0 -213
- package/simsimd/scripts/test.c +0 -187
- package/simsimd/scripts/test.mjs +0 -220
- package/simsimd/scripts/test.py +0 -1505
- package/simsimd/swift/SimSIMD.swift +0 -149
- package/simsimd/swift/Test.swift +0 -128
- /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
- /package/{simsimd → numkong}/LICENSE +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
|
@@ -0,0 +1,2344 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @brief SIMD-accelerated Point Cloud Alignment for Skylake.
|
|
3
|
+
* @file include/numkong/mesh/skylake.h
|
|
4
|
+
* @author Ash Vardanian
|
|
5
|
+
* @date December 27, 2025
|
|
6
|
+
*
|
|
7
|
+
* @sa include/numkong/mesh.h
|
|
8
|
+
*
|
|
9
|
+
* @section skylake_mesh_instructions Key AVX-512 Mesh Instructions
|
|
10
|
+
*
|
|
11
|
+
* Intrinsic Instruction Skylake-X Genoa
|
|
12
|
+
* _mm512_fmadd_ps VFMADD132PS (ZMM, ZMM, ZMM) 4cy @ p05 4cy @ p01
|
|
13
|
+
* _mm512_permutexvar_ps VPERMPS (ZMM, ZMM, ZMM) 3cy @ p5 4cy @ p12
|
|
14
|
+
* _mm512_permutex2var_ps VPERMT2PS (ZMM, ZMM, ZMM) 3cy @ p5 4cy @ p12
|
|
15
|
+
* _mm512_extractf32x8_ps VEXTRACTF32X8 (YMM, ZMM, I8) 3cy @ p5 1cy @ p0123
|
|
16
|
+
*
|
|
17
|
+
* Point cloud operations use VPERMT2PS for stride-3 deinterleaving of xyz coordinates, avoiding
|
|
18
|
+
* expensive gather instructions. This achieves ~1.8x speedup over scalar deinterleaving. Dual FMA
|
|
19
|
+
* accumulators on Skylake-X server chips hide the 4cy latency for centroid and covariance computation.
|
|
20
|
+
*/
|
|
21
|
+
#ifndef NK_MESH_SKYLAKE_H
|
|
22
|
+
#define NK_MESH_SKYLAKE_H
|
|
23
|
+
|
|
24
|
+
#if NK_TARGET_X8664_
|
|
25
|
+
#if NK_TARGET_SKYLAKE
|
|
26
|
+
|
|
27
|
+
#include "numkong/types.h"
|
|
28
|
+
#include "numkong/dot/skylake.h"
|
|
29
|
+
#include "numkong/mesh/serial.h"
|
|
30
|
+
#include "numkong/spatial/haswell.h"
|
|
31
|
+
#include "numkong/cast/skylake.h"
|
|
32
|
+
|
|
33
|
+
#if defined(__cplusplus)
|
|
34
|
+
extern "C" {
|
|
35
|
+
#endif
|
|
36
|
+
|
|
37
|
+
#if defined(__clang__)
|
|
38
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512dq,f16c,fma,bmi,bmi2"))), \
|
|
39
|
+
apply_to = function)
|
|
40
|
+
#elif defined(__GNUC__)
|
|
41
|
+
#pragma GCC push_options
|
|
42
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "f16c", "fma", "bmi", "bmi2")
|
|
43
|
+
#endif
|
|
44
|
+
|
|
45
|
+
/* Deinterleave 48 floats (16 xyz triplets) into separate x, y, z vectors.
|
|
46
|
+
* Uses permutex2var shuffles instead of gather for ~1.8x speedup.
|
|
47
|
+
*
|
|
48
|
+
* Input: 48 contiguous floats [x0,y0,z0, x1,y1,z1, ..., x15,y15,z15]
|
|
49
|
+
* Output: x[16], y[16], z[16] vectors
|
|
50
|
+
*
|
|
51
|
+
* Implementation: Load 3 registers (r0,r1,r2), use 6 permutex2var ops to separate.
|
|
52
|
+
* Phase analysis: r0 starts at float 0 (phase 0), r1 at float 16 (phase 1), r2 at float 32 (phase 2)
|
|
53
|
+
*
|
|
54
|
+
* X elements at memory positions: 0,3,6,9,12,15,18,21,24,27,30,33,36,39,42,45
|
|
55
|
+
* = r0[0,3,6,9,12,15], r1[2,5,8,11,14], r2[1,4,7,10,13]
|
|
56
|
+
* Y elements at memory positions: 1,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46
|
|
57
|
+
* = r0[1,4,7,10,13], r1[0,3,6,9,12,15], r2[2,5,8,11,14]
|
|
58
|
+
* Z elements at memory positions: 2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47
|
|
59
|
+
* = r0[2,5,8,11,14], r1[1,4,7,10,13], r2[0,3,6,9,12,15]
|
|
60
|
+
*/
|
|
61
|
+
NK_INTERNAL void nk_deinterleave_f32x16_skylake_( //
|
|
62
|
+
nk_f32_t const *ptr, __m512 *x_f32x16_out, __m512 *y_f32x16_out, __m512 *z_f32x16_out) { //
|
|
63
|
+
__m512 reg0_f32x16 = _mm512_loadu_ps(ptr);
|
|
64
|
+
__m512 reg1_f32x16 = _mm512_loadu_ps(ptr + 16);
|
|
65
|
+
__m512 reg2_f32x16 = _mm512_loadu_ps(ptr + 32);
|
|
66
|
+
|
|
67
|
+
// X: reg0[0,3,6,9,12,15] + reg1[2,5,8,11,14] → 11 elements, then + reg2[1,4,7,10,13] → 16 elements
|
|
68
|
+
// Indices for permutex2var: 0-15 = from first operand, 16-31 = from second operand
|
|
69
|
+
__m512i idx_x_01_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0, 0, 0, 0, 0);
|
|
70
|
+
__m512i idx_x_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 20, 23, 26, 29);
|
|
71
|
+
__m512 x01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_x_01_i32x16, reg1_f32x16);
|
|
72
|
+
*x_f32x16_out = _mm512_permutex2var_ps(x01_f32x16, idx_x_2_i32x16, reg2_f32x16);
|
|
73
|
+
|
|
74
|
+
// Y: reg0[1,4,7,10,13] + reg1[0,3,6,9,12,15] → 11 elements, then + reg2[2,5,8,11,14] → 16 elements
|
|
75
|
+
__m512i idx_y_01_i32x16 = _mm512_setr_epi32(1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 0, 0, 0, 0, 0);
|
|
76
|
+
__m512i idx_y_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 18, 21, 24, 27, 30);
|
|
77
|
+
__m512 y01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_y_01_i32x16, reg1_f32x16);
|
|
78
|
+
*y_f32x16_out = _mm512_permutex2var_ps(y01_f32x16, idx_y_2_i32x16, reg2_f32x16);
|
|
79
|
+
|
|
80
|
+
// Z: reg0[2,5,8,11,14] + reg1[1,4,7,10,13] → 10 elements, then + reg2[0,3,6,9,12,15] → 16 elements
|
|
81
|
+
__m512i idx_z_01_i32x16 = _mm512_setr_epi32(2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0, 0, 0, 0, 0, 0);
|
|
82
|
+
__m512i idx_z_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 19, 22, 25, 28, 31);
|
|
83
|
+
__m512 z01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_z_01_i32x16, reg1_f32x16);
|
|
84
|
+
*z_f32x16_out = _mm512_permutex2var_ps(z01_f32x16, idx_z_2_i32x16, reg2_f32x16);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/* Deinterleave 8 f64 3D points from xyz,xyz,xyz... to separate x,y,z vectors.
|
|
88
|
+
* Input: 24 consecutive f64 values (8 points * 3 coordinates)
|
|
89
|
+
* Output: Three __m512d vectors containing the x, y, z coordinates separately.
|
|
90
|
+
*/
|
|
91
|
+
NK_INTERNAL void nk_deinterleave_f64x8_skylake_( //
|
|
92
|
+
nk_f64_t const *ptr, __m512d *x_f64x8_out, __m512d *y_f64x8_out, __m512d *z_f64x8_out) { //
|
|
93
|
+
__m512d reg0_f64x8 = _mm512_loadu_pd(ptr); // elements 0-7
|
|
94
|
+
__m512d reg1_f64x8 = _mm512_loadu_pd(ptr + 8); // elements 8-15
|
|
95
|
+
__m512d reg2_f64x8 = _mm512_loadu_pd(ptr + 16); // elements 16-23
|
|
96
|
+
|
|
97
|
+
// X: positions 0,3,6,9,12,15,18,21 → reg0[0,3,6] + reg1[1,4,7] + reg2[2,5]
|
|
98
|
+
__m512i idx_x_01_i64x8 = _mm512_setr_epi64(0, 3, 6, 9, 12, 15, 0, 0);
|
|
99
|
+
__m512i idx_x_2_i64x8 = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 10, 13);
|
|
100
|
+
__m512d x01_f64x8 = _mm512_permutex2var_pd(reg0_f64x8, idx_x_01_i64x8, reg1_f64x8);
|
|
101
|
+
*x_f64x8_out = _mm512_permutex2var_pd(x01_f64x8, idx_x_2_i64x8, reg2_f64x8);
|
|
102
|
+
|
|
103
|
+
// Y: positions 1,4,7,10,13,16,19,22 → reg0[1,4,7] + reg1[2,5] + reg2[0,3,6]
|
|
104
|
+
__m512i idx_y_01_i64x8 = _mm512_setr_epi64(1, 4, 7, 10, 13, 0, 0, 0);
|
|
105
|
+
__m512i idx_y_2_i64x8 = _mm512_setr_epi64(0, 1, 2, 3, 4, 8, 11, 14);
|
|
106
|
+
__m512d y01_f64x8 = _mm512_permutex2var_pd(reg0_f64x8, idx_y_01_i64x8, reg1_f64x8);
|
|
107
|
+
*y_f64x8_out = _mm512_permutex2var_pd(y01_f64x8, idx_y_2_i64x8, reg2_f64x8);
|
|
108
|
+
|
|
109
|
+
// Z: positions 2,5,8,11,14,17,20,23 → reg0[2,5] + reg1[0,3,6] + reg2[1,4,7]
|
|
110
|
+
__m512i idx_z_01_i64x8 = _mm512_setr_epi64(2, 5, 8, 11, 14, 0, 0, 0);
|
|
111
|
+
__m512i idx_z_2_i64x8 = _mm512_setr_epi64(0, 1, 2, 3, 4, 9, 12, 15);
|
|
112
|
+
__m512d z01_f64x8 = _mm512_permutex2var_pd(reg0_f64x8, idx_z_01_i64x8, reg1_f64x8);
|
|
113
|
+
*z_f64x8_out = _mm512_permutex2var_pd(z01_f64x8, idx_z_2_i64x8, reg2_f64x8);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/* Deinterleave 16 f16 3D points from xyz,xyz,xyz... to separate x,y,z vectors in f32.
|
|
117
|
+
* Input: 48 consecutive f16 values (16 points * 3 coordinates)
|
|
118
|
+
* Output: Three __m512 vectors containing the x, y, z coordinates separately (as f32).
|
|
119
|
+
*/
|
|
120
|
+
NK_INTERNAL void nk_deinterleave_f16x16_to_f32x16_skylake_( //
|
|
121
|
+
nk_f16_t const *ptr, __m512 *x_f32x16_out, __m512 *y_f32x16_out, __m512 *z_f32x16_out) { //
|
|
122
|
+
__m512 reg0_f32x16 = _mm512_cvtph_ps(_mm256_loadu_si256((__m256i const *)(ptr)));
|
|
123
|
+
__m512 reg1_f32x16 = _mm512_cvtph_ps(_mm256_loadu_si256((__m256i const *)(ptr + 16)));
|
|
124
|
+
__m512 reg2_f32x16 = _mm512_cvtph_ps(_mm256_loadu_si256((__m256i const *)(ptr + 32)));
|
|
125
|
+
|
|
126
|
+
__m512i idx_x_01_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0, 0, 0, 0, 0);
|
|
127
|
+
__m512i idx_x_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 20, 23, 26, 29);
|
|
128
|
+
__m512 x01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_x_01_i32x16, reg1_f32x16);
|
|
129
|
+
*x_f32x16_out = _mm512_permutex2var_ps(x01_f32x16, idx_x_2_i32x16, reg2_f32x16);
|
|
130
|
+
|
|
131
|
+
__m512i idx_y_01_i32x16 = _mm512_setr_epi32(1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 0, 0, 0, 0, 0);
|
|
132
|
+
__m512i idx_y_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 18, 21, 24, 27, 30);
|
|
133
|
+
__m512 y01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_y_01_i32x16, reg1_f32x16);
|
|
134
|
+
*y_f32x16_out = _mm512_permutex2var_ps(y01_f32x16, idx_y_2_i32x16, reg2_f32x16);
|
|
135
|
+
|
|
136
|
+
__m512i idx_z_01_i32x16 = _mm512_setr_epi32(2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0, 0, 0, 0, 0, 0);
|
|
137
|
+
__m512i idx_z_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 19, 22, 25, 28, 31);
|
|
138
|
+
__m512 z01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_z_01_i32x16, reg1_f32x16);
|
|
139
|
+
*z_f32x16_out = _mm512_permutex2var_ps(z01_f32x16, idx_z_2_i32x16, reg2_f32x16);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/* Deinterleave 16 bf16 3D points from xyz,xyz,xyz... to separate x,y,z vectors in f32.
|
|
143
|
+
* Input: 48 consecutive bf16 values (16 points * 3 coordinates)
|
|
144
|
+
* Output: Three __m512 vectors containing the x, y, z coordinates separately (as f32).
|
|
145
|
+
*/
|
|
146
|
+
NK_INTERNAL void nk_deinterleave_bf16x16_to_f32x16_skylake_( //
|
|
147
|
+
nk_bf16_t const *ptr, __m512 *x_f32x16_out, __m512 *y_f32x16_out, __m512 *z_f32x16_out) { //
|
|
148
|
+
__m512 reg0_f32x16 = nk_bf16x16_to_f32x16_skylake_(_mm256_loadu_si256((__m256i const *)(ptr)));
|
|
149
|
+
__m512 reg1_f32x16 = nk_bf16x16_to_f32x16_skylake_(_mm256_loadu_si256((__m256i const *)(ptr + 16)));
|
|
150
|
+
__m512 reg2_f32x16 = nk_bf16x16_to_f32x16_skylake_(_mm256_loadu_si256((__m256i const *)(ptr + 32)));
|
|
151
|
+
|
|
152
|
+
__m512i idx_x_01_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0, 0, 0, 0, 0);
|
|
153
|
+
__m512i idx_x_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 20, 23, 26, 29);
|
|
154
|
+
__m512 x01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_x_01_i32x16, reg1_f32x16);
|
|
155
|
+
*x_f32x16_out = _mm512_permutex2var_ps(x01_f32x16, idx_x_2_i32x16, reg2_f32x16);
|
|
156
|
+
|
|
157
|
+
__m512i idx_y_01_i32x16 = _mm512_setr_epi32(1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 0, 0, 0, 0, 0);
|
|
158
|
+
__m512i idx_y_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 18, 21, 24, 27, 30);
|
|
159
|
+
__m512 y01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_y_01_i32x16, reg1_f32x16);
|
|
160
|
+
*y_f32x16_out = _mm512_permutex2var_ps(y01_f32x16, idx_y_2_i32x16, reg2_f32x16);
|
|
161
|
+
|
|
162
|
+
__m512i idx_z_01_i32x16 = _mm512_setr_epi32(2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0, 0, 0, 0, 0, 0);
|
|
163
|
+
__m512i idx_z_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 19, 22, 25, 28, 31);
|
|
164
|
+
__m512 z01_f32x16 = _mm512_permutex2var_ps(reg0_f32x16, idx_z_01_i32x16, reg1_f32x16);
|
|
165
|
+
*z_f32x16_out = _mm512_permutex2var_ps(z01_f32x16, idx_z_2_i32x16, reg2_f32x16);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/* Masked-tail deinterleave for f16: loads up to 16 xyz points using AVX-512 masked loads,
|
|
169
|
+
* converts f16→f32, and deinterleaves into separate x,y,z vectors.
|
|
170
|
+
* Unused lanes are zero. Uses the same permutex2var shuffle as the full-width version.
|
|
171
|
+
*/
|
|
172
|
+
NK_INTERNAL void nk_deinterleave_f16_tail_to_f32x16_skylake_( //
|
|
173
|
+
nk_f16_t const *ptr, nk_size_t count, __m512 *x_f32x16_out, __m512 *y_f32x16_out, __m512 *z_f32x16_out) { //
|
|
174
|
+
nk_size_t total = count * 3;
|
|
175
|
+
__mmask16 mask0_i16x16 = (__mmask16)_bzhi_u32(0xFFFF, total >= 16 ? 16 : total);
|
|
176
|
+
__mmask16 mask1_i16x16 = total > 16 ? (__mmask16)_bzhi_u32(0xFFFF, total >= 32 ? 16 : total - 16) : 0;
|
|
177
|
+
__mmask16 mask2_i16x16 = total > 32 ? (__mmask16)_bzhi_u32(0xFFFF, total - 32) : 0;
|
|
178
|
+
__m512 reg0_f32x16 = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask0_i16x16, ptr));
|
|
179
|
+
__m512 reg1_f32x16 = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask1_i16x16, ptr + 16));
|
|
180
|
+
__m512 reg2_f32x16 = _mm512_cvtph_ps(_mm256_maskz_loadu_epi16(mask2_i16x16, ptr + 32));
|
|
181
|
+
|
|
182
|
+
__m512i idx_x_01_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0, 0, 0, 0, 0);
|
|
183
|
+
__m512i idx_x_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 20, 23, 26, 29);
|
|
184
|
+
*x_f32x16_out = _mm512_permutex2var_ps(_mm512_permutex2var_ps(reg0_f32x16, idx_x_01_i32x16, reg1_f32x16),
|
|
185
|
+
idx_x_2_i32x16, reg2_f32x16);
|
|
186
|
+
|
|
187
|
+
__m512i idx_y_01_i32x16 = _mm512_setr_epi32(1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 0, 0, 0, 0, 0);
|
|
188
|
+
__m512i idx_y_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 18, 21, 24, 27, 30);
|
|
189
|
+
*y_f32x16_out = _mm512_permutex2var_ps(_mm512_permutex2var_ps(reg0_f32x16, idx_y_01_i32x16, reg1_f32x16),
|
|
190
|
+
idx_y_2_i32x16, reg2_f32x16);
|
|
191
|
+
|
|
192
|
+
__m512i idx_z_01_i32x16 = _mm512_setr_epi32(2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0, 0, 0, 0, 0, 0);
|
|
193
|
+
__m512i idx_z_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 19, 22, 25, 28, 31);
|
|
194
|
+
*z_f32x16_out = _mm512_permutex2var_ps(_mm512_permutex2var_ps(reg0_f32x16, idx_z_01_i32x16, reg1_f32x16),
|
|
195
|
+
idx_z_2_i32x16, reg2_f32x16);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/* Masked-tail deinterleave for bf16: same as f16 but with bf16→f32 conversion. */
|
|
199
|
+
NK_INTERNAL void nk_deinterleave_bf16_tail_to_f32x16_skylake_( //
|
|
200
|
+
nk_bf16_t const *ptr, nk_size_t count, __m512 *x_f32x16_out, __m512 *y_f32x16_out, __m512 *z_f32x16_out) { //
|
|
201
|
+
nk_size_t total = count * 3;
|
|
202
|
+
__mmask16 mask0_i16x16 = (__mmask16)_bzhi_u32(0xFFFF, total >= 16 ? 16 : total);
|
|
203
|
+
__mmask16 mask1_i16x16 = total > 16 ? (__mmask16)_bzhi_u32(0xFFFF, total >= 32 ? 16 : total - 16) : 0;
|
|
204
|
+
__mmask16 mask2_i16x16 = total > 32 ? (__mmask16)_bzhi_u32(0xFFFF, total - 32) : 0;
|
|
205
|
+
__m512 reg0_f32x16 = nk_bf16x16_to_f32x16_skylake_(_mm256_maskz_loadu_epi16(mask0_i16x16, ptr));
|
|
206
|
+
__m512 reg1_f32x16 = nk_bf16x16_to_f32x16_skylake_(_mm256_maskz_loadu_epi16(mask1_i16x16, ptr + 16));
|
|
207
|
+
__m512 reg2_f32x16 = nk_bf16x16_to_f32x16_skylake_(_mm256_maskz_loadu_epi16(mask2_i16x16, ptr + 32));
|
|
208
|
+
|
|
209
|
+
__m512i idx_x_01_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0, 0, 0, 0, 0);
|
|
210
|
+
__m512i idx_x_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 20, 23, 26, 29);
|
|
211
|
+
*x_f32x16_out = _mm512_permutex2var_ps(_mm512_permutex2var_ps(reg0_f32x16, idx_x_01_i32x16, reg1_f32x16),
|
|
212
|
+
idx_x_2_i32x16, reg2_f32x16);
|
|
213
|
+
|
|
214
|
+
__m512i idx_y_01_i32x16 = _mm512_setr_epi32(1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 0, 0, 0, 0, 0);
|
|
215
|
+
__m512i idx_y_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 18, 21, 24, 27, 30);
|
|
216
|
+
*y_f32x16_out = _mm512_permutex2var_ps(_mm512_permutex2var_ps(reg0_f32x16, idx_y_01_i32x16, reg1_f32x16),
|
|
217
|
+
idx_y_2_i32x16, reg2_f32x16);
|
|
218
|
+
|
|
219
|
+
__m512i idx_z_01_i32x16 = _mm512_setr_epi32(2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0, 0, 0, 0, 0, 0);
|
|
220
|
+
__m512i idx_z_2_i32x16 = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16, 19, 22, 25, 28, 31);
|
|
221
|
+
*z_f32x16_out = _mm512_permutex2var_ps(_mm512_permutex2var_ps(reg0_f32x16, idx_z_01_i32x16, reg1_f32x16),
|
|
222
|
+
idx_z_2_i32x16, reg2_f32x16);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
NK_INTERNAL nk_f64_t nk_reduce_stable_f64x8_skylake_(__m512d values_f64x8) {
|
|
226
|
+
nk_b512_vec_t values;
|
|
227
|
+
values.zmm_pd = values_f64x8;
|
|
228
|
+
nk_f64_t sum = 0.0, compensation = 0.0;
|
|
229
|
+
for (nk_size_t lane_index = 0; lane_index != 8; ++lane_index)
|
|
230
|
+
nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[lane_index]);
|
|
231
|
+
return sum + compensation;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
NK_INTERNAL void nk_rotation_from_svd_f64_skylake_(nk_f64_t const *svd_u, nk_f64_t const *svd_v, nk_f64_t *rotation) {
|
|
235
|
+
nk_rotation_from_svd_f64_serial_(svd_u, svd_v, rotation);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
NK_INTERNAL void nk_accumulate_square_f64x8_skylake_(__m512d *sum_f64x8, __m512d *compensation_f64x8,
|
|
239
|
+
__m512d values_f64x8) {
|
|
240
|
+
__m512d product_f64x8 = _mm512_mul_pd(values_f64x8, values_f64x8);
|
|
241
|
+
__m512d product_error_f64x8 = _mm512_fmsub_pd(values_f64x8, values_f64x8, product_f64x8);
|
|
242
|
+
__m512d tentative_sum_f64x8 = _mm512_add_pd(*sum_f64x8, product_f64x8);
|
|
243
|
+
__m512d virtual_addend_f64x8 = _mm512_sub_pd(tentative_sum_f64x8, *sum_f64x8);
|
|
244
|
+
__m512d sum_error_f64x8 = _mm512_add_pd(
|
|
245
|
+
_mm512_sub_pd(*sum_f64x8, _mm512_sub_pd(tentative_sum_f64x8, virtual_addend_f64x8)),
|
|
246
|
+
_mm512_sub_pd(product_f64x8, virtual_addend_f64x8));
|
|
247
|
+
*sum_f64x8 = tentative_sum_f64x8;
|
|
248
|
+
*compensation_f64x8 = _mm512_add_pd(*compensation_f64x8, _mm512_add_pd(sum_error_f64x8, product_error_f64x8));
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/* Compute sum of squared distances after applying rotation (and optional scale).
|
|
252
|
+
* Used by kabsch (scale=1.0) and umeyama (scale=computed_scale).
|
|
253
|
+
* Returns sum_squared, caller computes √(sum_squared / n).
|
|
254
|
+
*/
|
|
255
|
+
NK_INTERNAL nk_f64_t nk_transformed_ssd_f32_skylake_(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n,
|
|
256
|
+
nk_f64_t const *r, nk_f64_t scale, nk_f64_t centroid_a_x,
|
|
257
|
+
nk_f64_t centroid_a_y, nk_f64_t centroid_a_z,
|
|
258
|
+
nk_f64_t centroid_b_x, nk_f64_t centroid_b_y,
|
|
259
|
+
nk_f64_t centroid_b_z) {
|
|
260
|
+
__m512d scaled_rotation_x_x_f64x8 = _mm512_set1_pd(scale * r[0]);
|
|
261
|
+
__m512d scaled_rotation_x_y_f64x8 = _mm512_set1_pd(scale * r[1]);
|
|
262
|
+
__m512d scaled_rotation_x_z_f64x8 = _mm512_set1_pd(scale * r[2]);
|
|
263
|
+
__m512d scaled_rotation_y_x_f64x8 = _mm512_set1_pd(scale * r[3]);
|
|
264
|
+
__m512d scaled_rotation_y_y_f64x8 = _mm512_set1_pd(scale * r[4]);
|
|
265
|
+
__m512d scaled_rotation_y_z_f64x8 = _mm512_set1_pd(scale * r[5]);
|
|
266
|
+
__m512d scaled_rotation_z_x_f64x8 = _mm512_set1_pd(scale * r[6]);
|
|
267
|
+
__m512d scaled_rotation_z_y_f64x8 = _mm512_set1_pd(scale * r[7]);
|
|
268
|
+
__m512d scaled_rotation_z_z_f64x8 = _mm512_set1_pd(scale * r[8]);
|
|
269
|
+
__m512d centroid_a_x_f64x8 = _mm512_set1_pd(centroid_a_x), centroid_a_y_f64x8 = _mm512_set1_pd(centroid_a_y);
|
|
270
|
+
__m512d centroid_a_z_f64x8 = _mm512_set1_pd(centroid_a_z), centroid_b_x_f64x8 = _mm512_set1_pd(centroid_b_x);
|
|
271
|
+
__m512d centroid_b_y_f64x8 = _mm512_set1_pd(centroid_b_y), centroid_b_z_f64x8 = _mm512_set1_pd(centroid_b_z);
|
|
272
|
+
__m512d sum_squared_f64x8 = _mm512_setzero_pd();
|
|
273
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
274
|
+
nk_size_t index = 0;
|
|
275
|
+
|
|
276
|
+
for (; index + 16 <= n; index += 16) {
|
|
277
|
+
nk_deinterleave_f32x16_skylake_(a + index * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16),
|
|
278
|
+
nk_deinterleave_f32x16_skylake_(b + index * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
279
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
280
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
281
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
282
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
283
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
284
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
285
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
286
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
287
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
288
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
289
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
290
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
291
|
+
|
|
292
|
+
__m512d centered_a_x_low_f64x8 = _mm512_sub_pd(a_x_low_f64x8, centroid_a_x_f64x8);
|
|
293
|
+
__m512d centered_a_x_high_f64x8 = _mm512_sub_pd(a_x_high_f64x8, centroid_a_x_f64x8);
|
|
294
|
+
__m512d centered_a_y_low_f64x8 = _mm512_sub_pd(a_y_low_f64x8, centroid_a_y_f64x8);
|
|
295
|
+
__m512d centered_a_y_high_f64x8 = _mm512_sub_pd(a_y_high_f64x8, centroid_a_y_f64x8);
|
|
296
|
+
__m512d centered_a_z_low_f64x8 = _mm512_sub_pd(a_z_low_f64x8, centroid_a_z_f64x8);
|
|
297
|
+
__m512d centered_a_z_high_f64x8 = _mm512_sub_pd(a_z_high_f64x8, centroid_a_z_f64x8);
|
|
298
|
+
__m512d centered_b_x_low_f64x8 = _mm512_sub_pd(b_x_low_f64x8, centroid_b_x_f64x8);
|
|
299
|
+
__m512d centered_b_x_high_f64x8 = _mm512_sub_pd(b_x_high_f64x8, centroid_b_x_f64x8);
|
|
300
|
+
__m512d centered_b_y_low_f64x8 = _mm512_sub_pd(b_y_low_f64x8, centroid_b_y_f64x8);
|
|
301
|
+
__m512d centered_b_y_high_f64x8 = _mm512_sub_pd(b_y_high_f64x8, centroid_b_y_f64x8);
|
|
302
|
+
__m512d centered_b_z_low_f64x8 = _mm512_sub_pd(b_z_low_f64x8, centroid_b_z_f64x8);
|
|
303
|
+
__m512d centered_b_z_high_f64x8 = _mm512_sub_pd(b_z_high_f64x8, centroid_b_z_f64x8);
|
|
304
|
+
|
|
305
|
+
__m512d rotated_a_x_low_f64x8 = _mm512_fmadd_pd(
|
|
306
|
+
scaled_rotation_x_z_f64x8, centered_a_z_low_f64x8,
|
|
307
|
+
_mm512_fmadd_pd(scaled_rotation_x_y_f64x8, centered_a_y_low_f64x8,
|
|
308
|
+
_mm512_mul_pd(scaled_rotation_x_x_f64x8, centered_a_x_low_f64x8)));
|
|
309
|
+
__m512d rotated_a_x_high_f64x8 = _mm512_fmadd_pd(
|
|
310
|
+
scaled_rotation_x_z_f64x8, centered_a_z_high_f64x8,
|
|
311
|
+
_mm512_fmadd_pd(scaled_rotation_x_y_f64x8, centered_a_y_high_f64x8,
|
|
312
|
+
_mm512_mul_pd(scaled_rotation_x_x_f64x8, centered_a_x_high_f64x8)));
|
|
313
|
+
__m512d rotated_a_y_low_f64x8 = _mm512_fmadd_pd(
|
|
314
|
+
scaled_rotation_y_z_f64x8, centered_a_z_low_f64x8,
|
|
315
|
+
_mm512_fmadd_pd(scaled_rotation_y_y_f64x8, centered_a_y_low_f64x8,
|
|
316
|
+
_mm512_mul_pd(scaled_rotation_y_x_f64x8, centered_a_x_low_f64x8)));
|
|
317
|
+
__m512d rotated_a_y_high_f64x8 = _mm512_fmadd_pd(
|
|
318
|
+
scaled_rotation_y_z_f64x8, centered_a_z_high_f64x8,
|
|
319
|
+
_mm512_fmadd_pd(scaled_rotation_y_y_f64x8, centered_a_y_high_f64x8,
|
|
320
|
+
_mm512_mul_pd(scaled_rotation_y_x_f64x8, centered_a_x_high_f64x8)));
|
|
321
|
+
__m512d rotated_a_z_low_f64x8 = _mm512_fmadd_pd(
|
|
322
|
+
scaled_rotation_z_z_f64x8, centered_a_z_low_f64x8,
|
|
323
|
+
_mm512_fmadd_pd(scaled_rotation_z_y_f64x8, centered_a_y_low_f64x8,
|
|
324
|
+
_mm512_mul_pd(scaled_rotation_z_x_f64x8, centered_a_x_low_f64x8)));
|
|
325
|
+
__m512d rotated_a_z_high_f64x8 = _mm512_fmadd_pd(
|
|
326
|
+
scaled_rotation_z_z_f64x8, centered_a_z_high_f64x8,
|
|
327
|
+
_mm512_fmadd_pd(scaled_rotation_z_y_f64x8, centered_a_y_high_f64x8,
|
|
328
|
+
_mm512_mul_pd(scaled_rotation_z_x_f64x8, centered_a_x_high_f64x8)));
|
|
329
|
+
|
|
330
|
+
__m512d delta_x_low_f64x8 = _mm512_sub_pd(rotated_a_x_low_f64x8, centered_b_x_low_f64x8);
|
|
331
|
+
__m512d delta_x_high_f64x8 = _mm512_sub_pd(rotated_a_x_high_f64x8, centered_b_x_high_f64x8);
|
|
332
|
+
__m512d delta_y_low_f64x8 = _mm512_sub_pd(rotated_a_y_low_f64x8, centered_b_y_low_f64x8);
|
|
333
|
+
__m512d delta_y_high_f64x8 = _mm512_sub_pd(rotated_a_y_high_f64x8, centered_b_y_high_f64x8);
|
|
334
|
+
__m512d delta_z_low_f64x8 = _mm512_sub_pd(rotated_a_z_low_f64x8, centered_b_z_low_f64x8);
|
|
335
|
+
__m512d delta_z_high_f64x8 = _mm512_sub_pd(rotated_a_z_high_f64x8, centered_b_z_high_f64x8);
|
|
336
|
+
|
|
337
|
+
__m512d batch_sum_squared_f64x8 = _mm512_add_pd(_mm512_mul_pd(delta_x_low_f64x8, delta_x_low_f64x8),
|
|
338
|
+
_mm512_mul_pd(delta_x_high_f64x8, delta_x_high_f64x8));
|
|
339
|
+
batch_sum_squared_f64x8 = _mm512_fmadd_pd(delta_y_low_f64x8, delta_y_low_f64x8, batch_sum_squared_f64x8);
|
|
340
|
+
batch_sum_squared_f64x8 = _mm512_fmadd_pd(delta_y_high_f64x8, delta_y_high_f64x8, batch_sum_squared_f64x8);
|
|
341
|
+
batch_sum_squared_f64x8 = _mm512_fmadd_pd(delta_z_low_f64x8, delta_z_low_f64x8, batch_sum_squared_f64x8);
|
|
342
|
+
batch_sum_squared_f64x8 = _mm512_fmadd_pd(delta_z_high_f64x8, delta_z_high_f64x8, batch_sum_squared_f64x8);
|
|
343
|
+
sum_squared_f64x8 = _mm512_add_pd(sum_squared_f64x8, batch_sum_squared_f64x8);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
nk_f64_t sum_squared = _mm512_reduce_add_pd(sum_squared_f64x8);
|
|
347
|
+
for (; index < n; ++index) {
|
|
348
|
+
nk_f64_t centered_a_x = (nk_f64_t)a[index * 3 + 0] - centroid_a_x,
|
|
349
|
+
centered_a_y = (nk_f64_t)a[index * 3 + 1] - centroid_a_y,
|
|
350
|
+
centered_a_z = (nk_f64_t)a[index * 3 + 2] - centroid_a_z;
|
|
351
|
+
nk_f64_t centered_b_x = (nk_f64_t)b[index * 3 + 0] - centroid_b_x,
|
|
352
|
+
centered_b_y = (nk_f64_t)b[index * 3 + 1] - centroid_b_y,
|
|
353
|
+
centered_b_z = (nk_f64_t)b[index * 3 + 2] - centroid_b_z;
|
|
354
|
+
nk_f64_t rotated_a_x = scale * (r[0] * centered_a_x + r[1] * centered_a_y + r[2] * centered_a_z),
|
|
355
|
+
rotated_a_y = scale * (r[3] * centered_a_x + r[4] * centered_a_y + r[5] * centered_a_z),
|
|
356
|
+
rotated_a_z = scale * (r[6] * centered_a_x + r[7] * centered_a_y + r[8] * centered_a_z);
|
|
357
|
+
nk_f64_t delta_x = rotated_a_x - centered_b_x, delta_y = rotated_a_y - centered_b_y,
|
|
358
|
+
delta_z = rotated_a_z - centered_b_z;
|
|
359
|
+
sum_squared += delta_x * delta_x + delta_y * delta_y + delta_z * delta_z;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
return sum_squared;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/* Compute sum of squared distances for f64 after applying rotation (and optional scale).
|
|
366
|
+
* Rotation matrix, scale and data are all f64 for full precision.
|
|
367
|
+
*/
|
|
368
|
+
NK_INTERNAL nk_f64_t nk_transformed_ssd_f64_skylake_(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n,
|
|
369
|
+
nk_f64_t const *r, nk_f64_t scale, nk_f64_t centroid_a_x,
|
|
370
|
+
nk_f64_t centroid_a_y, nk_f64_t centroid_a_z,
|
|
371
|
+
nk_f64_t centroid_b_x, nk_f64_t centroid_b_y,
|
|
372
|
+
nk_f64_t centroid_b_z) {
|
|
373
|
+
// Broadcast scaled rotation matrix elements
|
|
374
|
+
__m512d scaled_rotation_x_x_f64x8 = _mm512_set1_pd(scale * r[0]);
|
|
375
|
+
__m512d scaled_rotation_x_y_f64x8 = _mm512_set1_pd(scale * r[1]);
|
|
376
|
+
__m512d scaled_rotation_x_z_f64x8 = _mm512_set1_pd(scale * r[2]);
|
|
377
|
+
__m512d scaled_rotation_y_x_f64x8 = _mm512_set1_pd(scale * r[3]);
|
|
378
|
+
__m512d scaled_rotation_y_y_f64x8 = _mm512_set1_pd(scale * r[4]);
|
|
379
|
+
__m512d scaled_rotation_y_z_f64x8 = _mm512_set1_pd(scale * r[5]);
|
|
380
|
+
__m512d scaled_rotation_z_x_f64x8 = _mm512_set1_pd(scale * r[6]);
|
|
381
|
+
__m512d scaled_rotation_z_y_f64x8 = _mm512_set1_pd(scale * r[7]);
|
|
382
|
+
__m512d scaled_rotation_z_z_f64x8 = _mm512_set1_pd(scale * r[8]);
|
|
383
|
+
|
|
384
|
+
// Broadcast centroids
|
|
385
|
+
__m512d centroid_a_x_f64x8 = _mm512_set1_pd(centroid_a_x);
|
|
386
|
+
__m512d centroid_a_y_f64x8 = _mm512_set1_pd(centroid_a_y);
|
|
387
|
+
__m512d centroid_a_z_f64x8 = _mm512_set1_pd(centroid_a_z);
|
|
388
|
+
__m512d centroid_b_x_f64x8 = _mm512_set1_pd(centroid_b_x);
|
|
389
|
+
__m512d centroid_b_y_f64x8 = _mm512_set1_pd(centroid_b_y);
|
|
390
|
+
__m512d centroid_b_z_f64x8 = _mm512_set1_pd(centroid_b_z);
|
|
391
|
+
|
|
392
|
+
__m512d sum_squared_f64x8 = _mm512_setzero_pd();
|
|
393
|
+
__m512d sum_squared_compensation_f64x8 = _mm512_setzero_pd();
|
|
394
|
+
__m512d a_x_f64x8, a_y_f64x8, a_z_f64x8, b_x_f64x8, b_y_f64x8, b_z_f64x8;
|
|
395
|
+
nk_size_t j = 0;
|
|
396
|
+
|
|
397
|
+
for (; j + 8 <= n; j += 8) {
|
|
398
|
+
nk_deinterleave_f64x8_skylake_(a + j * 3, &a_x_f64x8, &a_y_f64x8, &a_z_f64x8);
|
|
399
|
+
nk_deinterleave_f64x8_skylake_(b + j * 3, &b_x_f64x8, &b_y_f64x8, &b_z_f64x8);
|
|
400
|
+
|
|
401
|
+
// Center points
|
|
402
|
+
__m512d pa_x_f64x8 = _mm512_sub_pd(a_x_f64x8, centroid_a_x_f64x8);
|
|
403
|
+
__m512d pa_y_f64x8 = _mm512_sub_pd(a_y_f64x8, centroid_a_y_f64x8);
|
|
404
|
+
__m512d pa_z_f64x8 = _mm512_sub_pd(a_z_f64x8, centroid_a_z_f64x8);
|
|
405
|
+
__m512d pb_x_f64x8 = _mm512_sub_pd(b_x_f64x8, centroid_b_x_f64x8);
|
|
406
|
+
__m512d pb_y_f64x8 = _mm512_sub_pd(b_y_f64x8, centroid_b_y_f64x8);
|
|
407
|
+
__m512d pb_z_f64x8 = _mm512_sub_pd(b_z_f64x8, centroid_b_z_f64x8);
|
|
408
|
+
|
|
409
|
+
// Rotate and scale: ra = scale * R * pa
|
|
410
|
+
__m512d ra_x_f64x8 = _mm512_fmadd_pd(scaled_rotation_x_z_f64x8, pa_z_f64x8,
|
|
411
|
+
_mm512_fmadd_pd(scaled_rotation_x_y_f64x8, pa_y_f64x8,
|
|
412
|
+
_mm512_mul_pd(scaled_rotation_x_x_f64x8, pa_x_f64x8)));
|
|
413
|
+
__m512d ra_y_f64x8 = _mm512_fmadd_pd(scaled_rotation_y_z_f64x8, pa_z_f64x8,
|
|
414
|
+
_mm512_fmadd_pd(scaled_rotation_y_y_f64x8, pa_y_f64x8,
|
|
415
|
+
_mm512_mul_pd(scaled_rotation_y_x_f64x8, pa_x_f64x8)));
|
|
416
|
+
__m512d ra_z_f64x8 = _mm512_fmadd_pd(scaled_rotation_z_z_f64x8, pa_z_f64x8,
|
|
417
|
+
_mm512_fmadd_pd(scaled_rotation_z_y_f64x8, pa_y_f64x8,
|
|
418
|
+
_mm512_mul_pd(scaled_rotation_z_x_f64x8, pa_x_f64x8)));
|
|
419
|
+
|
|
420
|
+
// Delta and accumulate
|
|
421
|
+
__m512d delta_x_f64x8 = _mm512_sub_pd(ra_x_f64x8, pb_x_f64x8);
|
|
422
|
+
__m512d delta_y_f64x8 = _mm512_sub_pd(ra_y_f64x8, pb_y_f64x8);
|
|
423
|
+
__m512d delta_z_f64x8 = _mm512_sub_pd(ra_z_f64x8, pb_z_f64x8);
|
|
424
|
+
|
|
425
|
+
nk_accumulate_square_f64x8_skylake_(&sum_squared_f64x8, &sum_squared_compensation_f64x8, delta_x_f64x8);
|
|
426
|
+
nk_accumulate_square_f64x8_skylake_(&sum_squared_f64x8, &sum_squared_compensation_f64x8, delta_y_f64x8);
|
|
427
|
+
nk_accumulate_square_f64x8_skylake_(&sum_squared_f64x8, &sum_squared_compensation_f64x8, delta_z_f64x8);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
nk_f64_t sum_squared = nk_dot_stable_sum_f64x8_skylake_(sum_squared_f64x8, sum_squared_compensation_f64x8);
|
|
431
|
+
nk_f64_t sum_squared_compensation = 0.0;
|
|
432
|
+
|
|
433
|
+
// Scalar tail
|
|
434
|
+
for (; j < n; ++j) {
|
|
435
|
+
nk_f64_t pa_x = a[j * 3 + 0] - centroid_a_x, pa_y = a[j * 3 + 1] - centroid_a_y,
|
|
436
|
+
pa_z = a[j * 3 + 2] - centroid_a_z;
|
|
437
|
+
nk_f64_t pb_x = b[j * 3 + 0] - centroid_b_x, pb_y = b[j * 3 + 1] - centroid_b_y,
|
|
438
|
+
pb_z = b[j * 3 + 2] - centroid_b_z;
|
|
439
|
+
|
|
440
|
+
nk_f64_t ra_x = scale * (r[0] * pa_x + r[1] * pa_y + r[2] * pa_z),
|
|
441
|
+
ra_y = scale * (r[3] * pa_x + r[4] * pa_y + r[5] * pa_z),
|
|
442
|
+
ra_z = scale * (r[6] * pa_x + r[7] * pa_y + r[8] * pa_z);
|
|
443
|
+
|
|
444
|
+
nk_f64_t delta_x = ra_x - pb_x, delta_y = ra_y - pb_y, delta_z = ra_z - pb_z;
|
|
445
|
+
nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_x);
|
|
446
|
+
nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_y);
|
|
447
|
+
nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_z);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
return sum_squared + sum_squared_compensation;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/* Compute sum of squared distances for f16 data after applying rotation (and optional scale).
|
|
454
|
+
* Loads f16, converts to f32 for computation. Rotation matrix, scale, and centroids are f32.
|
|
455
|
+
*/
|
|
456
|
+
NK_INTERNAL nk_f32_t nk_transformed_ssd_f16_skylake_(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n,
|
|
457
|
+
nk_f32_t const *r, nk_f32_t scale, nk_f32_t centroid_a_x,
|
|
458
|
+
nk_f32_t centroid_a_y, nk_f32_t centroid_a_z,
|
|
459
|
+
nk_f32_t centroid_b_x, nk_f32_t centroid_b_y,
|
|
460
|
+
nk_f32_t centroid_b_z) {
|
|
461
|
+
__m512 scaled_rotation_x_x_f32x16 = _mm512_set1_ps(scale * r[0]);
|
|
462
|
+
__m512 scaled_rotation_x_y_f32x16 = _mm512_set1_ps(scale * r[1]);
|
|
463
|
+
__m512 scaled_rotation_x_z_f32x16 = _mm512_set1_ps(scale * r[2]);
|
|
464
|
+
__m512 scaled_rotation_y_x_f32x16 = _mm512_set1_ps(scale * r[3]);
|
|
465
|
+
__m512 scaled_rotation_y_y_f32x16 = _mm512_set1_ps(scale * r[4]);
|
|
466
|
+
__m512 scaled_rotation_y_z_f32x16 = _mm512_set1_ps(scale * r[5]);
|
|
467
|
+
__m512 scaled_rotation_z_x_f32x16 = _mm512_set1_ps(scale * r[6]);
|
|
468
|
+
__m512 scaled_rotation_z_y_f32x16 = _mm512_set1_ps(scale * r[7]);
|
|
469
|
+
__m512 scaled_rotation_z_z_f32x16 = _mm512_set1_ps(scale * r[8]);
|
|
470
|
+
|
|
471
|
+
__m512 centroid_a_x_f32x16 = _mm512_set1_ps(centroid_a_x);
|
|
472
|
+
__m512 centroid_a_y_f32x16 = _mm512_set1_ps(centroid_a_y);
|
|
473
|
+
__m512 centroid_a_z_f32x16 = _mm512_set1_ps(centroid_a_z);
|
|
474
|
+
__m512 centroid_b_x_f32x16 = _mm512_set1_ps(centroid_b_x);
|
|
475
|
+
__m512 centroid_b_y_f32x16 = _mm512_set1_ps(centroid_b_y);
|
|
476
|
+
__m512 centroid_b_z_f32x16 = _mm512_set1_ps(centroid_b_z);
|
|
477
|
+
|
|
478
|
+
__m512 sum_squared_f32x16 = _mm512_setzero_ps();
|
|
479
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
480
|
+
nk_size_t j = 0;
|
|
481
|
+
|
|
482
|
+
for (; j + 16 <= n; j += 16) {
|
|
483
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(a + j * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
484
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(b + j * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
485
|
+
|
|
486
|
+
__m512 pa_x_f32x16 = _mm512_sub_ps(a_x_f32x16, centroid_a_x_f32x16);
|
|
487
|
+
__m512 pa_y_f32x16 = _mm512_sub_ps(a_y_f32x16, centroid_a_y_f32x16);
|
|
488
|
+
__m512 pa_z_f32x16 = _mm512_sub_ps(a_z_f32x16, centroid_a_z_f32x16);
|
|
489
|
+
__m512 pb_x_f32x16 = _mm512_sub_ps(b_x_f32x16, centroid_b_x_f32x16);
|
|
490
|
+
__m512 pb_y_f32x16 = _mm512_sub_ps(b_y_f32x16, centroid_b_y_f32x16);
|
|
491
|
+
__m512 pb_z_f32x16 = _mm512_sub_ps(b_z_f32x16, centroid_b_z_f32x16);
|
|
492
|
+
|
|
493
|
+
__m512 ra_x_f32x16 = _mm512_fmadd_ps(scaled_rotation_x_z_f32x16, pa_z_f32x16,
|
|
494
|
+
_mm512_fmadd_ps(scaled_rotation_x_y_f32x16, pa_y_f32x16,
|
|
495
|
+
_mm512_mul_ps(scaled_rotation_x_x_f32x16, pa_x_f32x16)));
|
|
496
|
+
__m512 ra_y_f32x16 = _mm512_fmadd_ps(scaled_rotation_y_z_f32x16, pa_z_f32x16,
|
|
497
|
+
_mm512_fmadd_ps(scaled_rotation_y_y_f32x16, pa_y_f32x16,
|
|
498
|
+
_mm512_mul_ps(scaled_rotation_y_x_f32x16, pa_x_f32x16)));
|
|
499
|
+
__m512 ra_z_f32x16 = _mm512_fmadd_ps(scaled_rotation_z_z_f32x16, pa_z_f32x16,
|
|
500
|
+
_mm512_fmadd_ps(scaled_rotation_z_y_f32x16, pa_y_f32x16,
|
|
501
|
+
_mm512_mul_ps(scaled_rotation_z_x_f32x16, pa_x_f32x16)));
|
|
502
|
+
|
|
503
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(ra_x_f32x16, pb_x_f32x16);
|
|
504
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(ra_y_f32x16, pb_y_f32x16);
|
|
505
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(ra_z_f32x16, pb_z_f32x16);
|
|
506
|
+
|
|
507
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_f32x16);
|
|
508
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_f32x16);
|
|
509
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_f32x16);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
513
|
+
if (j < n) {
|
|
514
|
+
nk_size_t tail = n - j;
|
|
515
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(a + j * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
516
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(b + j * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
517
|
+
|
|
518
|
+
__m512 pa_x_f32x16 = _mm512_sub_ps(a_x_f32x16, centroid_a_x_f32x16);
|
|
519
|
+
__m512 pa_y_f32x16 = _mm512_sub_ps(a_y_f32x16, centroid_a_y_f32x16);
|
|
520
|
+
__m512 pa_z_f32x16 = _mm512_sub_ps(a_z_f32x16, centroid_a_z_f32x16);
|
|
521
|
+
__m512 pb_x_f32x16 = _mm512_sub_ps(b_x_f32x16, centroid_b_x_f32x16);
|
|
522
|
+
__m512 pb_y_f32x16 = _mm512_sub_ps(b_y_f32x16, centroid_b_y_f32x16);
|
|
523
|
+
__m512 pb_z_f32x16 = _mm512_sub_ps(b_z_f32x16, centroid_b_z_f32x16);
|
|
524
|
+
|
|
525
|
+
__m512 ra_x_f32x16 = _mm512_fmadd_ps(scaled_rotation_x_z_f32x16, pa_z_f32x16,
|
|
526
|
+
_mm512_fmadd_ps(scaled_rotation_x_y_f32x16, pa_y_f32x16,
|
|
527
|
+
_mm512_mul_ps(scaled_rotation_x_x_f32x16, pa_x_f32x16)));
|
|
528
|
+
__m512 ra_y_f32x16 = _mm512_fmadd_ps(scaled_rotation_y_z_f32x16, pa_z_f32x16,
|
|
529
|
+
_mm512_fmadd_ps(scaled_rotation_y_y_f32x16, pa_y_f32x16,
|
|
530
|
+
_mm512_mul_ps(scaled_rotation_y_x_f32x16, pa_x_f32x16)));
|
|
531
|
+
__m512 ra_z_f32x16 = _mm512_fmadd_ps(scaled_rotation_z_z_f32x16, pa_z_f32x16,
|
|
532
|
+
_mm512_fmadd_ps(scaled_rotation_z_y_f32x16, pa_y_f32x16,
|
|
533
|
+
_mm512_mul_ps(scaled_rotation_z_x_f32x16, pa_x_f32x16)));
|
|
534
|
+
|
|
535
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(ra_x_f32x16, pb_x_f32x16);
|
|
536
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(ra_y_f32x16, pb_y_f32x16);
|
|
537
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(ra_z_f32x16, pb_z_f32x16);
|
|
538
|
+
|
|
539
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_f32x16);
|
|
540
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_f32x16);
|
|
541
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_f32x16);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
return _mm512_reduce_add_ps(sum_squared_f32x16);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/* Compute sum of squared distances for bf16 data after applying rotation (and optional scale).
|
|
548
|
+
* Loads bf16, converts to f32 for computation. Rotation matrix, scale, and centroids are f32.
|
|
549
|
+
*/
|
|
550
|
+
NK_INTERNAL nk_f32_t nk_transformed_ssd_bf16_skylake_(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n,
|
|
551
|
+
nk_f32_t const *r, nk_f32_t scale, nk_f32_t centroid_a_x,
|
|
552
|
+
nk_f32_t centroid_a_y, nk_f32_t centroid_a_z,
|
|
553
|
+
nk_f32_t centroid_b_x, nk_f32_t centroid_b_y,
|
|
554
|
+
nk_f32_t centroid_b_z) {
|
|
555
|
+
__m512 scaled_rotation_x_x_f32x16 = _mm512_set1_ps(scale * r[0]);
|
|
556
|
+
__m512 scaled_rotation_x_y_f32x16 = _mm512_set1_ps(scale * r[1]);
|
|
557
|
+
__m512 scaled_rotation_x_z_f32x16 = _mm512_set1_ps(scale * r[2]);
|
|
558
|
+
__m512 scaled_rotation_y_x_f32x16 = _mm512_set1_ps(scale * r[3]);
|
|
559
|
+
__m512 scaled_rotation_y_y_f32x16 = _mm512_set1_ps(scale * r[4]);
|
|
560
|
+
__m512 scaled_rotation_y_z_f32x16 = _mm512_set1_ps(scale * r[5]);
|
|
561
|
+
__m512 scaled_rotation_z_x_f32x16 = _mm512_set1_ps(scale * r[6]);
|
|
562
|
+
__m512 scaled_rotation_z_y_f32x16 = _mm512_set1_ps(scale * r[7]);
|
|
563
|
+
__m512 scaled_rotation_z_z_f32x16 = _mm512_set1_ps(scale * r[8]);
|
|
564
|
+
|
|
565
|
+
__m512 centroid_a_x_f32x16 = _mm512_set1_ps(centroid_a_x);
|
|
566
|
+
__m512 centroid_a_y_f32x16 = _mm512_set1_ps(centroid_a_y);
|
|
567
|
+
__m512 centroid_a_z_f32x16 = _mm512_set1_ps(centroid_a_z);
|
|
568
|
+
__m512 centroid_b_x_f32x16 = _mm512_set1_ps(centroid_b_x);
|
|
569
|
+
__m512 centroid_b_y_f32x16 = _mm512_set1_ps(centroid_b_y);
|
|
570
|
+
__m512 centroid_b_z_f32x16 = _mm512_set1_ps(centroid_b_z);
|
|
571
|
+
|
|
572
|
+
__m512 sum_squared_f32x16 = _mm512_setzero_ps();
|
|
573
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
574
|
+
nk_size_t j = 0;
|
|
575
|
+
|
|
576
|
+
for (; j + 16 <= n; j += 16) {
|
|
577
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(a + j * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
578
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(b + j * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
579
|
+
|
|
580
|
+
__m512 pa_x_f32x16 = _mm512_sub_ps(a_x_f32x16, centroid_a_x_f32x16);
|
|
581
|
+
__m512 pa_y_f32x16 = _mm512_sub_ps(a_y_f32x16, centroid_a_y_f32x16);
|
|
582
|
+
__m512 pa_z_f32x16 = _mm512_sub_ps(a_z_f32x16, centroid_a_z_f32x16);
|
|
583
|
+
__m512 pb_x_f32x16 = _mm512_sub_ps(b_x_f32x16, centroid_b_x_f32x16);
|
|
584
|
+
__m512 pb_y_f32x16 = _mm512_sub_ps(b_y_f32x16, centroid_b_y_f32x16);
|
|
585
|
+
__m512 pb_z_f32x16 = _mm512_sub_ps(b_z_f32x16, centroid_b_z_f32x16);
|
|
586
|
+
|
|
587
|
+
__m512 ra_x_f32x16 = _mm512_fmadd_ps(scaled_rotation_x_z_f32x16, pa_z_f32x16,
|
|
588
|
+
_mm512_fmadd_ps(scaled_rotation_x_y_f32x16, pa_y_f32x16,
|
|
589
|
+
_mm512_mul_ps(scaled_rotation_x_x_f32x16, pa_x_f32x16)));
|
|
590
|
+
__m512 ra_y_f32x16 = _mm512_fmadd_ps(scaled_rotation_y_z_f32x16, pa_z_f32x16,
|
|
591
|
+
_mm512_fmadd_ps(scaled_rotation_y_y_f32x16, pa_y_f32x16,
|
|
592
|
+
_mm512_mul_ps(scaled_rotation_y_x_f32x16, pa_x_f32x16)));
|
|
593
|
+
__m512 ra_z_f32x16 = _mm512_fmadd_ps(scaled_rotation_z_z_f32x16, pa_z_f32x16,
|
|
594
|
+
_mm512_fmadd_ps(scaled_rotation_z_y_f32x16, pa_y_f32x16,
|
|
595
|
+
_mm512_mul_ps(scaled_rotation_z_x_f32x16, pa_x_f32x16)));
|
|
596
|
+
|
|
597
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(ra_x_f32x16, pb_x_f32x16);
|
|
598
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(ra_y_f32x16, pb_y_f32x16);
|
|
599
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(ra_z_f32x16, pb_z_f32x16);
|
|
600
|
+
|
|
601
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_f32x16);
|
|
602
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_f32x16);
|
|
603
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_f32x16);
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
607
|
+
if (j < n) {
|
|
608
|
+
nk_size_t tail = n - j;
|
|
609
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(a + j * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
610
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(b + j * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
611
|
+
|
|
612
|
+
__m512 pa_x_f32x16 = _mm512_sub_ps(a_x_f32x16, centroid_a_x_f32x16);
|
|
613
|
+
__m512 pa_y_f32x16 = _mm512_sub_ps(a_y_f32x16, centroid_a_y_f32x16);
|
|
614
|
+
__m512 pa_z_f32x16 = _mm512_sub_ps(a_z_f32x16, centroid_a_z_f32x16);
|
|
615
|
+
__m512 pb_x_f32x16 = _mm512_sub_ps(b_x_f32x16, centroid_b_x_f32x16);
|
|
616
|
+
__m512 pb_y_f32x16 = _mm512_sub_ps(b_y_f32x16, centroid_b_y_f32x16);
|
|
617
|
+
__m512 pb_z_f32x16 = _mm512_sub_ps(b_z_f32x16, centroid_b_z_f32x16);
|
|
618
|
+
|
|
619
|
+
__m512 ra_x_f32x16 = _mm512_fmadd_ps(scaled_rotation_x_z_f32x16, pa_z_f32x16,
|
|
620
|
+
_mm512_fmadd_ps(scaled_rotation_x_y_f32x16, pa_y_f32x16,
|
|
621
|
+
_mm512_mul_ps(scaled_rotation_x_x_f32x16, pa_x_f32x16)));
|
|
622
|
+
__m512 ra_y_f32x16 = _mm512_fmadd_ps(scaled_rotation_y_z_f32x16, pa_z_f32x16,
|
|
623
|
+
_mm512_fmadd_ps(scaled_rotation_y_y_f32x16, pa_y_f32x16,
|
|
624
|
+
_mm512_mul_ps(scaled_rotation_y_x_f32x16, pa_x_f32x16)));
|
|
625
|
+
__m512 ra_z_f32x16 = _mm512_fmadd_ps(scaled_rotation_z_z_f32x16, pa_z_f32x16,
|
|
626
|
+
_mm512_fmadd_ps(scaled_rotation_z_y_f32x16, pa_y_f32x16,
|
|
627
|
+
_mm512_mul_ps(scaled_rotation_z_x_f32x16, pa_x_f32x16)));
|
|
628
|
+
|
|
629
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(ra_x_f32x16, pb_x_f32x16);
|
|
630
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(ra_y_f32x16, pb_y_f32x16);
|
|
631
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(ra_z_f32x16, pb_z_f32x16);
|
|
632
|
+
|
|
633
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_f32x16);
|
|
634
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_f32x16);
|
|
635
|
+
sum_squared_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_f32x16);
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
return _mm512_reduce_add_ps(sum_squared_f32x16);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
NK_PUBLIC void nk_rmsd_f32_skylake(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
642
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
|
|
643
|
+
if (rotation)
|
|
644
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
645
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
646
|
+
if (scale) *scale = 1.0f;
|
|
647
|
+
if (a_centroid) a_centroid[0] = 0, a_centroid[1] = 0, a_centroid[2] = 0;
|
|
648
|
+
if (b_centroid) b_centroid[0] = 0, b_centroid[1] = 0, b_centroid[2] = 0;
|
|
649
|
+
|
|
650
|
+
__m512d const zeros_f64x8 = _mm512_setzero_pd();
|
|
651
|
+
__m512d sum_squared_x_f64x8 = zeros_f64x8, sum_squared_y_f64x8 = zeros_f64x8, sum_squared_z_f64x8 = zeros_f64x8;
|
|
652
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
653
|
+
nk_size_t i = 0;
|
|
654
|
+
|
|
655
|
+
// Main loop with 2x unrolling (32 points per iteration)
|
|
656
|
+
for (; i + 32 <= n; i += 32) {
|
|
657
|
+
// Iteration 0: points i..i+15
|
|
658
|
+
nk_deinterleave_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
659
|
+
nk_deinterleave_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
660
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
661
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
662
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
663
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
664
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
665
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
666
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
667
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
668
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
669
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
670
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
671
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
672
|
+
|
|
673
|
+
__m512d delta_x_low_f64x8 = _mm512_sub_pd(a_x_low_f64x8, b_x_low_f64x8);
|
|
674
|
+
__m512d delta_x_high_f64x8 = _mm512_sub_pd(a_x_high_f64x8, b_x_high_f64x8);
|
|
675
|
+
__m512d delta_y_low_f64x8 = _mm512_sub_pd(a_y_low_f64x8, b_y_low_f64x8);
|
|
676
|
+
__m512d delta_y_high_f64x8 = _mm512_sub_pd(a_y_high_f64x8, b_y_high_f64x8);
|
|
677
|
+
__m512d delta_z_low_f64x8 = _mm512_sub_pd(a_z_low_f64x8, b_z_low_f64x8);
|
|
678
|
+
__m512d delta_z_high_f64x8 = _mm512_sub_pd(a_z_high_f64x8, b_z_high_f64x8);
|
|
679
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_low_f64x8, delta_x_low_f64x8, sum_squared_x_f64x8);
|
|
680
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_high_f64x8, delta_x_high_f64x8, sum_squared_x_f64x8);
|
|
681
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_low_f64x8, delta_y_low_f64x8, sum_squared_y_f64x8);
|
|
682
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_high_f64x8, delta_y_high_f64x8, sum_squared_y_f64x8);
|
|
683
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_low_f64x8, delta_z_low_f64x8, sum_squared_z_f64x8);
|
|
684
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_high_f64x8, delta_z_high_f64x8, sum_squared_z_f64x8);
|
|
685
|
+
|
|
686
|
+
// Iteration 1: points i+16..i+31
|
|
687
|
+
nk_deinterleave_f32x16_skylake_(a + (i + 16) * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
688
|
+
nk_deinterleave_f32x16_skylake_(b + (i + 16) * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
689
|
+
a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
690
|
+
a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
691
|
+
a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
692
|
+
a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
693
|
+
a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
694
|
+
a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
695
|
+
b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
696
|
+
b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
697
|
+
b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
698
|
+
b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
699
|
+
b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
700
|
+
b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
701
|
+
|
|
702
|
+
delta_x_low_f64x8 = _mm512_sub_pd(a_x_low_f64x8, b_x_low_f64x8);
|
|
703
|
+
delta_x_high_f64x8 = _mm512_sub_pd(a_x_high_f64x8, b_x_high_f64x8);
|
|
704
|
+
delta_y_low_f64x8 = _mm512_sub_pd(a_y_low_f64x8, b_y_low_f64x8);
|
|
705
|
+
delta_y_high_f64x8 = _mm512_sub_pd(a_y_high_f64x8, b_y_high_f64x8);
|
|
706
|
+
delta_z_low_f64x8 = _mm512_sub_pd(a_z_low_f64x8, b_z_low_f64x8);
|
|
707
|
+
delta_z_high_f64x8 = _mm512_sub_pd(a_z_high_f64x8, b_z_high_f64x8);
|
|
708
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_low_f64x8, delta_x_low_f64x8, sum_squared_x_f64x8);
|
|
709
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_high_f64x8, delta_x_high_f64x8, sum_squared_x_f64x8);
|
|
710
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_low_f64x8, delta_y_low_f64x8, sum_squared_y_f64x8);
|
|
711
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_high_f64x8, delta_y_high_f64x8, sum_squared_y_f64x8);
|
|
712
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_low_f64x8, delta_z_low_f64x8, sum_squared_z_f64x8);
|
|
713
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_high_f64x8, delta_z_high_f64x8, sum_squared_z_f64x8);
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
// Handle 16-point remainder
|
|
717
|
+
for (; i + 16 <= n; i += 16) {
|
|
718
|
+
nk_deinterleave_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
719
|
+
nk_deinterleave_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
720
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
721
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
722
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
723
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
724
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
725
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
726
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
727
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
728
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
729
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
730
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
731
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
732
|
+
|
|
733
|
+
__m512d delta_x_low_f64x8 = _mm512_sub_pd(a_x_low_f64x8, b_x_low_f64x8);
|
|
734
|
+
__m512d delta_x_high_f64x8 = _mm512_sub_pd(a_x_high_f64x8, b_x_high_f64x8);
|
|
735
|
+
__m512d delta_y_low_f64x8 = _mm512_sub_pd(a_y_low_f64x8, b_y_low_f64x8);
|
|
736
|
+
__m512d delta_y_high_f64x8 = _mm512_sub_pd(a_y_high_f64x8, b_y_high_f64x8);
|
|
737
|
+
__m512d delta_z_low_f64x8 = _mm512_sub_pd(a_z_low_f64x8, b_z_low_f64x8);
|
|
738
|
+
__m512d delta_z_high_f64x8 = _mm512_sub_pd(a_z_high_f64x8, b_z_high_f64x8);
|
|
739
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_low_f64x8, delta_x_low_f64x8, sum_squared_x_f64x8);
|
|
740
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_high_f64x8, delta_x_high_f64x8, sum_squared_x_f64x8);
|
|
741
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_low_f64x8, delta_y_low_f64x8, sum_squared_y_f64x8);
|
|
742
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_high_f64x8, delta_y_high_f64x8, sum_squared_y_f64x8);
|
|
743
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_low_f64x8, delta_z_low_f64x8, sum_squared_z_f64x8);
|
|
744
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_high_f64x8, delta_z_high_f64x8, sum_squared_z_f64x8);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
// Tail: use masked gather for remaining < 16 points
|
|
748
|
+
if (i < n) {
|
|
749
|
+
nk_size_t tail = n - i;
|
|
750
|
+
__mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, tail);
|
|
751
|
+
__m512i const gather_idx_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45);
|
|
752
|
+
__m512 zeros_f32x16 = _mm512_setzero_ps();
|
|
753
|
+
nk_f32_t const *a_tail = a + i * 3;
|
|
754
|
+
nk_f32_t const *b_tail = b + i * 3;
|
|
755
|
+
|
|
756
|
+
a_x_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 0, 4);
|
|
757
|
+
a_y_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 1, 4);
|
|
758
|
+
a_z_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 2, 4);
|
|
759
|
+
b_x_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 0, 4);
|
|
760
|
+
b_y_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 1, 4);
|
|
761
|
+
b_z_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 2, 4);
|
|
762
|
+
|
|
763
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
764
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
765
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
766
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
767
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
768
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
769
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
770
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
771
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
772
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
773
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
774
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
775
|
+
|
|
776
|
+
__m512d delta_x_low_f64x8 = _mm512_sub_pd(a_x_low_f64x8, b_x_low_f64x8);
|
|
777
|
+
__m512d delta_x_high_f64x8 = _mm512_sub_pd(a_x_high_f64x8, b_x_high_f64x8);
|
|
778
|
+
__m512d delta_y_low_f64x8 = _mm512_sub_pd(a_y_low_f64x8, b_y_low_f64x8);
|
|
779
|
+
__m512d delta_y_high_f64x8 = _mm512_sub_pd(a_y_high_f64x8, b_y_high_f64x8);
|
|
780
|
+
__m512d delta_z_low_f64x8 = _mm512_sub_pd(a_z_low_f64x8, b_z_low_f64x8);
|
|
781
|
+
__m512d delta_z_high_f64x8 = _mm512_sub_pd(a_z_high_f64x8, b_z_high_f64x8);
|
|
782
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_low_f64x8, delta_x_low_f64x8, sum_squared_x_f64x8);
|
|
783
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_high_f64x8, delta_x_high_f64x8, sum_squared_x_f64x8);
|
|
784
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_low_f64x8, delta_y_low_f64x8, sum_squared_y_f64x8);
|
|
785
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_high_f64x8, delta_y_high_f64x8, sum_squared_y_f64x8);
|
|
786
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_low_f64x8, delta_z_low_f64x8, sum_squared_z_f64x8);
|
|
787
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_high_f64x8, delta_z_high_f64x8, sum_squared_z_f64x8);
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
nk_f64_t total_sq_x = _mm512_reduce_add_pd(sum_squared_x_f64x8);
|
|
791
|
+
nk_f64_t total_sq_y = _mm512_reduce_add_pd(sum_squared_y_f64x8);
|
|
792
|
+
nk_f64_t total_sq_z = _mm512_reduce_add_pd(sum_squared_z_f64x8);
|
|
793
|
+
*result = nk_f64_sqrt_haswell((total_sq_x + total_sq_y + total_sq_z) / (nk_f64_t)n);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
NK_PUBLIC void nk_kabsch_f32_skylake(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
797
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
|
|
798
|
+
// Fused single-pass: centroids + covariance in f64
|
|
799
|
+
__m512d const zeros_f64x8 = _mm512_setzero_pd();
|
|
800
|
+
__m512d sum_a_x_f64x8 = zeros_f64x8, sum_a_y_f64x8 = zeros_f64x8, sum_a_z_f64x8 = zeros_f64x8;
|
|
801
|
+
__m512d sum_b_x_f64x8 = zeros_f64x8, sum_b_y_f64x8 = zeros_f64x8, sum_b_z_f64x8 = zeros_f64x8;
|
|
802
|
+
__m512d cov_xx_f64x8 = zeros_f64x8, cov_xy_f64x8 = zeros_f64x8, cov_xz_f64x8 = zeros_f64x8;
|
|
803
|
+
__m512d cov_yx_f64x8 = zeros_f64x8, cov_yy_f64x8 = zeros_f64x8, cov_yz_f64x8 = zeros_f64x8;
|
|
804
|
+
__m512d cov_zx_f64x8 = zeros_f64x8, cov_zy_f64x8 = zeros_f64x8, cov_zz_f64x8 = zeros_f64x8;
|
|
805
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
806
|
+
nk_size_t i = 0;
|
|
807
|
+
|
|
808
|
+
for (; i + 16 <= n; i += 16) {
|
|
809
|
+
nk_deinterleave_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
810
|
+
nk_deinterleave_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
811
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
812
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
813
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
814
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
815
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
816
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
817
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
818
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
819
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
820
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
821
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
822
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
823
|
+
|
|
824
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, _mm512_add_pd(a_x_low_f64x8, a_x_high_f64x8));
|
|
825
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, _mm512_add_pd(a_y_low_f64x8, a_y_high_f64x8));
|
|
826
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, _mm512_add_pd(a_z_low_f64x8, a_z_high_f64x8));
|
|
827
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, _mm512_add_pd(b_x_low_f64x8, b_x_high_f64x8));
|
|
828
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, _mm512_add_pd(b_y_low_f64x8, b_y_high_f64x8));
|
|
829
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, _mm512_add_pd(b_z_low_f64x8, b_z_high_f64x8));
|
|
830
|
+
|
|
831
|
+
cov_xx_f64x8 = _mm512_add_pd(cov_xx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_x_low_f64x8),
|
|
832
|
+
_mm512_mul_pd(a_x_high_f64x8, b_x_high_f64x8)));
|
|
833
|
+
cov_xy_f64x8 = _mm512_add_pd(cov_xy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_y_low_f64x8),
|
|
834
|
+
_mm512_mul_pd(a_x_high_f64x8, b_y_high_f64x8)));
|
|
835
|
+
cov_xz_f64x8 = _mm512_add_pd(cov_xz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_z_low_f64x8),
|
|
836
|
+
_mm512_mul_pd(a_x_high_f64x8, b_z_high_f64x8)));
|
|
837
|
+
cov_yx_f64x8 = _mm512_add_pd(cov_yx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_x_low_f64x8),
|
|
838
|
+
_mm512_mul_pd(a_y_high_f64x8, b_x_high_f64x8)));
|
|
839
|
+
cov_yy_f64x8 = _mm512_add_pd(cov_yy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_y_low_f64x8),
|
|
840
|
+
_mm512_mul_pd(a_y_high_f64x8, b_y_high_f64x8)));
|
|
841
|
+
cov_yz_f64x8 = _mm512_add_pd(cov_yz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_z_low_f64x8),
|
|
842
|
+
_mm512_mul_pd(a_y_high_f64x8, b_z_high_f64x8)));
|
|
843
|
+
cov_zx_f64x8 = _mm512_add_pd(cov_zx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_x_low_f64x8),
|
|
844
|
+
_mm512_mul_pd(a_z_high_f64x8, b_x_high_f64x8)));
|
|
845
|
+
cov_zy_f64x8 = _mm512_add_pd(cov_zy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_y_low_f64x8),
|
|
846
|
+
_mm512_mul_pd(a_z_high_f64x8, b_y_high_f64x8)));
|
|
847
|
+
cov_zz_f64x8 = _mm512_add_pd(cov_zz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_z_low_f64x8),
|
|
848
|
+
_mm512_mul_pd(a_z_high_f64x8, b_z_high_f64x8)));
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
// Tail: use masked gather for remaining < 16 points
|
|
852
|
+
if (i < n) {
|
|
853
|
+
nk_size_t tail = n - i;
|
|
854
|
+
__mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, tail);
|
|
855
|
+
__m512i const gather_idx_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45);
|
|
856
|
+
__m512 zeros_f32x16 = _mm512_setzero_ps();
|
|
857
|
+
nk_f32_t const *a_tail = a + i * 3;
|
|
858
|
+
nk_f32_t const *b_tail = b + i * 3;
|
|
859
|
+
|
|
860
|
+
a_x_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 0, 4);
|
|
861
|
+
a_y_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 1, 4);
|
|
862
|
+
a_z_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 2, 4);
|
|
863
|
+
b_x_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 0, 4);
|
|
864
|
+
b_y_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 1, 4);
|
|
865
|
+
b_z_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 2, 4);
|
|
866
|
+
|
|
867
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
868
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
869
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
870
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
871
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
872
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
873
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
874
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
875
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
876
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
877
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
878
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
879
|
+
|
|
880
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, _mm512_add_pd(a_x_low_f64x8, a_x_high_f64x8));
|
|
881
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, _mm512_add_pd(a_y_low_f64x8, a_y_high_f64x8));
|
|
882
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, _mm512_add_pd(a_z_low_f64x8, a_z_high_f64x8));
|
|
883
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, _mm512_add_pd(b_x_low_f64x8, b_x_high_f64x8));
|
|
884
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, _mm512_add_pd(b_y_low_f64x8, b_y_high_f64x8));
|
|
885
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, _mm512_add_pd(b_z_low_f64x8, b_z_high_f64x8));
|
|
886
|
+
|
|
887
|
+
cov_xx_f64x8 = _mm512_add_pd(cov_xx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_x_low_f64x8),
|
|
888
|
+
_mm512_mul_pd(a_x_high_f64x8, b_x_high_f64x8)));
|
|
889
|
+
cov_xy_f64x8 = _mm512_add_pd(cov_xy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_y_low_f64x8),
|
|
890
|
+
_mm512_mul_pd(a_x_high_f64x8, b_y_high_f64x8)));
|
|
891
|
+
cov_xz_f64x8 = _mm512_add_pd(cov_xz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_z_low_f64x8),
|
|
892
|
+
_mm512_mul_pd(a_x_high_f64x8, b_z_high_f64x8)));
|
|
893
|
+
cov_yx_f64x8 = _mm512_add_pd(cov_yx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_x_low_f64x8),
|
|
894
|
+
_mm512_mul_pd(a_y_high_f64x8, b_x_high_f64x8)));
|
|
895
|
+
cov_yy_f64x8 = _mm512_add_pd(cov_yy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_y_low_f64x8),
|
|
896
|
+
_mm512_mul_pd(a_y_high_f64x8, b_y_high_f64x8)));
|
|
897
|
+
cov_yz_f64x8 = _mm512_add_pd(cov_yz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_z_low_f64x8),
|
|
898
|
+
_mm512_mul_pd(a_y_high_f64x8, b_z_high_f64x8)));
|
|
899
|
+
cov_zx_f64x8 = _mm512_add_pd(cov_zx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_x_low_f64x8),
|
|
900
|
+
_mm512_mul_pd(a_z_high_f64x8, b_x_high_f64x8)));
|
|
901
|
+
cov_zy_f64x8 = _mm512_add_pd(cov_zy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_y_low_f64x8),
|
|
902
|
+
_mm512_mul_pd(a_z_high_f64x8, b_y_high_f64x8)));
|
|
903
|
+
cov_zz_f64x8 = _mm512_add_pd(cov_zz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_z_low_f64x8),
|
|
904
|
+
_mm512_mul_pd(a_z_high_f64x8, b_z_high_f64x8)));
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
nk_f64_t sum_a_x = _mm512_reduce_add_pd(sum_a_x_f64x8), sum_a_y = _mm512_reduce_add_pd(sum_a_y_f64x8),
|
|
908
|
+
sum_a_z = _mm512_reduce_add_pd(sum_a_z_f64x8);
|
|
909
|
+
nk_f64_t sum_b_x = _mm512_reduce_add_pd(sum_b_x_f64x8), sum_b_y = _mm512_reduce_add_pd(sum_b_y_f64x8),
|
|
910
|
+
sum_b_z = _mm512_reduce_add_pd(sum_b_z_f64x8);
|
|
911
|
+
nk_f64_t covariance_x_x = _mm512_reduce_add_pd(cov_xx_f64x8), covariance_x_y = _mm512_reduce_add_pd(cov_xy_f64x8),
|
|
912
|
+
covariance_x_z = _mm512_reduce_add_pd(cov_xz_f64x8);
|
|
913
|
+
nk_f64_t covariance_y_x = _mm512_reduce_add_pd(cov_yx_f64x8), covariance_y_y = _mm512_reduce_add_pd(cov_yy_f64x8),
|
|
914
|
+
covariance_y_z = _mm512_reduce_add_pd(cov_yz_f64x8);
|
|
915
|
+
nk_f64_t covariance_z_x = _mm512_reduce_add_pd(cov_zx_f64x8), covariance_z_y = _mm512_reduce_add_pd(cov_zy_f64x8),
|
|
916
|
+
covariance_z_z = _mm512_reduce_add_pd(cov_zz_f64x8);
|
|
917
|
+
|
|
918
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
919
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
920
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
921
|
+
if (a_centroid)
|
|
922
|
+
a_centroid[0] = (nk_f32_t)centroid_a_x, a_centroid[1] = (nk_f32_t)centroid_a_y,
|
|
923
|
+
a_centroid[2] = (nk_f32_t)centroid_a_z;
|
|
924
|
+
if (b_centroid)
|
|
925
|
+
b_centroid[0] = (nk_f32_t)centroid_b_x, b_centroid[1] = (nk_f32_t)centroid_b_y,
|
|
926
|
+
b_centroid[2] = (nk_f32_t)centroid_b_z;
|
|
927
|
+
if (scale) *scale = 1.0f;
|
|
928
|
+
|
|
929
|
+
nk_f64_t n_f64 = (nk_f64_t)n;
|
|
930
|
+
nk_f64_t cross_covariance[9];
|
|
931
|
+
cross_covariance[0] = covariance_x_x - n_f64 * centroid_a_x * centroid_b_x;
|
|
932
|
+
cross_covariance[1] = covariance_x_y - n_f64 * centroid_a_x * centroid_b_y;
|
|
933
|
+
cross_covariance[2] = covariance_x_z - n_f64 * centroid_a_x * centroid_b_z;
|
|
934
|
+
cross_covariance[3] = covariance_y_x - n_f64 * centroid_a_y * centroid_b_x;
|
|
935
|
+
cross_covariance[4] = covariance_y_y - n_f64 * centroid_a_y * centroid_b_y;
|
|
936
|
+
cross_covariance[5] = covariance_y_z - n_f64 * centroid_a_y * centroid_b_z;
|
|
937
|
+
cross_covariance[6] = covariance_z_x - n_f64 * centroid_a_z * centroid_b_x;
|
|
938
|
+
cross_covariance[7] = covariance_z_y - n_f64 * centroid_a_z * centroid_b_y;
|
|
939
|
+
cross_covariance[8] = covariance_z_z - n_f64 * centroid_a_z * centroid_b_z;
|
|
940
|
+
|
|
941
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
942
|
+
nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
|
|
943
|
+
nk_f64_t r[9];
|
|
944
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
945
|
+
if (nk_det3x3_f64_(r) < 0) {
|
|
946
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
947
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
948
|
+
}
|
|
949
|
+
if (rotation)
|
|
950
|
+
for (int j = 0; j < 9; ++j) rotation[j] = (nk_f32_t)r[j];
|
|
951
|
+
*result = nk_f64_sqrt_haswell(nk_transformed_ssd_f32_skylake_(a, b, n, r, 1.0, centroid_a_x, centroid_a_y,
|
|
952
|
+
centroid_a_z, centroid_b_x, centroid_b_y,
|
|
953
|
+
centroid_b_z) /
|
|
954
|
+
n_f64);
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
NK_PUBLIC void nk_rmsd_f64_skylake(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
|
|
958
|
+
nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
|
|
959
|
+
if (rotation)
|
|
960
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
961
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
962
|
+
if (scale) *scale = 1.0;
|
|
963
|
+
if (a_centroid) a_centroid[0] = 0, a_centroid[1] = 0, a_centroid[2] = 0;
|
|
964
|
+
if (b_centroid) b_centroid[0] = 0, b_centroid[1] = 0, b_centroid[2] = 0;
|
|
965
|
+
|
|
966
|
+
__m512i const gather_idx_i64x8 = _mm512_setr_epi64(0, 3, 6, 9, 12, 15, 18, 21);
|
|
967
|
+
__m512d const zeros_f64x8 = _mm512_setzero_pd();
|
|
968
|
+
__m512d sum_squared_x_f64x8 = zeros_f64x8, sum_squared_y_f64x8 = zeros_f64x8, sum_squared_z_f64x8 = zeros_f64x8;
|
|
969
|
+
|
|
970
|
+
__m512d a_x_f64x8, a_y_f64x8, a_z_f64x8, b_x_f64x8, b_y_f64x8, b_z_f64x8;
|
|
971
|
+
nk_size_t i = 0;
|
|
972
|
+
|
|
973
|
+
// Main loop with 2x unrolling for better latency hiding
|
|
974
|
+
for (; i + 16 <= n; i += 16) {
|
|
975
|
+
// Iteration 0
|
|
976
|
+
nk_deinterleave_f64x8_skylake_(a + i * 3, &a_x_f64x8, &a_y_f64x8, &a_z_f64x8);
|
|
977
|
+
nk_deinterleave_f64x8_skylake_(b + i * 3, &b_x_f64x8, &b_y_f64x8, &b_z_f64x8);
|
|
978
|
+
|
|
979
|
+
__m512d delta_x_f64x8 = _mm512_sub_pd(a_x_f64x8, b_x_f64x8),
|
|
980
|
+
delta_y_f64x8 = _mm512_sub_pd(a_y_f64x8, b_y_f64x8),
|
|
981
|
+
delta_z_f64x8 = _mm512_sub_pd(a_z_f64x8, b_z_f64x8);
|
|
982
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_f64x8, delta_x_f64x8, sum_squared_x_f64x8);
|
|
983
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_f64x8, delta_y_f64x8, sum_squared_y_f64x8);
|
|
984
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_f64x8, delta_z_f64x8, sum_squared_z_f64x8);
|
|
985
|
+
|
|
986
|
+
// Iteration 1
|
|
987
|
+
__m512d a_x1_f64x8, a_y1_f64x8, a_z1_f64x8, b_x1_f64x8, b_y1_f64x8, b_z1_f64x8;
|
|
988
|
+
nk_deinterleave_f64x8_skylake_(a + (i + 8) * 3, &a_x1_f64x8, &a_y1_f64x8, &a_z1_f64x8);
|
|
989
|
+
nk_deinterleave_f64x8_skylake_(b + (i + 8) * 3, &b_x1_f64x8, &b_y1_f64x8, &b_z1_f64x8);
|
|
990
|
+
|
|
991
|
+
__m512d delta_x1_f64x8 = _mm512_sub_pd(a_x1_f64x8, b_x1_f64x8),
|
|
992
|
+
delta_y1_f64x8 = _mm512_sub_pd(a_y1_f64x8, b_y1_f64x8),
|
|
993
|
+
delta_z1_f64x8 = _mm512_sub_pd(a_z1_f64x8, b_z1_f64x8);
|
|
994
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x1_f64x8, delta_x1_f64x8, sum_squared_x_f64x8);
|
|
995
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y1_f64x8, delta_y1_f64x8, sum_squared_y_f64x8);
|
|
996
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z1_f64x8, delta_z1_f64x8, sum_squared_z_f64x8);
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
// Handle 8-point remainder
|
|
1000
|
+
for (; i + 8 <= n; i += 8) {
|
|
1001
|
+
nk_deinterleave_f64x8_skylake_(a + i * 3, &a_x_f64x8, &a_y_f64x8, &a_z_f64x8);
|
|
1002
|
+
nk_deinterleave_f64x8_skylake_(b + i * 3, &b_x_f64x8, &b_y_f64x8, &b_z_f64x8);
|
|
1003
|
+
|
|
1004
|
+
__m512d delta_x_f64x8 = _mm512_sub_pd(a_x_f64x8, b_x_f64x8),
|
|
1005
|
+
delta_y_f64x8 = _mm512_sub_pd(a_y_f64x8, b_y_f64x8),
|
|
1006
|
+
delta_z_f64x8 = _mm512_sub_pd(a_z_f64x8, b_z_f64x8);
|
|
1007
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_f64x8, delta_x_f64x8, sum_squared_x_f64x8);
|
|
1008
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_f64x8, delta_y_f64x8, sum_squared_y_f64x8);
|
|
1009
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_f64x8, delta_z_f64x8, sum_squared_z_f64x8);
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
// Tail: use masked gather
|
|
1013
|
+
if (i < n) {
|
|
1014
|
+
nk_size_t tail = n - i;
|
|
1015
|
+
__mmask8 mask = (__mmask8)_bzhi_u32(0xFF, tail);
|
|
1016
|
+
nk_f64_t const *a_tail = a + i * 3;
|
|
1017
|
+
nk_f64_t const *b_tail = b + i * 3;
|
|
1018
|
+
|
|
1019
|
+
a_x_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 0, 8);
|
|
1020
|
+
a_y_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 1, 8);
|
|
1021
|
+
a_z_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 2, 8);
|
|
1022
|
+
b_x_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 0, 8);
|
|
1023
|
+
b_y_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 1, 8);
|
|
1024
|
+
b_z_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 2, 8);
|
|
1025
|
+
|
|
1026
|
+
__m512d delta_x_f64x8 = _mm512_sub_pd(a_x_f64x8, b_x_f64x8),
|
|
1027
|
+
delta_y_f64x8 = _mm512_sub_pd(a_y_f64x8, b_y_f64x8),
|
|
1028
|
+
delta_z_f64x8 = _mm512_sub_pd(a_z_f64x8, b_z_f64x8);
|
|
1029
|
+
sum_squared_x_f64x8 = _mm512_fmadd_pd(delta_x_f64x8, delta_x_f64x8, sum_squared_x_f64x8);
|
|
1030
|
+
sum_squared_y_f64x8 = _mm512_fmadd_pd(delta_y_f64x8, delta_y_f64x8, sum_squared_y_f64x8);
|
|
1031
|
+
sum_squared_z_f64x8 = _mm512_fmadd_pd(delta_z_f64x8, delta_z_f64x8, sum_squared_z_f64x8);
|
|
1032
|
+
i = n;
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
nk_f64_t total_squared_x = nk_reduce_stable_f64x8_skylake_(sum_squared_x_f64x8), total_squared_x_compensation = 0.0;
|
|
1036
|
+
nk_f64_t total_squared_y = nk_reduce_stable_f64x8_skylake_(sum_squared_y_f64x8), total_squared_y_compensation = 0.0;
|
|
1037
|
+
nk_f64_t total_squared_z = nk_reduce_stable_f64x8_skylake_(sum_squared_z_f64x8), total_squared_z_compensation = 0.0;
|
|
1038
|
+
|
|
1039
|
+
for (; i < n; ++i) {
|
|
1040
|
+
nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
|
|
1041
|
+
nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
|
|
1042
|
+
nk_f64_t delta_x = ax - bx, delta_y = ay - by, delta_z = az - bz;
|
|
1043
|
+
nk_accumulate_square_f64_(&total_squared_x, &total_squared_x_compensation, delta_x);
|
|
1044
|
+
nk_accumulate_square_f64_(&total_squared_y, &total_squared_y_compensation, delta_y);
|
|
1045
|
+
nk_accumulate_square_f64_(&total_squared_z, &total_squared_z_compensation, delta_z);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
total_squared_x += total_squared_x_compensation, total_squared_y += total_squared_y_compensation,
|
|
1049
|
+
total_squared_z += total_squared_z_compensation;
|
|
1050
|
+
*result = nk_f64_sqrt_haswell((total_squared_x + total_squared_y + total_squared_z) / (nk_f64_t)n);
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
NK_PUBLIC void nk_kabsch_f64_skylake(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
|
|
1054
|
+
nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
|
|
1055
|
+
// Optimized fused single-pass implementation for f64.
|
|
1056
|
+
// Computes centroids and covariance matrix in one pass using the identity:
|
|
1057
|
+
// Hᵢⱼ = Σ((aᵢ - ā) × (bⱼ - b̄))
|
|
1058
|
+
// = Σ(aᵢ × bⱼ) - Σaᵢ × Σbⱼ / n
|
|
1059
|
+
__m512i const gather_idx_i64x8 = _mm512_setr_epi64(0, 3, 6, 9, 12, 15, 18, 21);
|
|
1060
|
+
__m512d const zeros_f64x8 = _mm512_setzero_pd();
|
|
1061
|
+
|
|
1062
|
+
// Accumulators for centroids
|
|
1063
|
+
__m512d sum_a_x_f64x8 = zeros_f64x8, sum_a_y_f64x8 = zeros_f64x8, sum_a_z_f64x8 = zeros_f64x8;
|
|
1064
|
+
__m512d sum_b_x_f64x8 = zeros_f64x8, sum_b_y_f64x8 = zeros_f64x8, sum_b_z_f64x8 = zeros_f64x8;
|
|
1065
|
+
|
|
1066
|
+
// Accumulators for covariance matrix (sum of outer products)
|
|
1067
|
+
__m512d cov_xx_f64x8 = zeros_f64x8, cov_xy_f64x8 = zeros_f64x8, cov_xz_f64x8 = zeros_f64x8;
|
|
1068
|
+
__m512d cov_yx_f64x8 = zeros_f64x8, cov_yy_f64x8 = zeros_f64x8, cov_yz_f64x8 = zeros_f64x8;
|
|
1069
|
+
__m512d cov_zx_f64x8 = zeros_f64x8, cov_zy_f64x8 = zeros_f64x8, cov_zz_f64x8 = zeros_f64x8;
|
|
1070
|
+
|
|
1071
|
+
nk_size_t i = 0;
|
|
1072
|
+
__m512d a_x_f64x8, a_y_f64x8, a_z_f64x8, b_x_f64x8, b_y_f64x8, b_z_f64x8;
|
|
1073
|
+
|
|
1074
|
+
// Fused single-pass: accumulate sums and outer products together
|
|
1075
|
+
for (; i + 8 <= n; i += 8) {
|
|
1076
|
+
nk_deinterleave_f64x8_skylake_(a + i * 3, &a_x_f64x8, &a_y_f64x8, &a_z_f64x8);
|
|
1077
|
+
nk_deinterleave_f64x8_skylake_(b + i * 3, &b_x_f64x8, &b_y_f64x8, &b_z_f64x8);
|
|
1078
|
+
|
|
1079
|
+
// Accumulate centroids
|
|
1080
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, a_x_f64x8),
|
|
1081
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, a_y_f64x8),
|
|
1082
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, a_z_f64x8);
|
|
1083
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, b_x_f64x8),
|
|
1084
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, b_y_f64x8),
|
|
1085
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, b_z_f64x8);
|
|
1086
|
+
|
|
1087
|
+
// Accumulate outer products (raw, not centered)
|
|
1088
|
+
cov_xx_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_x_f64x8, cov_xx_f64x8),
|
|
1089
|
+
cov_xy_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_y_f64x8, cov_xy_f64x8),
|
|
1090
|
+
cov_xz_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_z_f64x8, cov_xz_f64x8);
|
|
1091
|
+
cov_yx_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_x_f64x8, cov_yx_f64x8),
|
|
1092
|
+
cov_yy_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_y_f64x8, cov_yy_f64x8),
|
|
1093
|
+
cov_yz_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_z_f64x8, cov_yz_f64x8);
|
|
1094
|
+
cov_zx_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_x_f64x8, cov_zx_f64x8),
|
|
1095
|
+
cov_zy_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_y_f64x8, cov_zy_f64x8),
|
|
1096
|
+
cov_zz_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_z_f64x8, cov_zz_f64x8);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
// Tail: masked gather for remaining points
|
|
1100
|
+
if (i < n) {
|
|
1101
|
+
nk_size_t tail = n - i;
|
|
1102
|
+
__mmask8 mask = (__mmask8)_bzhi_u32(0xFF, tail);
|
|
1103
|
+
nk_f64_t const *a_tail = a + i * 3;
|
|
1104
|
+
nk_f64_t const *b_tail = b + i * 3;
|
|
1105
|
+
|
|
1106
|
+
a_x_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 0, 8);
|
|
1107
|
+
a_y_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 1, 8);
|
|
1108
|
+
a_z_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 2, 8);
|
|
1109
|
+
b_x_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 0, 8);
|
|
1110
|
+
b_y_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 1, 8);
|
|
1111
|
+
b_z_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 2, 8);
|
|
1112
|
+
|
|
1113
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, a_x_f64x8),
|
|
1114
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, a_y_f64x8),
|
|
1115
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, a_z_f64x8);
|
|
1116
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, b_x_f64x8),
|
|
1117
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, b_y_f64x8),
|
|
1118
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, b_z_f64x8);
|
|
1119
|
+
|
|
1120
|
+
cov_xx_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_x_f64x8, cov_xx_f64x8),
|
|
1121
|
+
cov_xy_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_y_f64x8, cov_xy_f64x8),
|
|
1122
|
+
cov_xz_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_z_f64x8, cov_xz_f64x8);
|
|
1123
|
+
cov_yx_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_x_f64x8, cov_yx_f64x8),
|
|
1124
|
+
cov_yy_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_y_f64x8, cov_yy_f64x8),
|
|
1125
|
+
cov_yz_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_z_f64x8, cov_yz_f64x8);
|
|
1126
|
+
cov_zx_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_x_f64x8, cov_zx_f64x8),
|
|
1127
|
+
cov_zy_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_y_f64x8, cov_zy_f64x8),
|
|
1128
|
+
cov_zz_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_z_f64x8, cov_zz_f64x8);
|
|
1129
|
+
i = n;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
// Reduce centroids and covariance.
|
|
1133
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
1134
|
+
nk_f64_t sum_a_x = nk_reduce_stable_f64x8_skylake_(sum_a_x_f64x8), sum_a_x_compensation = 0.0;
|
|
1135
|
+
nk_f64_t sum_a_y = nk_reduce_stable_f64x8_skylake_(sum_a_y_f64x8), sum_a_y_compensation = 0.0;
|
|
1136
|
+
nk_f64_t sum_a_z = nk_reduce_stable_f64x8_skylake_(sum_a_z_f64x8), sum_a_z_compensation = 0.0;
|
|
1137
|
+
nk_f64_t sum_b_x = nk_reduce_stable_f64x8_skylake_(sum_b_x_f64x8), sum_b_x_compensation = 0.0;
|
|
1138
|
+
nk_f64_t sum_b_y = nk_reduce_stable_f64x8_skylake_(sum_b_y_f64x8), sum_b_y_compensation = 0.0;
|
|
1139
|
+
nk_f64_t sum_b_z = nk_reduce_stable_f64x8_skylake_(sum_b_z_f64x8), sum_b_z_compensation = 0.0;
|
|
1140
|
+
nk_f64_t covariance_x_x = nk_reduce_stable_f64x8_skylake_(cov_xx_f64x8), covariance_x_x_compensation = 0.0;
|
|
1141
|
+
nk_f64_t covariance_x_y = nk_reduce_stable_f64x8_skylake_(cov_xy_f64x8), covariance_x_y_compensation = 0.0;
|
|
1142
|
+
nk_f64_t covariance_x_z = nk_reduce_stable_f64x8_skylake_(cov_xz_f64x8), covariance_x_z_compensation = 0.0;
|
|
1143
|
+
nk_f64_t covariance_y_x = nk_reduce_stable_f64x8_skylake_(cov_yx_f64x8), covariance_y_x_compensation = 0.0;
|
|
1144
|
+
nk_f64_t covariance_y_y = nk_reduce_stable_f64x8_skylake_(cov_yy_f64x8), covariance_y_y_compensation = 0.0;
|
|
1145
|
+
nk_f64_t covariance_y_z = nk_reduce_stable_f64x8_skylake_(cov_yz_f64x8), covariance_y_z_compensation = 0.0;
|
|
1146
|
+
nk_f64_t covariance_z_x = nk_reduce_stable_f64x8_skylake_(cov_zx_f64x8), covariance_z_x_compensation = 0.0;
|
|
1147
|
+
nk_f64_t covariance_z_y = nk_reduce_stable_f64x8_skylake_(cov_zy_f64x8), covariance_z_y_compensation = 0.0;
|
|
1148
|
+
nk_f64_t covariance_z_z = nk_reduce_stable_f64x8_skylake_(cov_zz_f64x8), covariance_z_z_compensation = 0.0;
|
|
1149
|
+
|
|
1150
|
+
for (; i < n; ++i) {
|
|
1151
|
+
nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
|
|
1152
|
+
nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
|
|
1153
|
+
nk_accumulate_sum_f64_(&sum_a_x, &sum_a_x_compensation, ax);
|
|
1154
|
+
nk_accumulate_sum_f64_(&sum_a_y, &sum_a_y_compensation, ay);
|
|
1155
|
+
nk_accumulate_sum_f64_(&sum_a_z, &sum_a_z_compensation, az);
|
|
1156
|
+
nk_accumulate_sum_f64_(&sum_b_x, &sum_b_x_compensation, bx);
|
|
1157
|
+
nk_accumulate_sum_f64_(&sum_b_y, &sum_b_y_compensation, by);
|
|
1158
|
+
nk_accumulate_sum_f64_(&sum_b_z, &sum_b_z_compensation, bz);
|
|
1159
|
+
nk_accumulate_product_f64_(&covariance_x_x, &covariance_x_x_compensation, ax, bx);
|
|
1160
|
+
nk_accumulate_product_f64_(&covariance_x_y, &covariance_x_y_compensation, ax, by);
|
|
1161
|
+
nk_accumulate_product_f64_(&covariance_x_z, &covariance_x_z_compensation, ax, bz);
|
|
1162
|
+
nk_accumulate_product_f64_(&covariance_y_x, &covariance_y_x_compensation, ay, bx);
|
|
1163
|
+
nk_accumulate_product_f64_(&covariance_y_y, &covariance_y_y_compensation, ay, by);
|
|
1164
|
+
nk_accumulate_product_f64_(&covariance_y_z, &covariance_y_z_compensation, ay, bz);
|
|
1165
|
+
nk_accumulate_product_f64_(&covariance_z_x, &covariance_z_x_compensation, az, bx);
|
|
1166
|
+
nk_accumulate_product_f64_(&covariance_z_y, &covariance_z_y_compensation, az, by);
|
|
1167
|
+
nk_accumulate_product_f64_(&covariance_z_z, &covariance_z_z_compensation, az, bz);
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
sum_a_x += sum_a_x_compensation, sum_a_y += sum_a_y_compensation, sum_a_z += sum_a_z_compensation;
|
|
1171
|
+
sum_b_x += sum_b_x_compensation, sum_b_y += sum_b_y_compensation, sum_b_z += sum_b_z_compensation;
|
|
1172
|
+
covariance_x_x += covariance_x_x_compensation, covariance_x_y += covariance_x_y_compensation,
|
|
1173
|
+
covariance_x_z += covariance_x_z_compensation;
|
|
1174
|
+
covariance_y_x += covariance_y_x_compensation, covariance_y_y += covariance_y_y_compensation,
|
|
1175
|
+
covariance_y_z += covariance_y_z_compensation;
|
|
1176
|
+
covariance_z_x += covariance_z_x_compensation, covariance_z_y += covariance_z_y_compensation,
|
|
1177
|
+
covariance_z_z += covariance_z_z_compensation;
|
|
1178
|
+
|
|
1179
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1180
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1181
|
+
|
|
1182
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1183
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1184
|
+
|
|
1185
|
+
// Compute centered covariance matrix: Hᵢⱼ = Σ(aᵢ×bⱼ) - Σaᵢ × Σbⱼ / n.
|
|
1186
|
+
nk_f64_t cross_covariance[9];
|
|
1187
|
+
cross_covariance[0] = covariance_x_x - sum_a_x * sum_b_x * inv_n;
|
|
1188
|
+
cross_covariance[1] = covariance_x_y - sum_a_x * sum_b_y * inv_n;
|
|
1189
|
+
cross_covariance[2] = covariance_x_z - sum_a_x * sum_b_z * inv_n;
|
|
1190
|
+
cross_covariance[3] = covariance_y_x - sum_a_y * sum_b_x * inv_n;
|
|
1191
|
+
cross_covariance[4] = covariance_y_y - sum_a_y * sum_b_y * inv_n;
|
|
1192
|
+
cross_covariance[5] = covariance_y_z - sum_a_y * sum_b_z * inv_n;
|
|
1193
|
+
cross_covariance[6] = covariance_z_x - sum_a_z * sum_b_x * inv_n;
|
|
1194
|
+
cross_covariance[7] = covariance_z_y - sum_a_z * sum_b_y * inv_n;
|
|
1195
|
+
cross_covariance[8] = covariance_z_z - sum_a_z * sum_b_z * inv_n;
|
|
1196
|
+
|
|
1197
|
+
// SVD using f64 for full precision
|
|
1198
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
1199
|
+
nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
|
|
1200
|
+
|
|
1201
|
+
nk_f64_t r[9];
|
|
1202
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
1203
|
+
|
|
1204
|
+
// Handle reflection
|
|
1205
|
+
if (nk_det3x3_f64_(r) < 0) {
|
|
1206
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1207
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
// Output rotation matrix and scale=1.0.
|
|
1211
|
+
if (rotation)
|
|
1212
|
+
for (int j = 0; j < 9; ++j) rotation[j] = (nk_f64_t)r[j];
|
|
1213
|
+
if (scale) *scale = 1.0;
|
|
1214
|
+
|
|
1215
|
+
// Compute RMSD after optimal rotation
|
|
1216
|
+
nk_f64_t sum_squared = nk_transformed_ssd_f64_skylake_(a, b, n, r, 1.0, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
1217
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
1218
|
+
*result = nk_f64_sqrt_haswell(sum_squared * inv_n);
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
NK_PUBLIC void nk_umeyama_f32_skylake(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1222
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
|
|
1223
|
+
// Fused single-pass: centroids + covariance + variance of A, all in f64
|
|
1224
|
+
__m512d const zeros_f64x8 = _mm512_setzero_pd();
|
|
1225
|
+
__m512d sum_a_x_f64x8 = zeros_f64x8, sum_a_y_f64x8 = zeros_f64x8, sum_a_z_f64x8 = zeros_f64x8;
|
|
1226
|
+
__m512d sum_b_x_f64x8 = zeros_f64x8, sum_b_y_f64x8 = zeros_f64x8, sum_b_z_f64x8 = zeros_f64x8;
|
|
1227
|
+
__m512d cov_xx_f64x8 = zeros_f64x8, cov_xy_f64x8 = zeros_f64x8, cov_xz_f64x8 = zeros_f64x8;
|
|
1228
|
+
__m512d cov_yx_f64x8 = zeros_f64x8, cov_yy_f64x8 = zeros_f64x8, cov_yz_f64x8 = zeros_f64x8;
|
|
1229
|
+
__m512d cov_zx_f64x8 = zeros_f64x8, cov_zy_f64x8 = zeros_f64x8, cov_zz_f64x8 = zeros_f64x8;
|
|
1230
|
+
__m512d variance_a_f64x8 = zeros_f64x8;
|
|
1231
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
1232
|
+
nk_size_t i = 0;
|
|
1233
|
+
|
|
1234
|
+
for (; i + 16 <= n; i += 16) {
|
|
1235
|
+
nk_deinterleave_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1236
|
+
nk_deinterleave_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1237
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
1238
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
1239
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
1240
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
1241
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
1242
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
1243
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
1244
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
1245
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
1246
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
1247
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
1248
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
1249
|
+
|
|
1250
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, _mm512_add_pd(a_x_low_f64x8, a_x_high_f64x8));
|
|
1251
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, _mm512_add_pd(a_y_low_f64x8, a_y_high_f64x8));
|
|
1252
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, _mm512_add_pd(a_z_low_f64x8, a_z_high_f64x8));
|
|
1253
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, _mm512_add_pd(b_x_low_f64x8, b_x_high_f64x8));
|
|
1254
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, _mm512_add_pd(b_y_low_f64x8, b_y_high_f64x8));
|
|
1255
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, _mm512_add_pd(b_z_low_f64x8, b_z_high_f64x8));
|
|
1256
|
+
|
|
1257
|
+
cov_xx_f64x8 = _mm512_add_pd(cov_xx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_x_low_f64x8),
|
|
1258
|
+
_mm512_mul_pd(a_x_high_f64x8, b_x_high_f64x8)));
|
|
1259
|
+
cov_xy_f64x8 = _mm512_add_pd(cov_xy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_y_low_f64x8),
|
|
1260
|
+
_mm512_mul_pd(a_x_high_f64x8, b_y_high_f64x8)));
|
|
1261
|
+
cov_xz_f64x8 = _mm512_add_pd(cov_xz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_z_low_f64x8),
|
|
1262
|
+
_mm512_mul_pd(a_x_high_f64x8, b_z_high_f64x8)));
|
|
1263
|
+
cov_yx_f64x8 = _mm512_add_pd(cov_yx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_x_low_f64x8),
|
|
1264
|
+
_mm512_mul_pd(a_y_high_f64x8, b_x_high_f64x8)));
|
|
1265
|
+
cov_yy_f64x8 = _mm512_add_pd(cov_yy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_y_low_f64x8),
|
|
1266
|
+
_mm512_mul_pd(a_y_high_f64x8, b_y_high_f64x8)));
|
|
1267
|
+
cov_yz_f64x8 = _mm512_add_pd(cov_yz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_z_low_f64x8),
|
|
1268
|
+
_mm512_mul_pd(a_y_high_f64x8, b_z_high_f64x8)));
|
|
1269
|
+
cov_zx_f64x8 = _mm512_add_pd(cov_zx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_x_low_f64x8),
|
|
1270
|
+
_mm512_mul_pd(a_z_high_f64x8, b_x_high_f64x8)));
|
|
1271
|
+
cov_zy_f64x8 = _mm512_add_pd(cov_zy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_y_low_f64x8),
|
|
1272
|
+
_mm512_mul_pd(a_z_high_f64x8, b_y_high_f64x8)));
|
|
1273
|
+
cov_zz_f64x8 = _mm512_add_pd(cov_zz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_z_low_f64x8),
|
|
1274
|
+
_mm512_mul_pd(a_z_high_f64x8, b_z_high_f64x8)));
|
|
1275
|
+
|
|
1276
|
+
variance_a_f64x8 = _mm512_add_pd(
|
|
1277
|
+
variance_a_f64x8,
|
|
1278
|
+
_mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, a_x_low_f64x8), _mm512_mul_pd(a_x_high_f64x8, a_x_high_f64x8)));
|
|
1279
|
+
variance_a_f64x8 = _mm512_add_pd(
|
|
1280
|
+
variance_a_f64x8,
|
|
1281
|
+
_mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, a_y_low_f64x8), _mm512_mul_pd(a_y_high_f64x8, a_y_high_f64x8)));
|
|
1282
|
+
variance_a_f64x8 = _mm512_add_pd(
|
|
1283
|
+
variance_a_f64x8,
|
|
1284
|
+
_mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, a_z_low_f64x8), _mm512_mul_pd(a_z_high_f64x8, a_z_high_f64x8)));
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
// Tail: use masked gather for remaining < 16 points
|
|
1288
|
+
if (i < n) {
|
|
1289
|
+
nk_size_t tail = n - i;
|
|
1290
|
+
__mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, tail);
|
|
1291
|
+
__m512i const gather_idx_i32x16 = _mm512_setr_epi32(0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45);
|
|
1292
|
+
__m512 zeros_f32x16 = _mm512_setzero_ps();
|
|
1293
|
+
nk_f32_t const *a_tail = a + i * 3;
|
|
1294
|
+
nk_f32_t const *b_tail = b + i * 3;
|
|
1295
|
+
|
|
1296
|
+
a_x_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 0, 4);
|
|
1297
|
+
a_y_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 1, 4);
|
|
1298
|
+
a_z_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, a_tail + 2, 4);
|
|
1299
|
+
b_x_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 0, 4);
|
|
1300
|
+
b_y_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 1, 4);
|
|
1301
|
+
b_z_f32x16 = _mm512_mask_i32gather_ps(zeros_f32x16, mask, gather_idx_i32x16, b_tail + 2, 4);
|
|
1302
|
+
|
|
1303
|
+
__m512d a_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_x_f32x16));
|
|
1304
|
+
__m512d a_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_x_f32x16, 1));
|
|
1305
|
+
__m512d a_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_y_f32x16));
|
|
1306
|
+
__m512d a_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_y_f32x16, 1));
|
|
1307
|
+
__m512d a_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(a_z_f32x16));
|
|
1308
|
+
__m512d a_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(a_z_f32x16, 1));
|
|
1309
|
+
__m512d b_x_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_x_f32x16));
|
|
1310
|
+
__m512d b_x_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_x_f32x16, 1));
|
|
1311
|
+
__m512d b_y_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_y_f32x16));
|
|
1312
|
+
__m512d b_y_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_y_f32x16, 1));
|
|
1313
|
+
__m512d b_z_low_f64x8 = _mm512_cvtps_pd(_mm512_castps512_ps256(b_z_f32x16));
|
|
1314
|
+
__m512d b_z_high_f64x8 = _mm512_cvtps_pd(_mm512_extractf32x8_ps(b_z_f32x16, 1));
|
|
1315
|
+
|
|
1316
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, _mm512_add_pd(a_x_low_f64x8, a_x_high_f64x8));
|
|
1317
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, _mm512_add_pd(a_y_low_f64x8, a_y_high_f64x8));
|
|
1318
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, _mm512_add_pd(a_z_low_f64x8, a_z_high_f64x8));
|
|
1319
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, _mm512_add_pd(b_x_low_f64x8, b_x_high_f64x8));
|
|
1320
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, _mm512_add_pd(b_y_low_f64x8, b_y_high_f64x8));
|
|
1321
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, _mm512_add_pd(b_z_low_f64x8, b_z_high_f64x8));
|
|
1322
|
+
|
|
1323
|
+
cov_xx_f64x8 = _mm512_add_pd(cov_xx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_x_low_f64x8),
|
|
1324
|
+
_mm512_mul_pd(a_x_high_f64x8, b_x_high_f64x8)));
|
|
1325
|
+
cov_xy_f64x8 = _mm512_add_pd(cov_xy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_y_low_f64x8),
|
|
1326
|
+
_mm512_mul_pd(a_x_high_f64x8, b_y_high_f64x8)));
|
|
1327
|
+
cov_xz_f64x8 = _mm512_add_pd(cov_xz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, b_z_low_f64x8),
|
|
1328
|
+
_mm512_mul_pd(a_x_high_f64x8, b_z_high_f64x8)));
|
|
1329
|
+
cov_yx_f64x8 = _mm512_add_pd(cov_yx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_x_low_f64x8),
|
|
1330
|
+
_mm512_mul_pd(a_y_high_f64x8, b_x_high_f64x8)));
|
|
1331
|
+
cov_yy_f64x8 = _mm512_add_pd(cov_yy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_y_low_f64x8),
|
|
1332
|
+
_mm512_mul_pd(a_y_high_f64x8, b_y_high_f64x8)));
|
|
1333
|
+
cov_yz_f64x8 = _mm512_add_pd(cov_yz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, b_z_low_f64x8),
|
|
1334
|
+
_mm512_mul_pd(a_y_high_f64x8, b_z_high_f64x8)));
|
|
1335
|
+
cov_zx_f64x8 = _mm512_add_pd(cov_zx_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_x_low_f64x8),
|
|
1336
|
+
_mm512_mul_pd(a_z_high_f64x8, b_x_high_f64x8)));
|
|
1337
|
+
cov_zy_f64x8 = _mm512_add_pd(cov_zy_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_y_low_f64x8),
|
|
1338
|
+
_mm512_mul_pd(a_z_high_f64x8, b_y_high_f64x8)));
|
|
1339
|
+
cov_zz_f64x8 = _mm512_add_pd(cov_zz_f64x8, _mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, b_z_low_f64x8),
|
|
1340
|
+
_mm512_mul_pd(a_z_high_f64x8, b_z_high_f64x8)));
|
|
1341
|
+
|
|
1342
|
+
variance_a_f64x8 = _mm512_add_pd(
|
|
1343
|
+
variance_a_f64x8,
|
|
1344
|
+
_mm512_add_pd(_mm512_mul_pd(a_x_low_f64x8, a_x_low_f64x8), _mm512_mul_pd(a_x_high_f64x8, a_x_high_f64x8)));
|
|
1345
|
+
variance_a_f64x8 = _mm512_add_pd(
|
|
1346
|
+
variance_a_f64x8,
|
|
1347
|
+
_mm512_add_pd(_mm512_mul_pd(a_y_low_f64x8, a_y_low_f64x8), _mm512_mul_pd(a_y_high_f64x8, a_y_high_f64x8)));
|
|
1348
|
+
variance_a_f64x8 = _mm512_add_pd(
|
|
1349
|
+
variance_a_f64x8,
|
|
1350
|
+
_mm512_add_pd(_mm512_mul_pd(a_z_low_f64x8, a_z_low_f64x8), _mm512_mul_pd(a_z_high_f64x8, a_z_high_f64x8)));
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
nk_f64_t sum_a_x = _mm512_reduce_add_pd(sum_a_x_f64x8), sum_a_y = _mm512_reduce_add_pd(sum_a_y_f64x8),
|
|
1354
|
+
sum_a_z = _mm512_reduce_add_pd(sum_a_z_f64x8);
|
|
1355
|
+
nk_f64_t sum_b_x = _mm512_reduce_add_pd(sum_b_x_f64x8), sum_b_y = _mm512_reduce_add_pd(sum_b_y_f64x8),
|
|
1356
|
+
sum_b_z = _mm512_reduce_add_pd(sum_b_z_f64x8);
|
|
1357
|
+
nk_f64_t covariance_x_x = _mm512_reduce_add_pd(cov_xx_f64x8), covariance_x_y = _mm512_reduce_add_pd(cov_xy_f64x8),
|
|
1358
|
+
covariance_x_z = _mm512_reduce_add_pd(cov_xz_f64x8);
|
|
1359
|
+
nk_f64_t covariance_y_x = _mm512_reduce_add_pd(cov_yx_f64x8), covariance_y_y = _mm512_reduce_add_pd(cov_yy_f64x8),
|
|
1360
|
+
covariance_y_z = _mm512_reduce_add_pd(cov_yz_f64x8);
|
|
1361
|
+
nk_f64_t covariance_z_x = _mm512_reduce_add_pd(cov_zx_f64x8), covariance_z_y = _mm512_reduce_add_pd(cov_zy_f64x8),
|
|
1362
|
+
covariance_z_z = _mm512_reduce_add_pd(cov_zz_f64x8);
|
|
1363
|
+
nk_f64_t variance_a_sum = _mm512_reduce_add_pd(variance_a_f64x8);
|
|
1364
|
+
|
|
1365
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
1366
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1367
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1368
|
+
if (a_centroid)
|
|
1369
|
+
a_centroid[0] = (nk_f32_t)centroid_a_x, a_centroid[1] = (nk_f32_t)centroid_a_y,
|
|
1370
|
+
a_centroid[2] = (nk_f32_t)centroid_a_z;
|
|
1371
|
+
if (b_centroid)
|
|
1372
|
+
b_centroid[0] = (nk_f32_t)centroid_b_x, b_centroid[1] = (nk_f32_t)centroid_b_y,
|
|
1373
|
+
b_centroid[2] = (nk_f32_t)centroid_b_z;
|
|
1374
|
+
|
|
1375
|
+
// Compute centered covariance and variance
|
|
1376
|
+
nk_f64_t variance_a = variance_a_sum * inv_n -
|
|
1377
|
+
(centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
|
|
1378
|
+
|
|
1379
|
+
// Compute centered covariance matrix: Hᵢⱼ = Σ(aᵢ×bⱼ) - Σaᵢ × Σbⱼ / n
|
|
1380
|
+
nk_f64_t n_f64 = (nk_f64_t)n;
|
|
1381
|
+
nk_f64_t cross_covariance[9];
|
|
1382
|
+
cross_covariance[0] = covariance_x_x - n_f64 * centroid_a_x * centroid_b_x;
|
|
1383
|
+
cross_covariance[1] = covariance_x_y - n_f64 * centroid_a_x * centroid_b_y;
|
|
1384
|
+
cross_covariance[2] = covariance_x_z - n_f64 * centroid_a_x * centroid_b_z;
|
|
1385
|
+
cross_covariance[3] = covariance_y_x - n_f64 * centroid_a_y * centroid_b_x;
|
|
1386
|
+
cross_covariance[4] = covariance_y_y - n_f64 * centroid_a_y * centroid_b_y;
|
|
1387
|
+
cross_covariance[5] = covariance_y_z - n_f64 * centroid_a_y * centroid_b_z;
|
|
1388
|
+
cross_covariance[6] = covariance_z_x - n_f64 * centroid_a_z * centroid_b_x;
|
|
1389
|
+
cross_covariance[7] = covariance_z_y - n_f64 * centroid_a_z * centroid_b_y;
|
|
1390
|
+
cross_covariance[8] = covariance_z_z - n_f64 * centroid_a_z * centroid_b_z;
|
|
1391
|
+
|
|
1392
|
+
// SVD using f64 for full precision
|
|
1393
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
1394
|
+
nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
|
|
1395
|
+
|
|
1396
|
+
nk_f64_t r[9];
|
|
1397
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
1398
|
+
|
|
1399
|
+
// Scale factor: c = trace(D × S) / (n × variance(a))
|
|
1400
|
+
nk_f64_t det = nk_det3x3_f64_(r);
|
|
1401
|
+
nk_f64_t d3 = det < 0 ? -1.0 : 1.0;
|
|
1402
|
+
nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_s[0], 1.0, svd_s[4], 1.0, svd_s[8], d3);
|
|
1403
|
+
nk_f64_t applied_scale = trace_ds / ((nk_f64_t)n * variance_a);
|
|
1404
|
+
if (scale) *scale = (nk_f32_t)applied_scale;
|
|
1405
|
+
|
|
1406
|
+
// Handle reflection
|
|
1407
|
+
if (det < 0) {
|
|
1408
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1409
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
if (rotation)
|
|
1413
|
+
for (int j = 0; j < 9; ++j) rotation[j] = (nk_f32_t)r[j];
|
|
1414
|
+
*result = nk_f64_sqrt_haswell(nk_transformed_ssd_f32_skylake_(a, b, n, r, applied_scale, centroid_a_x, centroid_a_y,
|
|
1415
|
+
centroid_a_z, centroid_b_x, centroid_b_y,
|
|
1416
|
+
centroid_b_z) /
|
|
1417
|
+
n_f64);
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
NK_PUBLIC void nk_umeyama_f64_skylake(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
|
|
1421
|
+
nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
|
|
1422
|
+
// Fused single-pass: centroids, covariance, and variance of A
|
|
1423
|
+
__m512i const gather_idx_i64x8 = _mm512_setr_epi64(0, 3, 6, 9, 12, 15, 18, 21);
|
|
1424
|
+
__m512d const zeros_f64x8 = _mm512_setzero_pd();
|
|
1425
|
+
|
|
1426
|
+
__m512d sum_a_x_f64x8 = zeros_f64x8, sum_a_y_f64x8 = zeros_f64x8, sum_a_z_f64x8 = zeros_f64x8;
|
|
1427
|
+
__m512d sum_b_x_f64x8 = zeros_f64x8, sum_b_y_f64x8 = zeros_f64x8, sum_b_z_f64x8 = zeros_f64x8;
|
|
1428
|
+
__m512d cov_xx_f64x8 = zeros_f64x8, cov_xy_f64x8 = zeros_f64x8, cov_xz_f64x8 = zeros_f64x8;
|
|
1429
|
+
__m512d cov_yx_f64x8 = zeros_f64x8, cov_yy_f64x8 = zeros_f64x8, cov_yz_f64x8 = zeros_f64x8;
|
|
1430
|
+
__m512d cov_zx_f64x8 = zeros_f64x8, cov_zy_f64x8 = zeros_f64x8, cov_zz_f64x8 = zeros_f64x8;
|
|
1431
|
+
__m512d variance_a_f64x8 = zeros_f64x8;
|
|
1432
|
+
|
|
1433
|
+
nk_size_t i = 0;
|
|
1434
|
+
__m512d a_x_f64x8, a_y_f64x8, a_z_f64x8, b_x_f64x8, b_y_f64x8, b_z_f64x8;
|
|
1435
|
+
|
|
1436
|
+
for (; i + 8 <= n; i += 8) {
|
|
1437
|
+
nk_deinterleave_f64x8_skylake_(a + i * 3, &a_x_f64x8, &a_y_f64x8, &a_z_f64x8);
|
|
1438
|
+
nk_deinterleave_f64x8_skylake_(b + i * 3, &b_x_f64x8, &b_y_f64x8, &b_z_f64x8);
|
|
1439
|
+
|
|
1440
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, a_x_f64x8),
|
|
1441
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, a_y_f64x8);
|
|
1442
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, a_z_f64x8);
|
|
1443
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, b_x_f64x8),
|
|
1444
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, b_y_f64x8);
|
|
1445
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, b_z_f64x8);
|
|
1446
|
+
|
|
1447
|
+
cov_xx_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_x_f64x8, cov_xx_f64x8),
|
|
1448
|
+
cov_xy_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_y_f64x8, cov_xy_f64x8);
|
|
1449
|
+
cov_xz_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_z_f64x8, cov_xz_f64x8);
|
|
1450
|
+
cov_yx_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_x_f64x8, cov_yx_f64x8),
|
|
1451
|
+
cov_yy_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_y_f64x8, cov_yy_f64x8);
|
|
1452
|
+
cov_yz_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_z_f64x8, cov_yz_f64x8);
|
|
1453
|
+
cov_zx_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_x_f64x8, cov_zx_f64x8),
|
|
1454
|
+
cov_zy_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_y_f64x8, cov_zy_f64x8);
|
|
1455
|
+
cov_zz_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_z_f64x8, cov_zz_f64x8);
|
|
1456
|
+
variance_a_f64x8 = _mm512_fmadd_pd(a_x_f64x8, a_x_f64x8, variance_a_f64x8);
|
|
1457
|
+
variance_a_f64x8 = _mm512_fmadd_pd(a_y_f64x8, a_y_f64x8, variance_a_f64x8);
|
|
1458
|
+
variance_a_f64x8 = _mm512_fmadd_pd(a_z_f64x8, a_z_f64x8, variance_a_f64x8);
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
if (i < n) {
|
|
1462
|
+
nk_size_t tail = n - i;
|
|
1463
|
+
__mmask8 mask = (__mmask8)_bzhi_u32(0xFF, tail);
|
|
1464
|
+
nk_f64_t const *a_tail = a + i * 3;
|
|
1465
|
+
nk_f64_t const *b_tail = b + i * 3;
|
|
1466
|
+
|
|
1467
|
+
a_x_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 0, 8);
|
|
1468
|
+
a_y_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 1, 8);
|
|
1469
|
+
a_z_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, a_tail + 2, 8);
|
|
1470
|
+
b_x_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 0, 8);
|
|
1471
|
+
b_y_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 1, 8);
|
|
1472
|
+
b_z_f64x8 = _mm512_mask_i64gather_pd(zeros_f64x8, mask, gather_idx_i64x8, b_tail + 2, 8);
|
|
1473
|
+
|
|
1474
|
+
sum_a_x_f64x8 = _mm512_add_pd(sum_a_x_f64x8, a_x_f64x8),
|
|
1475
|
+
sum_a_y_f64x8 = _mm512_add_pd(sum_a_y_f64x8, a_y_f64x8);
|
|
1476
|
+
sum_a_z_f64x8 = _mm512_add_pd(sum_a_z_f64x8, a_z_f64x8);
|
|
1477
|
+
sum_b_x_f64x8 = _mm512_add_pd(sum_b_x_f64x8, b_x_f64x8),
|
|
1478
|
+
sum_b_y_f64x8 = _mm512_add_pd(sum_b_y_f64x8, b_y_f64x8);
|
|
1479
|
+
sum_b_z_f64x8 = _mm512_add_pd(sum_b_z_f64x8, b_z_f64x8);
|
|
1480
|
+
|
|
1481
|
+
cov_xx_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_x_f64x8, cov_xx_f64x8),
|
|
1482
|
+
cov_xy_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_y_f64x8, cov_xy_f64x8);
|
|
1483
|
+
cov_xz_f64x8 = _mm512_fmadd_pd(a_x_f64x8, b_z_f64x8, cov_xz_f64x8);
|
|
1484
|
+
cov_yx_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_x_f64x8, cov_yx_f64x8),
|
|
1485
|
+
cov_yy_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_y_f64x8, cov_yy_f64x8);
|
|
1486
|
+
cov_yz_f64x8 = _mm512_fmadd_pd(a_y_f64x8, b_z_f64x8, cov_yz_f64x8);
|
|
1487
|
+
cov_zx_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_x_f64x8, cov_zx_f64x8),
|
|
1488
|
+
cov_zy_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_y_f64x8, cov_zy_f64x8);
|
|
1489
|
+
cov_zz_f64x8 = _mm512_fmadd_pd(a_z_f64x8, b_z_f64x8, cov_zz_f64x8);
|
|
1490
|
+
variance_a_f64x8 = _mm512_fmadd_pd(a_x_f64x8, a_x_f64x8, variance_a_f64x8);
|
|
1491
|
+
variance_a_f64x8 = _mm512_fmadd_pd(a_y_f64x8, a_y_f64x8, variance_a_f64x8);
|
|
1492
|
+
variance_a_f64x8 = _mm512_fmadd_pd(a_z_f64x8, a_z_f64x8, variance_a_f64x8);
|
|
1493
|
+
i = n;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
// Reduce centroids, covariance, and variance.
|
|
1497
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
1498
|
+
nk_f64_t sum_a_x = nk_reduce_stable_f64x8_skylake_(sum_a_x_f64x8), sum_a_x_compensation = 0.0;
|
|
1499
|
+
nk_f64_t sum_a_y = nk_reduce_stable_f64x8_skylake_(sum_a_y_f64x8), sum_a_y_compensation = 0.0;
|
|
1500
|
+
nk_f64_t sum_a_z = nk_reduce_stable_f64x8_skylake_(sum_a_z_f64x8), sum_a_z_compensation = 0.0;
|
|
1501
|
+
nk_f64_t sum_b_x = nk_reduce_stable_f64x8_skylake_(sum_b_x_f64x8), sum_b_x_compensation = 0.0;
|
|
1502
|
+
nk_f64_t sum_b_y = nk_reduce_stable_f64x8_skylake_(sum_b_y_f64x8), sum_b_y_compensation = 0.0;
|
|
1503
|
+
nk_f64_t sum_b_z = nk_reduce_stable_f64x8_skylake_(sum_b_z_f64x8), sum_b_z_compensation = 0.0;
|
|
1504
|
+
nk_f64_t covariance_x_x = nk_reduce_stable_f64x8_skylake_(cov_xx_f64x8), covariance_x_x_compensation = 0.0;
|
|
1505
|
+
nk_f64_t covariance_x_y = nk_reduce_stable_f64x8_skylake_(cov_xy_f64x8), covariance_x_y_compensation = 0.0;
|
|
1506
|
+
nk_f64_t covariance_x_z = nk_reduce_stable_f64x8_skylake_(cov_xz_f64x8), covariance_x_z_compensation = 0.0;
|
|
1507
|
+
nk_f64_t covariance_y_x = nk_reduce_stable_f64x8_skylake_(cov_yx_f64x8), covariance_y_x_compensation = 0.0;
|
|
1508
|
+
nk_f64_t covariance_y_y = nk_reduce_stable_f64x8_skylake_(cov_yy_f64x8), covariance_y_y_compensation = 0.0;
|
|
1509
|
+
nk_f64_t covariance_y_z = nk_reduce_stable_f64x8_skylake_(cov_yz_f64x8), covariance_y_z_compensation = 0.0;
|
|
1510
|
+
nk_f64_t covariance_z_x = nk_reduce_stable_f64x8_skylake_(cov_zx_f64x8), covariance_z_x_compensation = 0.0;
|
|
1511
|
+
nk_f64_t covariance_z_y = nk_reduce_stable_f64x8_skylake_(cov_zy_f64x8), covariance_z_y_compensation = 0.0;
|
|
1512
|
+
nk_f64_t covariance_z_z = nk_reduce_stable_f64x8_skylake_(cov_zz_f64x8), covariance_z_z_compensation = 0.0;
|
|
1513
|
+
nk_f64_t variance_a_sum = nk_reduce_stable_f64x8_skylake_(variance_a_f64x8), variance_a_compensation = 0.0;
|
|
1514
|
+
|
|
1515
|
+
for (; i < n; ++i) {
|
|
1516
|
+
nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
|
|
1517
|
+
nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
|
|
1518
|
+
nk_accumulate_sum_f64_(&sum_a_x, &sum_a_x_compensation, ax);
|
|
1519
|
+
nk_accumulate_sum_f64_(&sum_a_y, &sum_a_y_compensation, ay);
|
|
1520
|
+
nk_accumulate_sum_f64_(&sum_a_z, &sum_a_z_compensation, az);
|
|
1521
|
+
nk_accumulate_sum_f64_(&sum_b_x, &sum_b_x_compensation, bx);
|
|
1522
|
+
nk_accumulate_sum_f64_(&sum_b_y, &sum_b_y_compensation, by);
|
|
1523
|
+
nk_accumulate_sum_f64_(&sum_b_z, &sum_b_z_compensation, bz);
|
|
1524
|
+
nk_accumulate_product_f64_(&covariance_x_x, &covariance_x_x_compensation, ax, bx);
|
|
1525
|
+
nk_accumulate_product_f64_(&covariance_x_y, &covariance_x_y_compensation, ax, by);
|
|
1526
|
+
nk_accumulate_product_f64_(&covariance_x_z, &covariance_x_z_compensation, ax, bz);
|
|
1527
|
+
nk_accumulate_product_f64_(&covariance_y_x, &covariance_y_x_compensation, ay, bx);
|
|
1528
|
+
nk_accumulate_product_f64_(&covariance_y_y, &covariance_y_y_compensation, ay, by);
|
|
1529
|
+
nk_accumulate_product_f64_(&covariance_y_z, &covariance_y_z_compensation, ay, bz);
|
|
1530
|
+
nk_accumulate_product_f64_(&covariance_z_x, &covariance_z_x_compensation, az, bx);
|
|
1531
|
+
nk_accumulate_product_f64_(&covariance_z_y, &covariance_z_y_compensation, az, by);
|
|
1532
|
+
nk_accumulate_product_f64_(&covariance_z_z, &covariance_z_z_compensation, az, bz);
|
|
1533
|
+
nk_accumulate_square_f64_(&variance_a_sum, &variance_a_compensation, ax);
|
|
1534
|
+
nk_accumulate_square_f64_(&variance_a_sum, &variance_a_compensation, ay);
|
|
1535
|
+
nk_accumulate_square_f64_(&variance_a_sum, &variance_a_compensation, az);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
sum_a_x += sum_a_x_compensation, sum_a_y += sum_a_y_compensation, sum_a_z += sum_a_z_compensation;
|
|
1539
|
+
sum_b_x += sum_b_x_compensation, sum_b_y += sum_b_y_compensation, sum_b_z += sum_b_z_compensation;
|
|
1540
|
+
covariance_x_x += covariance_x_x_compensation, covariance_x_y += covariance_x_y_compensation,
|
|
1541
|
+
covariance_x_z += covariance_x_z_compensation;
|
|
1542
|
+
covariance_y_x += covariance_y_x_compensation, covariance_y_y += covariance_y_y_compensation,
|
|
1543
|
+
covariance_y_z += covariance_y_z_compensation;
|
|
1544
|
+
covariance_z_x += covariance_z_x_compensation, covariance_z_y += covariance_z_y_compensation,
|
|
1545
|
+
covariance_z_z += covariance_z_z_compensation;
|
|
1546
|
+
variance_a_sum += variance_a_compensation;
|
|
1547
|
+
|
|
1548
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1549
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1550
|
+
|
|
1551
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1552
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1553
|
+
|
|
1554
|
+
// Compute centered covariance and variance.
|
|
1555
|
+
nk_f64_t variance_a = variance_a_sum * inv_n -
|
|
1556
|
+
(centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
|
|
1557
|
+
|
|
1558
|
+
// Compute centered covariance matrix: Hᵢⱼ = Σ(aᵢ×bⱼ) - Σaᵢ × Σbⱼ / n.
|
|
1559
|
+
nk_f64_t cross_covariance[9];
|
|
1560
|
+
cross_covariance[0] = covariance_x_x - sum_a_x * sum_b_x * inv_n;
|
|
1561
|
+
cross_covariance[1] = covariance_x_y - sum_a_x * sum_b_y * inv_n;
|
|
1562
|
+
cross_covariance[2] = covariance_x_z - sum_a_x * sum_b_z * inv_n;
|
|
1563
|
+
cross_covariance[3] = covariance_y_x - sum_a_y * sum_b_x * inv_n;
|
|
1564
|
+
cross_covariance[4] = covariance_y_y - sum_a_y * sum_b_y * inv_n;
|
|
1565
|
+
cross_covariance[5] = covariance_y_z - sum_a_y * sum_b_z * inv_n;
|
|
1566
|
+
cross_covariance[6] = covariance_z_x - sum_a_z * sum_b_x * inv_n;
|
|
1567
|
+
cross_covariance[7] = covariance_z_y - sum_a_z * sum_b_y * inv_n;
|
|
1568
|
+
cross_covariance[8] = covariance_z_z - sum_a_z * sum_b_z * inv_n;
|
|
1569
|
+
|
|
1570
|
+
// SVD using f64 for full precision
|
|
1571
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
1572
|
+
nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
|
|
1573
|
+
|
|
1574
|
+
nk_f64_t r[9];
|
|
1575
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
1576
|
+
|
|
1577
|
+
// Scale factor: c = trace(D × S) / (n × variance(a))
|
|
1578
|
+
nk_f64_t det = nk_det3x3_f64_(r);
|
|
1579
|
+
nk_f64_t d3 = det < 0 ? -1.0 : 1.0;
|
|
1580
|
+
nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_s[0], 1.0, svd_s[4], 1.0, svd_s[8], d3);
|
|
1581
|
+
nk_f64_t c = trace_ds / ((nk_f64_t)n * variance_a);
|
|
1582
|
+
if (scale) *scale = c;
|
|
1583
|
+
|
|
1584
|
+
// Handle reflection
|
|
1585
|
+
if (det < 0) {
|
|
1586
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1587
|
+
nk_rotation_from_svd_f64_skylake_(svd_u, svd_v, r);
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
// Output rotation matrix.
|
|
1591
|
+
if (rotation)
|
|
1592
|
+
for (int j = 0; j < 9; ++j) rotation[j] = (nk_f64_t)r[j];
|
|
1593
|
+
|
|
1594
|
+
// Compute RMSD with scaling
|
|
1595
|
+
nk_f64_t sum_squared = nk_transformed_ssd_f64_skylake_(a, b, n, r, c, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
1596
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
1597
|
+
*result = nk_f64_sqrt_haswell(sum_squared * inv_n);
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
NK_PUBLIC void nk_rmsd_f16_skylake(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1601
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1602
|
+
if (rotation)
|
|
1603
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
1604
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
1605
|
+
if (scale) *scale = 1.0f;
|
|
1606
|
+
|
|
1607
|
+
__m512 const zeros_f32x16 = _mm512_setzero_ps();
|
|
1608
|
+
__m512 sum_a_x_f32x16 = zeros_f32x16, sum_a_y_f32x16 = zeros_f32x16, sum_a_z_f32x16 = zeros_f32x16;
|
|
1609
|
+
__m512 sum_b_x_f32x16 = zeros_f32x16, sum_b_y_f32x16 = zeros_f32x16, sum_b_z_f32x16 = zeros_f32x16;
|
|
1610
|
+
__m512 sum_squared_x_f32x16 = zeros_f32x16, sum_squared_y_f32x16 = zeros_f32x16;
|
|
1611
|
+
__m512 sum_squared_z_f32x16 = zeros_f32x16;
|
|
1612
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
1613
|
+
nk_size_t i = 0;
|
|
1614
|
+
|
|
1615
|
+
for (; i + 16 <= n; i += 16) {
|
|
1616
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1617
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1618
|
+
|
|
1619
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1620
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1621
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1622
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1623
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1624
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1625
|
+
|
|
1626
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(a_x_f32x16, b_x_f32x16);
|
|
1627
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(a_y_f32x16, b_y_f32x16);
|
|
1628
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(a_z_f32x16, b_z_f32x16);
|
|
1629
|
+
|
|
1630
|
+
sum_squared_x_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_x_f32x16);
|
|
1631
|
+
sum_squared_y_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_y_f32x16);
|
|
1632
|
+
sum_squared_z_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_z_f32x16);
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
1636
|
+
if (i < n) {
|
|
1637
|
+
nk_size_t tail = n - i;
|
|
1638
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(a + i * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1639
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(b + i * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1640
|
+
|
|
1641
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1642
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1643
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1644
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1645
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1646
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1647
|
+
|
|
1648
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(a_x_f32x16, b_x_f32x16);
|
|
1649
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(a_y_f32x16, b_y_f32x16);
|
|
1650
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(a_z_f32x16, b_z_f32x16);
|
|
1651
|
+
|
|
1652
|
+
sum_squared_x_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_x_f32x16);
|
|
1653
|
+
sum_squared_y_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_y_f32x16);
|
|
1654
|
+
sum_squared_z_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_z_f32x16);
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
nk_f32_t total_ax = _mm512_reduce_add_ps(sum_a_x_f32x16);
|
|
1658
|
+
nk_f32_t total_ay = _mm512_reduce_add_ps(sum_a_y_f32x16);
|
|
1659
|
+
nk_f32_t total_az = _mm512_reduce_add_ps(sum_a_z_f32x16);
|
|
1660
|
+
nk_f32_t total_bx = _mm512_reduce_add_ps(sum_b_x_f32x16);
|
|
1661
|
+
nk_f32_t total_by = _mm512_reduce_add_ps(sum_b_y_f32x16);
|
|
1662
|
+
nk_f32_t total_bz = _mm512_reduce_add_ps(sum_b_z_f32x16);
|
|
1663
|
+
nk_f32_t total_sq_x = _mm512_reduce_add_ps(sum_squared_x_f32x16);
|
|
1664
|
+
nk_f32_t total_sq_y = _mm512_reduce_add_ps(sum_squared_y_f32x16);
|
|
1665
|
+
nk_f32_t total_sq_z = _mm512_reduce_add_ps(sum_squared_z_f32x16);
|
|
1666
|
+
|
|
1667
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1668
|
+
nk_f32_t centroid_a_x = total_ax * inv_n;
|
|
1669
|
+
nk_f32_t centroid_a_y = total_ay * inv_n;
|
|
1670
|
+
nk_f32_t centroid_a_z = total_az * inv_n;
|
|
1671
|
+
nk_f32_t centroid_b_x = total_bx * inv_n;
|
|
1672
|
+
nk_f32_t centroid_b_y = total_by * inv_n;
|
|
1673
|
+
nk_f32_t centroid_b_z = total_bz * inv_n;
|
|
1674
|
+
|
|
1675
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1676
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1677
|
+
|
|
1678
|
+
nk_f32_t mean_diff_x = centroid_a_x - centroid_b_x;
|
|
1679
|
+
nk_f32_t mean_diff_y = centroid_a_y - centroid_b_y;
|
|
1680
|
+
nk_f32_t mean_diff_z = centroid_a_z - centroid_b_z;
|
|
1681
|
+
nk_f32_t sum_squared = total_sq_x + total_sq_y + total_sq_z;
|
|
1682
|
+
nk_f32_t mean_diff_sq = mean_diff_x * mean_diff_x + mean_diff_y * mean_diff_y + mean_diff_z * mean_diff_z;
|
|
1683
|
+
|
|
1684
|
+
*result = nk_f32_sqrt_haswell(sum_squared * inv_n - mean_diff_sq);
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
NK_PUBLIC void nk_rmsd_bf16_skylake(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1688
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1689
|
+
if (rotation)
|
|
1690
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
1691
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
1692
|
+
if (scale) *scale = 1.0f;
|
|
1693
|
+
|
|
1694
|
+
__m512 const zeros_f32x16 = _mm512_setzero_ps();
|
|
1695
|
+
__m512 sum_a_x_f32x16 = zeros_f32x16, sum_a_y_f32x16 = zeros_f32x16, sum_a_z_f32x16 = zeros_f32x16;
|
|
1696
|
+
__m512 sum_b_x_f32x16 = zeros_f32x16, sum_b_y_f32x16 = zeros_f32x16, sum_b_z_f32x16 = zeros_f32x16;
|
|
1697
|
+
__m512 sum_squared_x_f32x16 = zeros_f32x16, sum_squared_y_f32x16 = zeros_f32x16;
|
|
1698
|
+
__m512 sum_squared_z_f32x16 = zeros_f32x16;
|
|
1699
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
1700
|
+
nk_size_t i = 0;
|
|
1701
|
+
|
|
1702
|
+
for (; i + 16 <= n; i += 16) {
|
|
1703
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1704
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1705
|
+
|
|
1706
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1707
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1708
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1709
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1710
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1711
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1712
|
+
|
|
1713
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(a_x_f32x16, b_x_f32x16);
|
|
1714
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(a_y_f32x16, b_y_f32x16);
|
|
1715
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(a_z_f32x16, b_z_f32x16);
|
|
1716
|
+
|
|
1717
|
+
sum_squared_x_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_x_f32x16);
|
|
1718
|
+
sum_squared_y_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_y_f32x16);
|
|
1719
|
+
sum_squared_z_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_z_f32x16);
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
1723
|
+
if (i < n) {
|
|
1724
|
+
nk_size_t tail = n - i;
|
|
1725
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(a + i * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1726
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(b + i * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1727
|
+
|
|
1728
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1729
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1730
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1731
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1732
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1733
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1734
|
+
|
|
1735
|
+
__m512 delta_x_f32x16 = _mm512_sub_ps(a_x_f32x16, b_x_f32x16);
|
|
1736
|
+
__m512 delta_y_f32x16 = _mm512_sub_ps(a_y_f32x16, b_y_f32x16);
|
|
1737
|
+
__m512 delta_z_f32x16 = _mm512_sub_ps(a_z_f32x16, b_z_f32x16);
|
|
1738
|
+
|
|
1739
|
+
sum_squared_x_f32x16 = _mm512_fmadd_ps(delta_x_f32x16, delta_x_f32x16, sum_squared_x_f32x16);
|
|
1740
|
+
sum_squared_y_f32x16 = _mm512_fmadd_ps(delta_y_f32x16, delta_y_f32x16, sum_squared_y_f32x16);
|
|
1741
|
+
sum_squared_z_f32x16 = _mm512_fmadd_ps(delta_z_f32x16, delta_z_f32x16, sum_squared_z_f32x16);
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
nk_f32_t total_ax = _mm512_reduce_add_ps(sum_a_x_f32x16);
|
|
1745
|
+
nk_f32_t total_ay = _mm512_reduce_add_ps(sum_a_y_f32x16);
|
|
1746
|
+
nk_f32_t total_az = _mm512_reduce_add_ps(sum_a_z_f32x16);
|
|
1747
|
+
nk_f32_t total_bx = _mm512_reduce_add_ps(sum_b_x_f32x16);
|
|
1748
|
+
nk_f32_t total_by = _mm512_reduce_add_ps(sum_b_y_f32x16);
|
|
1749
|
+
nk_f32_t total_bz = _mm512_reduce_add_ps(sum_b_z_f32x16);
|
|
1750
|
+
nk_f32_t total_sq_x = _mm512_reduce_add_ps(sum_squared_x_f32x16);
|
|
1751
|
+
nk_f32_t total_sq_y = _mm512_reduce_add_ps(sum_squared_y_f32x16);
|
|
1752
|
+
nk_f32_t total_sq_z = _mm512_reduce_add_ps(sum_squared_z_f32x16);
|
|
1753
|
+
|
|
1754
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1755
|
+
nk_f32_t centroid_a_x = total_ax * inv_n;
|
|
1756
|
+
nk_f32_t centroid_a_y = total_ay * inv_n;
|
|
1757
|
+
nk_f32_t centroid_a_z = total_az * inv_n;
|
|
1758
|
+
nk_f32_t centroid_b_x = total_bx * inv_n;
|
|
1759
|
+
nk_f32_t centroid_b_y = total_by * inv_n;
|
|
1760
|
+
nk_f32_t centroid_b_z = total_bz * inv_n;
|
|
1761
|
+
|
|
1762
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1763
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1764
|
+
|
|
1765
|
+
nk_f32_t mean_diff_x = centroid_a_x - centroid_b_x;
|
|
1766
|
+
nk_f32_t mean_diff_y = centroid_a_y - centroid_b_y;
|
|
1767
|
+
nk_f32_t mean_diff_z = centroid_a_z - centroid_b_z;
|
|
1768
|
+
nk_f32_t sum_squared = total_sq_x + total_sq_y + total_sq_z;
|
|
1769
|
+
nk_f32_t mean_diff_sq = mean_diff_x * mean_diff_x + mean_diff_y * mean_diff_y + mean_diff_z * mean_diff_z;
|
|
1770
|
+
|
|
1771
|
+
*result = nk_f32_sqrt_haswell(sum_squared * inv_n - mean_diff_sq);
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
NK_PUBLIC void nk_kabsch_f16_skylake(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1775
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1776
|
+
__m512 const zeros_f32x16 = _mm512_setzero_ps();
|
|
1777
|
+
|
|
1778
|
+
__m512 sum_a_x_f32x16 = zeros_f32x16, sum_a_y_f32x16 = zeros_f32x16, sum_a_z_f32x16 = zeros_f32x16;
|
|
1779
|
+
__m512 sum_b_x_f32x16 = zeros_f32x16, sum_b_y_f32x16 = zeros_f32x16, sum_b_z_f32x16 = zeros_f32x16;
|
|
1780
|
+
__m512 cov_xx_f32x16 = zeros_f32x16, cov_xy_f32x16 = zeros_f32x16, cov_xz_f32x16 = zeros_f32x16;
|
|
1781
|
+
__m512 cov_yx_f32x16 = zeros_f32x16, cov_yy_f32x16 = zeros_f32x16, cov_yz_f32x16 = zeros_f32x16;
|
|
1782
|
+
__m512 cov_zx_f32x16 = zeros_f32x16, cov_zy_f32x16 = zeros_f32x16, cov_zz_f32x16 = zeros_f32x16;
|
|
1783
|
+
|
|
1784
|
+
nk_size_t i = 0;
|
|
1785
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
1786
|
+
|
|
1787
|
+
for (; i + 16 <= n; i += 16) {
|
|
1788
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1789
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1790
|
+
|
|
1791
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1792
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1793
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1794
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1795
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1796
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1797
|
+
|
|
1798
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
1799
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
1800
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
1801
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
1802
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
1803
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
1804
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
1805
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
1806
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
1810
|
+
if (i < n) {
|
|
1811
|
+
nk_size_t tail = n - i;
|
|
1812
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(a + i * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1813
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(b + i * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1814
|
+
|
|
1815
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1816
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1817
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1818
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1819
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1820
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1821
|
+
|
|
1822
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
1823
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
1824
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
1825
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
1826
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
1827
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
1828
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
1829
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
1830
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
nk_f32_t sum_a_x = _mm512_reduce_add_ps(sum_a_x_f32x16);
|
|
1834
|
+
nk_f32_t sum_a_y = _mm512_reduce_add_ps(sum_a_y_f32x16);
|
|
1835
|
+
nk_f32_t sum_a_z = _mm512_reduce_add_ps(sum_a_z_f32x16);
|
|
1836
|
+
nk_f32_t sum_b_x = _mm512_reduce_add_ps(sum_b_x_f32x16);
|
|
1837
|
+
nk_f32_t sum_b_y = _mm512_reduce_add_ps(sum_b_y_f32x16);
|
|
1838
|
+
nk_f32_t sum_b_z = _mm512_reduce_add_ps(sum_b_z_f32x16);
|
|
1839
|
+
nk_f32_t covariance_x_x = _mm512_reduce_add_ps(cov_xx_f32x16);
|
|
1840
|
+
nk_f32_t covariance_x_y = _mm512_reduce_add_ps(cov_xy_f32x16);
|
|
1841
|
+
nk_f32_t covariance_x_z = _mm512_reduce_add_ps(cov_xz_f32x16);
|
|
1842
|
+
nk_f32_t covariance_y_x = _mm512_reduce_add_ps(cov_yx_f32x16);
|
|
1843
|
+
nk_f32_t covariance_y_y = _mm512_reduce_add_ps(cov_yy_f32x16);
|
|
1844
|
+
nk_f32_t covariance_y_z = _mm512_reduce_add_ps(cov_yz_f32x16);
|
|
1845
|
+
nk_f32_t covariance_z_x = _mm512_reduce_add_ps(cov_zx_f32x16);
|
|
1846
|
+
nk_f32_t covariance_z_y = _mm512_reduce_add_ps(cov_zy_f32x16);
|
|
1847
|
+
nk_f32_t covariance_z_z = _mm512_reduce_add_ps(cov_zz_f32x16);
|
|
1848
|
+
|
|
1849
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1850
|
+
nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1851
|
+
nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1852
|
+
|
|
1853
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1854
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1855
|
+
|
|
1856
|
+
covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
|
|
1857
|
+
covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
|
|
1858
|
+
covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
|
|
1859
|
+
covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
|
|
1860
|
+
covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
|
|
1861
|
+
covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
|
|
1862
|
+
covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
|
|
1863
|
+
covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
|
|
1864
|
+
covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
|
|
1865
|
+
|
|
1866
|
+
nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
|
|
1867
|
+
covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
|
|
1868
|
+
nk_f32_t svd_u[9], svd_s[9], svd_v[9];
|
|
1869
|
+
nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
|
|
1870
|
+
|
|
1871
|
+
nk_f32_t r[9];
|
|
1872
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
1873
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
1874
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
1875
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
1876
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
1877
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
1878
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
1879
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
1880
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
1881
|
+
|
|
1882
|
+
if (nk_det3x3_f32_(r) < 0) {
|
|
1883
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1884
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
1885
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
1886
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
1887
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
1888
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
1889
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
1890
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
1891
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
1892
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
1893
|
+
}
|
|
1894
|
+
|
|
1895
|
+
if (rotation)
|
|
1896
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
1897
|
+
if (scale) *scale = 1.0f;
|
|
1898
|
+
|
|
1899
|
+
nk_f32_t sum_squared = nk_transformed_ssd_f16_skylake_(a, b, n, r, 1.0f, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
1900
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
1901
|
+
*result = nk_f32_sqrt_haswell(sum_squared * inv_n);
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
NK_PUBLIC void nk_kabsch_bf16_skylake(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1905
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1906
|
+
__m512 const zeros_f32x16 = _mm512_setzero_ps();
|
|
1907
|
+
|
|
1908
|
+
__m512 sum_a_x_f32x16 = zeros_f32x16, sum_a_y_f32x16 = zeros_f32x16, sum_a_z_f32x16 = zeros_f32x16;
|
|
1909
|
+
__m512 sum_b_x_f32x16 = zeros_f32x16, sum_b_y_f32x16 = zeros_f32x16, sum_b_z_f32x16 = zeros_f32x16;
|
|
1910
|
+
__m512 cov_xx_f32x16 = zeros_f32x16, cov_xy_f32x16 = zeros_f32x16, cov_xz_f32x16 = zeros_f32x16;
|
|
1911
|
+
__m512 cov_yx_f32x16 = zeros_f32x16, cov_yy_f32x16 = zeros_f32x16, cov_yz_f32x16 = zeros_f32x16;
|
|
1912
|
+
__m512 cov_zx_f32x16 = zeros_f32x16, cov_zy_f32x16 = zeros_f32x16, cov_zz_f32x16 = zeros_f32x16;
|
|
1913
|
+
|
|
1914
|
+
nk_size_t i = 0;
|
|
1915
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
1916
|
+
|
|
1917
|
+
for (; i + 16 <= n; i += 16) {
|
|
1918
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1919
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1920
|
+
|
|
1921
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1922
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1923
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1924
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1925
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1926
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1927
|
+
|
|
1928
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
1929
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
1930
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
1931
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
1932
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
1933
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
1934
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
1935
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
1936
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
1940
|
+
if (i < n) {
|
|
1941
|
+
nk_size_t tail = n - i;
|
|
1942
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(a + i * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
1943
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(b + i * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
1944
|
+
|
|
1945
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
1946
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
1947
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
1948
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
1949
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
1950
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
1951
|
+
|
|
1952
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
1953
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
1954
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
1955
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
1956
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
1957
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
1958
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
1959
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
1960
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
1961
|
+
}
|
|
1962
|
+
|
|
1963
|
+
nk_f32_t sum_a_x = _mm512_reduce_add_ps(sum_a_x_f32x16);
|
|
1964
|
+
nk_f32_t sum_a_y = _mm512_reduce_add_ps(sum_a_y_f32x16);
|
|
1965
|
+
nk_f32_t sum_a_z = _mm512_reduce_add_ps(sum_a_z_f32x16);
|
|
1966
|
+
nk_f32_t sum_b_x = _mm512_reduce_add_ps(sum_b_x_f32x16);
|
|
1967
|
+
nk_f32_t sum_b_y = _mm512_reduce_add_ps(sum_b_y_f32x16);
|
|
1968
|
+
nk_f32_t sum_b_z = _mm512_reduce_add_ps(sum_b_z_f32x16);
|
|
1969
|
+
nk_f32_t covariance_x_x = _mm512_reduce_add_ps(cov_xx_f32x16);
|
|
1970
|
+
nk_f32_t covariance_x_y = _mm512_reduce_add_ps(cov_xy_f32x16);
|
|
1971
|
+
nk_f32_t covariance_x_z = _mm512_reduce_add_ps(cov_xz_f32x16);
|
|
1972
|
+
nk_f32_t covariance_y_x = _mm512_reduce_add_ps(cov_yx_f32x16);
|
|
1973
|
+
nk_f32_t covariance_y_y = _mm512_reduce_add_ps(cov_yy_f32x16);
|
|
1974
|
+
nk_f32_t covariance_y_z = _mm512_reduce_add_ps(cov_yz_f32x16);
|
|
1975
|
+
nk_f32_t covariance_z_x = _mm512_reduce_add_ps(cov_zx_f32x16);
|
|
1976
|
+
nk_f32_t covariance_z_y = _mm512_reduce_add_ps(cov_zy_f32x16);
|
|
1977
|
+
nk_f32_t covariance_z_z = _mm512_reduce_add_ps(cov_zz_f32x16);
|
|
1978
|
+
|
|
1979
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1980
|
+
nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1981
|
+
nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1982
|
+
|
|
1983
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1984
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1985
|
+
|
|
1986
|
+
covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
|
|
1987
|
+
covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
|
|
1988
|
+
covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
|
|
1989
|
+
covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
|
|
1990
|
+
covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
|
|
1991
|
+
covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
|
|
1992
|
+
covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
|
|
1993
|
+
covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
|
|
1994
|
+
covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
|
|
1995
|
+
|
|
1996
|
+
nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
|
|
1997
|
+
covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
|
|
1998
|
+
nk_f32_t svd_u[9], svd_s[9], svd_v[9];
|
|
1999
|
+
nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
|
|
2000
|
+
|
|
2001
|
+
nk_f32_t r[9];
|
|
2002
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
2003
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
2004
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
2005
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
2006
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
2007
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
2008
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
2009
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
2010
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
2011
|
+
|
|
2012
|
+
if (nk_det3x3_f32_(r) < 0) {
|
|
2013
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
2014
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
2015
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
2016
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
2017
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
2018
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
2019
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
2020
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
2021
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
2022
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
if (rotation)
|
|
2026
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
2027
|
+
if (scale) *scale = 1.0f;
|
|
2028
|
+
|
|
2029
|
+
nk_f32_t sum_squared = nk_transformed_ssd_bf16_skylake_(a, b, n, r, 1.0f, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
2030
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
2031
|
+
*result = nk_f32_sqrt_haswell(sum_squared * inv_n);
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
NK_PUBLIC void nk_umeyama_f16_skylake(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
2035
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
2036
|
+
__m512 const zeros_f32x16 = _mm512_setzero_ps();
|
|
2037
|
+
|
|
2038
|
+
__m512 sum_a_x_f32x16 = zeros_f32x16, sum_a_y_f32x16 = zeros_f32x16, sum_a_z_f32x16 = zeros_f32x16;
|
|
2039
|
+
__m512 sum_b_x_f32x16 = zeros_f32x16, sum_b_y_f32x16 = zeros_f32x16, sum_b_z_f32x16 = zeros_f32x16;
|
|
2040
|
+
__m512 cov_xx_f32x16 = zeros_f32x16, cov_xy_f32x16 = zeros_f32x16, cov_xz_f32x16 = zeros_f32x16;
|
|
2041
|
+
__m512 cov_yx_f32x16 = zeros_f32x16, cov_yy_f32x16 = zeros_f32x16, cov_yz_f32x16 = zeros_f32x16;
|
|
2042
|
+
__m512 cov_zx_f32x16 = zeros_f32x16, cov_zy_f32x16 = zeros_f32x16, cov_zz_f32x16 = zeros_f32x16;
|
|
2043
|
+
__m512 variance_a_f32x16 = zeros_f32x16;
|
|
2044
|
+
|
|
2045
|
+
nk_size_t i = 0;
|
|
2046
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
2047
|
+
|
|
2048
|
+
for (; i + 16 <= n; i += 16) {
|
|
2049
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
2050
|
+
nk_deinterleave_f16x16_to_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
2051
|
+
|
|
2052
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
2053
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
2054
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
2055
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
2056
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
2057
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
2058
|
+
|
|
2059
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
2060
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
2061
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
2062
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
2063
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
2064
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
2065
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
2066
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
2067
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
2068
|
+
|
|
2069
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_x_f32x16, a_x_f32x16, variance_a_f32x16);
|
|
2070
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_y_f32x16, a_y_f32x16, variance_a_f32x16);
|
|
2071
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_z_f32x16, a_z_f32x16, variance_a_f32x16);
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
2075
|
+
if (i < n) {
|
|
2076
|
+
nk_size_t tail = n - i;
|
|
2077
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(a + i * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
2078
|
+
nk_deinterleave_f16_tail_to_f32x16_skylake_(b + i * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
2079
|
+
|
|
2080
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
2081
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
2082
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
2083
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
2084
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
2085
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
2086
|
+
|
|
2087
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
2088
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
2089
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
2090
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
2091
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
2092
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
2093
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
2094
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
2095
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
2096
|
+
|
|
2097
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_x_f32x16, a_x_f32x16, variance_a_f32x16);
|
|
2098
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_y_f32x16, a_y_f32x16, variance_a_f32x16);
|
|
2099
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_z_f32x16, a_z_f32x16, variance_a_f32x16);
|
|
2100
|
+
}
|
|
2101
|
+
|
|
2102
|
+
nk_f32_t sum_a_x = _mm512_reduce_add_ps(sum_a_x_f32x16);
|
|
2103
|
+
nk_f32_t sum_a_y = _mm512_reduce_add_ps(sum_a_y_f32x16);
|
|
2104
|
+
nk_f32_t sum_a_z = _mm512_reduce_add_ps(sum_a_z_f32x16);
|
|
2105
|
+
nk_f32_t sum_b_x = _mm512_reduce_add_ps(sum_b_x_f32x16);
|
|
2106
|
+
nk_f32_t sum_b_y = _mm512_reduce_add_ps(sum_b_y_f32x16);
|
|
2107
|
+
nk_f32_t sum_b_z = _mm512_reduce_add_ps(sum_b_z_f32x16);
|
|
2108
|
+
nk_f32_t covariance_x_x = _mm512_reduce_add_ps(cov_xx_f32x16);
|
|
2109
|
+
nk_f32_t covariance_x_y = _mm512_reduce_add_ps(cov_xy_f32x16);
|
|
2110
|
+
nk_f32_t covariance_x_z = _mm512_reduce_add_ps(cov_xz_f32x16);
|
|
2111
|
+
nk_f32_t covariance_y_x = _mm512_reduce_add_ps(cov_yx_f32x16);
|
|
2112
|
+
nk_f32_t covariance_y_y = _mm512_reduce_add_ps(cov_yy_f32x16);
|
|
2113
|
+
nk_f32_t covariance_y_z = _mm512_reduce_add_ps(cov_yz_f32x16);
|
|
2114
|
+
nk_f32_t covariance_z_x = _mm512_reduce_add_ps(cov_zx_f32x16);
|
|
2115
|
+
nk_f32_t covariance_z_y = _mm512_reduce_add_ps(cov_zy_f32x16);
|
|
2116
|
+
nk_f32_t covariance_z_z = _mm512_reduce_add_ps(cov_zz_f32x16);
|
|
2117
|
+
nk_f32_t variance_a_sum = _mm512_reduce_add_ps(variance_a_f32x16);
|
|
2118
|
+
|
|
2119
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
2120
|
+
nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
2121
|
+
nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
2122
|
+
|
|
2123
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
2124
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
2125
|
+
|
|
2126
|
+
nk_f32_t variance_a = variance_a_sum * inv_n -
|
|
2127
|
+
(centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
|
|
2128
|
+
|
|
2129
|
+
covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
|
|
2130
|
+
covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
|
|
2131
|
+
covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
|
|
2132
|
+
covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
|
|
2133
|
+
covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
|
|
2134
|
+
covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
|
|
2135
|
+
covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
|
|
2136
|
+
covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
|
|
2137
|
+
covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
|
|
2138
|
+
|
|
2139
|
+
nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
|
|
2140
|
+
covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
|
|
2141
|
+
|
|
2142
|
+
nk_f32_t svd_u[9], svd_s[9], svd_v[9];
|
|
2143
|
+
nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
|
|
2144
|
+
|
|
2145
|
+
nk_f32_t r[9];
|
|
2146
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
2147
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
2148
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
2149
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
2150
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
2151
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
2152
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
2153
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
2154
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
2155
|
+
|
|
2156
|
+
nk_f32_t det = nk_det3x3_f32_(r);
|
|
2157
|
+
nk_f32_t d3 = det < 0 ? -1.0f : 1.0f;
|
|
2158
|
+
nk_f32_t trace_ds = svd_s[0] + svd_s[4] + d3 * svd_s[8];
|
|
2159
|
+
nk_f32_t c = trace_ds / ((nk_f32_t)n * variance_a);
|
|
2160
|
+
if (scale) *scale = c;
|
|
2161
|
+
|
|
2162
|
+
if (det < 0) {
|
|
2163
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
2164
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
2165
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
2166
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
2167
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
2168
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
2169
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
2170
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
2171
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
2172
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
if (rotation)
|
|
2176
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
2177
|
+
|
|
2178
|
+
nk_f32_t sum_squared = nk_transformed_ssd_f16_skylake_(a, b, n, r, c, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
2179
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
2180
|
+
*result = nk_f32_sqrt_haswell(sum_squared * inv_n);
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
NK_PUBLIC void nk_umeyama_bf16_skylake(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
2184
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
2185
|
+
__m512 const zeros_f32x16 = _mm512_setzero_ps();
|
|
2186
|
+
|
|
2187
|
+
__m512 sum_a_x_f32x16 = zeros_f32x16, sum_a_y_f32x16 = zeros_f32x16, sum_a_z_f32x16 = zeros_f32x16;
|
|
2188
|
+
__m512 sum_b_x_f32x16 = zeros_f32x16, sum_b_y_f32x16 = zeros_f32x16, sum_b_z_f32x16 = zeros_f32x16;
|
|
2189
|
+
__m512 cov_xx_f32x16 = zeros_f32x16, cov_xy_f32x16 = zeros_f32x16, cov_xz_f32x16 = zeros_f32x16;
|
|
2190
|
+
__m512 cov_yx_f32x16 = zeros_f32x16, cov_yy_f32x16 = zeros_f32x16, cov_yz_f32x16 = zeros_f32x16;
|
|
2191
|
+
__m512 cov_zx_f32x16 = zeros_f32x16, cov_zy_f32x16 = zeros_f32x16, cov_zz_f32x16 = zeros_f32x16;
|
|
2192
|
+
__m512 variance_a_f32x16 = zeros_f32x16;
|
|
2193
|
+
|
|
2194
|
+
nk_size_t i = 0;
|
|
2195
|
+
__m512 a_x_f32x16, a_y_f32x16, a_z_f32x16, b_x_f32x16, b_y_f32x16, b_z_f32x16;
|
|
2196
|
+
|
|
2197
|
+
for (; i + 16 <= n; i += 16) {
|
|
2198
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(a + i * 3, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
2199
|
+
nk_deinterleave_bf16x16_to_f32x16_skylake_(b + i * 3, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
2200
|
+
|
|
2201
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
2202
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
2203
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
2204
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
2205
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
2206
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
2207
|
+
|
|
2208
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
2209
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
2210
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
2211
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
2212
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
2213
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
2214
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
2215
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
2216
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
2217
|
+
|
|
2218
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_x_f32x16, a_x_f32x16, variance_a_f32x16);
|
|
2219
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_y_f32x16, a_y_f32x16, variance_a_f32x16);
|
|
2220
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_z_f32x16, a_z_f32x16, variance_a_f32x16);
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
// Tail: deinterleave remaining points into zero-initialized vectors
|
|
2224
|
+
if (i < n) {
|
|
2225
|
+
nk_size_t tail = n - i;
|
|
2226
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(a + i * 3, tail, &a_x_f32x16, &a_y_f32x16, &a_z_f32x16);
|
|
2227
|
+
nk_deinterleave_bf16_tail_to_f32x16_skylake_(b + i * 3, tail, &b_x_f32x16, &b_y_f32x16, &b_z_f32x16);
|
|
2228
|
+
|
|
2229
|
+
sum_a_x_f32x16 = _mm512_add_ps(sum_a_x_f32x16, a_x_f32x16);
|
|
2230
|
+
sum_a_y_f32x16 = _mm512_add_ps(sum_a_y_f32x16, a_y_f32x16);
|
|
2231
|
+
sum_a_z_f32x16 = _mm512_add_ps(sum_a_z_f32x16, a_z_f32x16);
|
|
2232
|
+
sum_b_x_f32x16 = _mm512_add_ps(sum_b_x_f32x16, b_x_f32x16);
|
|
2233
|
+
sum_b_y_f32x16 = _mm512_add_ps(sum_b_y_f32x16, b_y_f32x16);
|
|
2234
|
+
sum_b_z_f32x16 = _mm512_add_ps(sum_b_z_f32x16, b_z_f32x16);
|
|
2235
|
+
|
|
2236
|
+
cov_xx_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_x_f32x16, cov_xx_f32x16);
|
|
2237
|
+
cov_xy_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_y_f32x16, cov_xy_f32x16);
|
|
2238
|
+
cov_xz_f32x16 = _mm512_fmadd_ps(a_x_f32x16, b_z_f32x16, cov_xz_f32x16);
|
|
2239
|
+
cov_yx_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_x_f32x16, cov_yx_f32x16);
|
|
2240
|
+
cov_yy_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_y_f32x16, cov_yy_f32x16);
|
|
2241
|
+
cov_yz_f32x16 = _mm512_fmadd_ps(a_y_f32x16, b_z_f32x16, cov_yz_f32x16);
|
|
2242
|
+
cov_zx_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_x_f32x16, cov_zx_f32x16);
|
|
2243
|
+
cov_zy_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_y_f32x16, cov_zy_f32x16);
|
|
2244
|
+
cov_zz_f32x16 = _mm512_fmadd_ps(a_z_f32x16, b_z_f32x16, cov_zz_f32x16);
|
|
2245
|
+
|
|
2246
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_x_f32x16, a_x_f32x16, variance_a_f32x16);
|
|
2247
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_y_f32x16, a_y_f32x16, variance_a_f32x16);
|
|
2248
|
+
variance_a_f32x16 = _mm512_fmadd_ps(a_z_f32x16, a_z_f32x16, variance_a_f32x16);
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
nk_f32_t sum_a_x = _mm512_reduce_add_ps(sum_a_x_f32x16);
|
|
2252
|
+
nk_f32_t sum_a_y = _mm512_reduce_add_ps(sum_a_y_f32x16);
|
|
2253
|
+
nk_f32_t sum_a_z = _mm512_reduce_add_ps(sum_a_z_f32x16);
|
|
2254
|
+
nk_f32_t sum_b_x = _mm512_reduce_add_ps(sum_b_x_f32x16);
|
|
2255
|
+
nk_f32_t sum_b_y = _mm512_reduce_add_ps(sum_b_y_f32x16);
|
|
2256
|
+
nk_f32_t sum_b_z = _mm512_reduce_add_ps(sum_b_z_f32x16);
|
|
2257
|
+
nk_f32_t covariance_x_x = _mm512_reduce_add_ps(cov_xx_f32x16);
|
|
2258
|
+
nk_f32_t covariance_x_y = _mm512_reduce_add_ps(cov_xy_f32x16);
|
|
2259
|
+
nk_f32_t covariance_x_z = _mm512_reduce_add_ps(cov_xz_f32x16);
|
|
2260
|
+
nk_f32_t covariance_y_x = _mm512_reduce_add_ps(cov_yx_f32x16);
|
|
2261
|
+
nk_f32_t covariance_y_y = _mm512_reduce_add_ps(cov_yy_f32x16);
|
|
2262
|
+
nk_f32_t covariance_y_z = _mm512_reduce_add_ps(cov_yz_f32x16);
|
|
2263
|
+
nk_f32_t covariance_z_x = _mm512_reduce_add_ps(cov_zx_f32x16);
|
|
2264
|
+
nk_f32_t covariance_z_y = _mm512_reduce_add_ps(cov_zy_f32x16);
|
|
2265
|
+
nk_f32_t covariance_z_z = _mm512_reduce_add_ps(cov_zz_f32x16);
|
|
2266
|
+
nk_f32_t variance_a_sum = _mm512_reduce_add_ps(variance_a_f32x16);
|
|
2267
|
+
|
|
2268
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
2269
|
+
nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
2270
|
+
nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
2271
|
+
|
|
2272
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
2273
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
2274
|
+
|
|
2275
|
+
nk_f32_t variance_a = variance_a_sum * inv_n -
|
|
2276
|
+
(centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
|
|
2277
|
+
|
|
2278
|
+
covariance_x_x -= (nk_f32_t)n * centroid_a_x * centroid_b_x;
|
|
2279
|
+
covariance_x_y -= (nk_f32_t)n * centroid_a_x * centroid_b_y;
|
|
2280
|
+
covariance_x_z -= (nk_f32_t)n * centroid_a_x * centroid_b_z;
|
|
2281
|
+
covariance_y_x -= (nk_f32_t)n * centroid_a_y * centroid_b_x;
|
|
2282
|
+
covariance_y_y -= (nk_f32_t)n * centroid_a_y * centroid_b_y;
|
|
2283
|
+
covariance_y_z -= (nk_f32_t)n * centroid_a_y * centroid_b_z;
|
|
2284
|
+
covariance_z_x -= (nk_f32_t)n * centroid_a_z * centroid_b_x;
|
|
2285
|
+
covariance_z_y -= (nk_f32_t)n * centroid_a_z * centroid_b_y;
|
|
2286
|
+
covariance_z_z -= (nk_f32_t)n * centroid_a_z * centroid_b_z;
|
|
2287
|
+
|
|
2288
|
+
nk_f32_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
|
|
2289
|
+
covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
|
|
2290
|
+
|
|
2291
|
+
nk_f32_t svd_u[9], svd_s[9], svd_v[9];
|
|
2292
|
+
nk_svd3x3_f32_(cross_covariance, svd_u, svd_s, svd_v);
|
|
2293
|
+
|
|
2294
|
+
nk_f32_t r[9];
|
|
2295
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
2296
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
2297
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
2298
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
2299
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
2300
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
2301
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
2302
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
2303
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
2304
|
+
|
|
2305
|
+
nk_f32_t det = nk_det3x3_f32_(r);
|
|
2306
|
+
nk_f32_t d3 = det < 0 ? -1.0f : 1.0f;
|
|
2307
|
+
nk_f32_t trace_ds = svd_s[0] + svd_s[4] + d3 * svd_s[8];
|
|
2308
|
+
nk_f32_t c = trace_ds / ((nk_f32_t)n * variance_a);
|
|
2309
|
+
if (scale) *scale = c;
|
|
2310
|
+
|
|
2311
|
+
if (det < 0) {
|
|
2312
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
2313
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
2314
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
2315
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
2316
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
2317
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
2318
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
2319
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
2320
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
2321
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
2322
|
+
}
|
|
2323
|
+
|
|
2324
|
+
if (rotation)
|
|
2325
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
2326
|
+
|
|
2327
|
+
nk_f32_t sum_squared = nk_transformed_ssd_bf16_skylake_(a, b, n, r, c, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
2328
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
2329
|
+
*result = nk_f32_sqrt_haswell(sum_squared * inv_n);
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
#if defined(__clang__)
|
|
2333
|
+
#pragma clang attribute pop
|
|
2334
|
+
#elif defined(__GNUC__)
|
|
2335
|
+
#pragma GCC pop_options
|
|
2336
|
+
#endif
|
|
2337
|
+
|
|
2338
|
+
#if defined(__cplusplus)
|
|
2339
|
+
} // extern "C"
|
|
2340
|
+
#endif
|
|
2341
|
+
|
|
2342
|
+
#endif // NK_TARGET_SKYLAKE
|
|
2343
|
+
#endif // NK_TARGET_X8664_
|
|
2344
|
+
#endif // NK_MESH_SKYLAKE_H
|