usearch 2.21.3 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -24
- package/binding.gyp +15 -37
- package/c/CMakeLists.txt +4 -5
- package/c/README.md +7 -7
- package/c/lib.cpp +32 -14
- package/c/test.c +83 -1
- package/c/usearch.h +24 -3
- package/cpp/CMakeLists.txt +4 -0
- package/cpp/README.md +5 -5
- package/cpp/bench.cpp +97 -116
- package/cpp/test.cpp +117 -20
- package/include/usearch/index.hpp +235 -52
- package/include/usearch/index_dense.hpp +105 -40
- package/include/usearch/index_plugins.hpp +1387 -204
- package/javascript/dist/cjs/usearch.d.ts +8 -3
- package/javascript/dist/cjs/usearch.js +12 -4
- package/javascript/dist/esm/usearch.d.ts +8 -3
- package/javascript/dist/esm/usearch.js +12 -4
- package/javascript/lib.cpp +17 -0
- package/javascript/usearch.ts +14 -6
- package/numkong/.clang-tidy +30 -0
- package/{simsimd → numkong}/.cmake-format.py +18 -0
- package/numkong/.editorconfig +12 -0
- package/numkong/.git-blame-ignore-revs +123 -0
- package/numkong/.mailmap +15 -0
- package/numkong/.spi.yml +11 -0
- package/numkong/.swift-format +14 -0
- package/numkong/CITATION.cff +81 -0
- package/numkong/CONTRIBUTING.md +500 -0
- package/numkong/Cargo.toml +54 -0
- package/numkong/MANIFEST.in +8 -0
- package/numkong/Package.swift +84 -0
- package/numkong/README.md +569 -0
- package/numkong/bench/README.md +217 -0
- package/numkong/bench/bench-browser.html +323 -0
- package/numkong/bench/bench.cpp +344 -0
- package/numkong/bench/bench.hpp +816 -0
- package/numkong/bench/bench.js +148 -0
- package/numkong/bench/bench.mjs +616 -0
- package/numkong/bench/bench_attention.cpp +113 -0
- package/numkong/bench/bench_cast.cpp +264 -0
- package/numkong/bench/bench_cross_amx.cpp +62 -0
- package/numkong/bench/bench_cross_arm.cpp +245 -0
- package/numkong/bench/bench_cross_blas.cpp +309 -0
- package/numkong/bench/bench_cross_loongarch.cpp +106 -0
- package/numkong/bench/bench_cross_power.cpp +84 -0
- package/numkong/bench/bench_cross_rvv.cpp +121 -0
- package/numkong/bench/bench_cross_serial.cpp +151 -0
- package/numkong/bench/bench_cross_sme.cpp +163 -0
- package/numkong/bench/bench_cross_wasm.cpp +142 -0
- package/numkong/bench/bench_cross_x86.cpp +455 -0
- package/numkong/bench/bench_curved.cpp +210 -0
- package/numkong/bench/bench_dot.cpp +315 -0
- package/numkong/bench/bench_each.cpp +503 -0
- package/numkong/bench/bench_geospatial.cpp +112 -0
- package/numkong/bench/bench_maxsim.cpp +154 -0
- package/numkong/bench/bench_mesh.cpp +148 -0
- package/numkong/bench/bench_probability.cpp +60 -0
- package/numkong/bench/bench_reduce.cpp +371 -0
- package/numkong/bench/bench_set.cpp +73 -0
- package/numkong/bench/bench_sparse.cpp +198 -0
- package/numkong/bench/bench_spatial.cpp +347 -0
- package/numkong/bench/bench_trigonometry.cpp +137 -0
- package/numkong/bench/swift/Bench.swift +506 -0
- package/numkong/binding.gyp +169 -0
- package/numkong/build.rs +484 -0
- package/numkong/c/dispatch.h +512 -0
- package/numkong/c/dispatch_bf16.c +447 -0
- package/numkong/c/dispatch_bf16c.c +52 -0
- package/numkong/c/dispatch_e2m3.c +296 -0
- package/numkong/c/dispatch_e3m2.c +284 -0
- package/numkong/c/dispatch_e4m3.c +318 -0
- package/numkong/c/dispatch_e5m2.c +316 -0
- package/numkong/c/dispatch_f16.c +452 -0
- package/numkong/c/dispatch_f16c.c +58 -0
- package/numkong/c/dispatch_f32.c +434 -0
- package/numkong/c/dispatch_f32c.c +99 -0
- package/numkong/c/dispatch_f64.c +348 -0
- package/numkong/c/dispatch_f64c.c +98 -0
- package/numkong/c/dispatch_i16.c +96 -0
- package/numkong/c/dispatch_i32.c +89 -0
- package/numkong/c/dispatch_i4.c +153 -0
- package/numkong/c/dispatch_i64.c +86 -0
- package/numkong/c/dispatch_i8.c +348 -0
- package/numkong/c/dispatch_other.c +348 -0
- package/numkong/c/dispatch_u1.c +193 -0
- package/numkong/c/dispatch_u16.c +124 -0
- package/numkong/c/dispatch_u32.c +118 -0
- package/numkong/c/dispatch_u4.c +153 -0
- package/numkong/c/dispatch_u64.c +102 -0
- package/numkong/c/dispatch_u8.c +364 -0
- package/numkong/c/numkong.c +940 -0
- package/numkong/cmake/nk_arm_isa_probes.cmake +61 -0
- package/numkong/cmake/nk_isa_probe.cmake +90 -0
- package/numkong/cmake/nk_loongarch_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_math_lib.cmake +46 -0
- package/numkong/cmake/nk_power_isa_probes.cmake +15 -0
- package/numkong/cmake/nk_riscv_isa_probes.cmake +19 -0
- package/numkong/cmake/nk_x86_isa_probes.cmake +35 -0
- package/numkong/cmake/toolchain-aarch64-gnu.cmake +86 -0
- package/numkong/cmake/toolchain-android-arm64.cmake +25 -0
- package/numkong/cmake/toolchain-android-armv7.cmake +25 -0
- package/numkong/cmake/toolchain-loongarch64-gnu.cmake +68 -0
- package/numkong/cmake/toolchain-ppc64le-gnu.cmake +74 -0
- package/numkong/cmake/toolchain-riscv64-gnu.cmake +107 -0
- package/numkong/cmake/toolchain-riscv64-llvm.cmake +114 -0
- package/numkong/cmake/toolchain-wasi.cmake +77 -0
- package/numkong/cmake/toolchain-wasm.cmake +70 -0
- package/numkong/cmake/toolchain-wasm64.cmake +62 -0
- package/numkong/cmake/toolchain-x86_64-llvm.cmake +47 -0
- package/numkong/include/README.md +643 -0
- package/numkong/include/module.modulemap +129 -0
- package/numkong/include/numkong/attention/sapphireamx.h +1363 -0
- package/numkong/include/numkong/attention/sme.h +2075 -0
- package/numkong/include/numkong/attention.h +49 -0
- package/numkong/include/numkong/capabilities.h +930 -0
- package/numkong/include/numkong/cast/README.md +262 -0
- package/numkong/include/numkong/cast/diamond.h +64 -0
- package/numkong/include/numkong/cast/haswell.h +823 -0
- package/numkong/include/numkong/cast/icelake.h +475 -0
- package/numkong/include/numkong/cast/loongsonasx.h +252 -0
- package/numkong/include/numkong/cast/neon.h +1174 -0
- package/numkong/include/numkong/cast/powervsx.h +449 -0
- package/numkong/include/numkong/cast/rvv.h +970 -0
- package/numkong/include/numkong/cast/sapphire.h +262 -0
- package/numkong/include/numkong/cast/serial.h +2336 -0
- package/numkong/include/numkong/cast/skylake.h +915 -0
- package/numkong/include/numkong/cast/v128relaxed.h +537 -0
- package/numkong/include/numkong/cast.h +254 -0
- package/numkong/include/numkong/cast.hpp +44 -0
- package/numkong/include/numkong/curved/README.md +223 -0
- package/numkong/include/numkong/curved/genoa.h +182 -0
- package/numkong/include/numkong/curved/haswell.h +276 -0
- package/numkong/include/numkong/curved/neon.h +329 -0
- package/numkong/include/numkong/curved/neonbfdot.h +211 -0
- package/numkong/include/numkong/curved/rvv.h +305 -0
- package/numkong/include/numkong/curved/serial.h +207 -0
- package/numkong/include/numkong/curved/skylake.h +457 -0
- package/numkong/include/numkong/curved/smef64.h +530 -0
- package/numkong/include/numkong/curved.h +513 -0
- package/numkong/include/numkong/curved.hpp +144 -0
- package/numkong/include/numkong/dot/README.md +442 -0
- package/numkong/include/numkong/dot/alder.h +563 -0
- package/numkong/include/numkong/dot/diamond.h +158 -0
- package/numkong/include/numkong/dot/genoa.h +289 -0
- package/numkong/include/numkong/dot/haswell.h +1723 -0
- package/numkong/include/numkong/dot/icelake.h +997 -0
- package/numkong/include/numkong/dot/loongsonasx.h +671 -0
- package/numkong/include/numkong/dot/neon.h +869 -0
- package/numkong/include/numkong/dot/neonbfdot.h +243 -0
- package/numkong/include/numkong/dot/neonfhm.h +358 -0
- package/numkong/include/numkong/dot/neonfp8.h +323 -0
- package/numkong/include/numkong/dot/neonsdot.h +622 -0
- package/numkong/include/numkong/dot/powervsx.h +752 -0
- package/numkong/include/numkong/dot/rvv.h +722 -0
- package/numkong/include/numkong/dot/rvvbb.h +72 -0
- package/numkong/include/numkong/dot/rvvbf16.h +123 -0
- package/numkong/include/numkong/dot/rvvhalf.h +129 -0
- package/numkong/include/numkong/dot/sapphire.h +141 -0
- package/numkong/include/numkong/dot/serial.h +874 -0
- package/numkong/include/numkong/dot/sierra.h +405 -0
- package/numkong/include/numkong/dot/skylake.h +1116 -0
- package/numkong/include/numkong/dot/sve.h +420 -0
- package/numkong/include/numkong/dot/svebfdot.h +75 -0
- package/numkong/include/numkong/dot/svehalf.h +168 -0
- package/numkong/include/numkong/dot/svesdot.h +90 -0
- package/numkong/include/numkong/dot/v128relaxed.h +1293 -0
- package/numkong/include/numkong/dot.h +1136 -0
- package/numkong/include/numkong/dot.hpp +94 -0
- package/numkong/include/numkong/dots/README.md +496 -0
- package/numkong/include/numkong/dots/alder.h +117 -0
- package/numkong/include/numkong/dots/diamond.h +86 -0
- package/numkong/include/numkong/dots/genoa.h +100 -0
- package/numkong/include/numkong/dots/graniteamx.h +733 -0
- package/numkong/include/numkong/dots/haswell.h +310 -0
- package/numkong/include/numkong/dots/icelake.h +180 -0
- package/numkong/include/numkong/dots/loongsonasx.h +176 -0
- package/numkong/include/numkong/dots/neon.h +123 -0
- package/numkong/include/numkong/dots/neonbfdot.h +59 -0
- package/numkong/include/numkong/dots/neonfhm.h +96 -0
- package/numkong/include/numkong/dots/neonfp8.h +99 -0
- package/numkong/include/numkong/dots/neonsdot.h +144 -0
- package/numkong/include/numkong/dots/powervsx.h +194 -0
- package/numkong/include/numkong/dots/rvv.h +2593 -0
- package/numkong/include/numkong/dots/sapphireamx.h +4017 -0
- package/numkong/include/numkong/dots/serial.h +2867 -0
- package/numkong/include/numkong/dots/sierra.h +100 -0
- package/numkong/include/numkong/dots/skylake.h +207 -0
- package/numkong/include/numkong/dots/sme.h +5041 -0
- package/numkong/include/numkong/dots/smebi32.h +477 -0
- package/numkong/include/numkong/dots/smef64.h +1328 -0
- package/numkong/include/numkong/dots/smehalf.h +47 -0
- package/numkong/include/numkong/dots/v128relaxed.h +317 -0
- package/numkong/include/numkong/dots.h +3110 -0
- package/numkong/include/numkong/dots.hpp +641 -0
- package/numkong/include/numkong/each/README.md +465 -0
- package/numkong/include/numkong/each/haswell.h +1658 -0
- package/numkong/include/numkong/each/icelake.h +272 -0
- package/numkong/include/numkong/each/neon.h +1138 -0
- package/numkong/include/numkong/each/neonbfdot.h +211 -0
- package/numkong/include/numkong/each/neonhalf.h +318 -0
- package/numkong/include/numkong/each/rvv.h +1121 -0
- package/numkong/include/numkong/each/sapphire.h +343 -0
- package/numkong/include/numkong/each/serial.h +282 -0
- package/numkong/include/numkong/each/skylake.h +1562 -0
- package/numkong/include/numkong/each/v128relaxed.h +562 -0
- package/numkong/include/numkong/each.h +2232 -0
- package/numkong/include/numkong/each.hpp +434 -0
- package/numkong/include/numkong/geospatial/README.md +147 -0
- package/numkong/include/numkong/geospatial/haswell.h +633 -0
- package/numkong/include/numkong/geospatial/neon.h +615 -0
- package/numkong/include/numkong/geospatial/rvv.h +701 -0
- package/numkong/include/numkong/geospatial/serial.h +309 -0
- package/numkong/include/numkong/geospatial/skylake.h +613 -0
- package/numkong/include/numkong/geospatial/v128relaxed.h +652 -0
- package/numkong/include/numkong/geospatial.h +453 -0
- package/numkong/include/numkong/geospatial.hpp +235 -0
- package/numkong/include/numkong/matrix.hpp +336 -0
- package/numkong/include/numkong/maxsim/README.md +187 -0
- package/numkong/include/numkong/maxsim/alder.h +511 -0
- package/numkong/include/numkong/maxsim/genoa.h +115 -0
- package/numkong/include/numkong/maxsim/haswell.h +553 -0
- package/numkong/include/numkong/maxsim/icelake.h +480 -0
- package/numkong/include/numkong/maxsim/neonsdot.h +394 -0
- package/numkong/include/numkong/maxsim/sapphireamx.h +877 -0
- package/numkong/include/numkong/maxsim/serial.h +490 -0
- package/numkong/include/numkong/maxsim/sme.h +993 -0
- package/numkong/include/numkong/maxsim/v128relaxed.h +280 -0
- package/numkong/include/numkong/maxsim.h +571 -0
- package/numkong/include/numkong/maxsim.hpp +133 -0
- package/numkong/include/numkong/mesh/README.md +219 -0
- package/numkong/include/numkong/mesh/haswell.h +2013 -0
- package/numkong/include/numkong/mesh/neon.h +1821 -0
- package/numkong/include/numkong/mesh/neonbfdot.h +771 -0
- package/numkong/include/numkong/mesh/rvv.h +936 -0
- package/numkong/include/numkong/mesh/serial.h +721 -0
- package/numkong/include/numkong/mesh/skylake.h +2344 -0
- package/numkong/include/numkong/mesh/v128relaxed.h +1081 -0
- package/numkong/include/numkong/mesh.h +666 -0
- package/numkong/include/numkong/mesh.hpp +701 -0
- package/numkong/include/numkong/numkong.h +78 -0
- package/numkong/include/numkong/numkong.hpp +58 -0
- package/numkong/include/numkong/probability/README.md +177 -0
- package/numkong/include/numkong/probability/haswell.h +267 -0
- package/numkong/include/numkong/probability/neon.h +255 -0
- package/numkong/include/numkong/probability/rvv.h +414 -0
- package/numkong/include/numkong/probability/serial.h +181 -0
- package/numkong/include/numkong/probability/skylake.h +324 -0
- package/numkong/include/numkong/probability.h +380 -0
- package/numkong/include/numkong/probability.hpp +120 -0
- package/numkong/include/numkong/random.h +50 -0
- package/numkong/include/numkong/random.hpp +285 -0
- package/numkong/include/numkong/reduce/README.md +552 -0
- package/numkong/include/numkong/reduce/alder.h +636 -0
- package/numkong/include/numkong/reduce/genoa.h +201 -0
- package/numkong/include/numkong/reduce/haswell.h +3785 -0
- package/numkong/include/numkong/reduce/icelake.h +549 -0
- package/numkong/include/numkong/reduce/neon.h +3980 -0
- package/numkong/include/numkong/reduce/neonbfdot.h +142 -0
- package/numkong/include/numkong/reduce/neonfhm.h +293 -0
- package/numkong/include/numkong/reduce/neonsdot.h +357 -0
- package/numkong/include/numkong/reduce/rvv.h +3533 -0
- package/numkong/include/numkong/reduce/serial.h +753 -0
- package/numkong/include/numkong/reduce/sierra.h +338 -0
- package/numkong/include/numkong/reduce/skylake.h +3794 -0
- package/numkong/include/numkong/reduce/sve.h +52 -0
- package/numkong/include/numkong/reduce/v128relaxed.h +2302 -0
- package/numkong/include/numkong/reduce.h +1582 -0
- package/numkong/include/numkong/reduce.hpp +778 -0
- package/numkong/include/numkong/scalar/README.md +89 -0
- package/numkong/include/numkong/scalar/haswell.h +122 -0
- package/numkong/include/numkong/scalar/loongsonasx.h +74 -0
- package/numkong/include/numkong/scalar/neon.h +122 -0
- package/numkong/include/numkong/scalar/neonhalf.h +70 -0
- package/numkong/include/numkong/scalar/powervsx.h +96 -0
- package/numkong/include/numkong/scalar/rvv.h +211 -0
- package/numkong/include/numkong/scalar/sapphire.h +74 -0
- package/numkong/include/numkong/scalar/serial.h +332 -0
- package/numkong/include/numkong/scalar/v128relaxed.h +56 -0
- package/numkong/include/numkong/scalar.h +696 -0
- package/numkong/include/numkong/set/README.md +179 -0
- package/numkong/include/numkong/set/haswell.h +334 -0
- package/numkong/include/numkong/set/icelake.h +485 -0
- package/numkong/include/numkong/set/loongsonasx.h +181 -0
- package/numkong/include/numkong/set/neon.h +363 -0
- package/numkong/include/numkong/set/powervsx.h +326 -0
- package/numkong/include/numkong/set/rvv.h +226 -0
- package/numkong/include/numkong/set/rvvbb.h +117 -0
- package/numkong/include/numkong/set/serial.h +174 -0
- package/numkong/include/numkong/set/sve.h +187 -0
- package/numkong/include/numkong/set/v128relaxed.h +240 -0
- package/numkong/include/numkong/set.h +471 -0
- package/numkong/include/numkong/set.hpp +114 -0
- package/numkong/include/numkong/sets/README.md +149 -0
- package/numkong/include/numkong/sets/haswell.h +63 -0
- package/numkong/include/numkong/sets/icelake.h +66 -0
- package/numkong/include/numkong/sets/loongsonasx.h +52 -0
- package/numkong/include/numkong/sets/neon.h +61 -0
- package/numkong/include/numkong/sets/powervsx.h +65 -0
- package/numkong/include/numkong/sets/serial.h +43 -0
- package/numkong/include/numkong/sets/smebi32.h +1135 -0
- package/numkong/include/numkong/sets/v128relaxed.h +58 -0
- package/numkong/include/numkong/sets.h +382 -0
- package/numkong/include/numkong/sparse/README.md +156 -0
- package/numkong/include/numkong/sparse/icelake.h +463 -0
- package/numkong/include/numkong/sparse/neon.h +288 -0
- package/numkong/include/numkong/sparse/serial.h +117 -0
- package/numkong/include/numkong/sparse/sve2.h +504 -0
- package/numkong/include/numkong/sparse/turin.h +322 -0
- package/numkong/include/numkong/sparse.h +363 -0
- package/numkong/include/numkong/sparse.hpp +113 -0
- package/numkong/include/numkong/spatial/README.md +439 -0
- package/numkong/include/numkong/spatial/alder.h +604 -0
- package/numkong/include/numkong/spatial/diamond.h +240 -0
- package/numkong/include/numkong/spatial/genoa.h +222 -0
- package/numkong/include/numkong/spatial/haswell.h +979 -0
- package/numkong/include/numkong/spatial/icelake.h +1067 -0
- package/numkong/include/numkong/spatial/loongsonasx.h +483 -0
- package/numkong/include/numkong/spatial/neon.h +846 -0
- package/numkong/include/numkong/spatial/neonbfdot.h +164 -0
- package/numkong/include/numkong/spatial/neonfp8.h +258 -0
- package/numkong/include/numkong/spatial/neonsdot.h +429 -0
- package/numkong/include/numkong/spatial/powervsx.h +738 -0
- package/numkong/include/numkong/spatial/rvv.h +991 -0
- package/numkong/include/numkong/spatial/rvvbf16.h +128 -0
- package/numkong/include/numkong/spatial/rvvhalf.h +120 -0
- package/numkong/include/numkong/spatial/serial.h +347 -0
- package/numkong/include/numkong/spatial/sierra.h +516 -0
- package/numkong/include/numkong/spatial/skylake.h +605 -0
- package/numkong/include/numkong/spatial/sve.h +246 -0
- package/numkong/include/numkong/spatial/svebfdot.h +125 -0
- package/numkong/include/numkong/spatial/svehalf.h +136 -0
- package/numkong/include/numkong/spatial/svesdot.h +143 -0
- package/numkong/include/numkong/spatial/v128relaxed.h +969 -0
- package/numkong/include/numkong/spatial.h +1681 -0
- package/numkong/include/numkong/spatial.hpp +183 -0
- package/numkong/include/numkong/spatials/README.md +580 -0
- package/numkong/include/numkong/spatials/alder.h +94 -0
- package/numkong/include/numkong/spatials/diamond.h +82 -0
- package/numkong/include/numkong/spatials/genoa.h +94 -0
- package/numkong/include/numkong/spatials/graniteamx.h +173 -0
- package/numkong/include/numkong/spatials/haswell.h +219 -0
- package/numkong/include/numkong/spatials/icelake.h +113 -0
- package/numkong/include/numkong/spatials/loongsonasx.h +153 -0
- package/numkong/include/numkong/spatials/neon.h +109 -0
- package/numkong/include/numkong/spatials/neonbfdot.h +60 -0
- package/numkong/include/numkong/spatials/neonfhm.h +92 -0
- package/numkong/include/numkong/spatials/neonfp8.h +111 -0
- package/numkong/include/numkong/spatials/neonsdot.h +143 -0
- package/numkong/include/numkong/spatials/powervsx.h +153 -0
- package/numkong/include/numkong/spatials/rvv.h +1976 -0
- package/numkong/include/numkong/spatials/sapphireamx.h +1149 -0
- package/numkong/include/numkong/spatials/serial.h +248 -0
- package/numkong/include/numkong/spatials/sierra.h +96 -0
- package/numkong/include/numkong/spatials/skylake.h +184 -0
- package/numkong/include/numkong/spatials/sme.h +1927 -0
- package/numkong/include/numkong/spatials/smef64.h +483 -0
- package/numkong/include/numkong/spatials/v128relaxed.h +240 -0
- package/numkong/include/numkong/spatials.h +3374 -0
- package/numkong/include/numkong/spatials.hpp +509 -0
- package/numkong/include/numkong/tensor.hpp +1981 -0
- package/numkong/include/numkong/trigonometry/README.md +184 -0
- package/numkong/include/numkong/trigonometry/haswell.h +653 -0
- package/numkong/include/numkong/trigonometry/neon.h +638 -0
- package/numkong/include/numkong/trigonometry/rvv.h +700 -0
- package/numkong/include/numkong/trigonometry/serial.h +703 -0
- package/numkong/include/numkong/trigonometry/skylake.h +725 -0
- package/numkong/include/numkong/trigonometry/v128relaxed.h +667 -0
- package/numkong/include/numkong/trigonometry.h +467 -0
- package/numkong/include/numkong/trigonometry.hpp +166 -0
- package/numkong/include/numkong/types.h +1695 -0
- package/numkong/include/numkong/types.hpp +6027 -0
- package/numkong/include/numkong/vector.hpp +766 -0
- package/numkong/javascript/@numkong-darwin-arm64/package.json +26 -0
- package/numkong/javascript/@numkong-darwin-x64/package.json +26 -0
- package/numkong/javascript/@numkong-linux-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-linux-x64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-arm64/package.json +25 -0
- package/numkong/javascript/@numkong-win32-x64/package.json +25 -0
- package/numkong/javascript/README.md +254 -0
- package/numkong/javascript/node-gyp-build.d.ts +4 -0
- package/numkong/javascript/numkong-browser.ts +40 -0
- package/numkong/javascript/numkong-wasm.ts +750 -0
- package/numkong/javascript/numkong.c +712 -0
- package/numkong/javascript/numkong.ts +613 -0
- package/{simsimd → numkong}/javascript/tsconfig-base.json +6 -3
- package/numkong/javascript/tsconfig-cjs.json +13 -0
- package/{simsimd → numkong}/javascript/tsconfig-esm.json +1 -1
- package/numkong/javascript/types.ts +784 -0
- package/numkong/numkong.gypi +104 -0
- package/numkong/package.json +108 -0
- package/numkong/probes/arm_neon.c +8 -0
- package/numkong/probes/arm_neon_bfdot.c +9 -0
- package/numkong/probes/arm_neon_fhm.c +9 -0
- package/numkong/probes/arm_neon_half.c +8 -0
- package/numkong/probes/arm_neon_sdot.c +9 -0
- package/numkong/probes/arm_neonfp8.c +9 -0
- package/numkong/probes/arm_sme.c +16 -0
- package/numkong/probes/arm_sme2.c +16 -0
- package/numkong/probes/arm_sme2p1.c +16 -0
- package/numkong/probes/arm_sme_bf16.c +16 -0
- package/numkong/probes/arm_sme_bi32.c +16 -0
- package/numkong/probes/arm_sme_f64.c +16 -0
- package/numkong/probes/arm_sme_fa64.c +14 -0
- package/numkong/probes/arm_sme_half.c +16 -0
- package/numkong/probes/arm_sme_lut2.c +15 -0
- package/numkong/probes/arm_sve.c +18 -0
- package/numkong/probes/arm_sve2.c +20 -0
- package/numkong/probes/arm_sve2p1.c +18 -0
- package/numkong/probes/arm_sve_bfdot.c +20 -0
- package/numkong/probes/arm_sve_half.c +18 -0
- package/numkong/probes/arm_sve_sdot.c +21 -0
- package/numkong/probes/loongarch_lasx.c +12 -0
- package/numkong/probes/power_vsx.c +12 -0
- package/numkong/probes/probe.js +127 -0
- package/numkong/probes/riscv_rvv.c +14 -0
- package/numkong/probes/riscv_rvv_bb.c +15 -0
- package/numkong/probes/riscv_rvv_bf16.c +17 -0
- package/numkong/probes/riscv_rvv_half.c +14 -0
- package/numkong/probes/wasm_v128relaxed.c +11 -0
- package/numkong/probes/x86_alder.c +17 -0
- package/numkong/probes/x86_diamond.c +17 -0
- package/numkong/probes/x86_genoa.c +17 -0
- package/numkong/probes/x86_graniteamx.c +19 -0
- package/numkong/probes/x86_haswell.c +11 -0
- package/numkong/probes/x86_icelake.c +17 -0
- package/numkong/probes/x86_sapphire.c +16 -0
- package/numkong/probes/x86_sapphireamx.c +18 -0
- package/numkong/probes/x86_sierra.c +17 -0
- package/numkong/probes/x86_skylake.c +15 -0
- package/numkong/probes/x86_turin.c +17 -0
- package/numkong/rust/README.md +698 -0
- package/numkong/rust/capabilities.rs +96 -0
- package/numkong/rust/cast.rs +314 -0
- package/numkong/rust/curved.rs +391 -0
- package/numkong/rust/each.rs +2721 -0
- package/numkong/rust/geospatial.rs +343 -0
- package/numkong/rust/matrix.rs +5424 -0
- package/numkong/rust/maxsim.rs +540 -0
- package/numkong/rust/mesh.rs +837 -0
- package/numkong/rust/numkong.rs +409 -0
- package/numkong/rust/probability.rs +272 -0
- package/numkong/rust/reduce.rs +1249 -0
- package/numkong/rust/set.rs +194 -0
- package/numkong/rust/sparse.rs +513 -0
- package/numkong/rust/spatial.rs +1815 -0
- package/numkong/rust/tensor.rs +7565 -0
- package/numkong/rust/types.rs +3666 -0
- package/numkong/rust/vector.rs +1899 -0
- package/numkong/rustfmt.toml +2 -0
- package/numkong/swift/Matrix.swift +1388 -0
- package/numkong/swift/MaxSim.swift +151 -0
- package/numkong/swift/NumKong.swift +283 -0
- package/numkong/swift/Numerics.swift +329 -0
- package/numkong/swift/README.md +433 -0
- package/numkong/swift/Scalar.swift +741 -0
- package/numkong/swift/Tensor.swift +209 -0
- package/numkong/test/README.md +315 -0
- package/numkong/test/playwright.config.ts +16 -0
- package/numkong/test/swift/Test.swift +534 -0
- package/numkong/test/test-browser-bundle-runner.mjs +36 -0
- package/numkong/test/test-browser-bundle.html +188 -0
- package/numkong/test/test-browser-runner.mjs +43 -0
- package/numkong/test/test-browser.html +305 -0
- package/numkong/test/test-dtypes.mjs +453 -0
- package/numkong/test/test-wasm.mjs +267 -0
- package/numkong/test/test.cpp +442 -0
- package/numkong/test/test.hpp +580 -0
- package/numkong/test/test.mjs +363 -0
- package/numkong/test/test_base.py +1041 -0
- package/numkong/test/test_cast.cpp +242 -0
- package/numkong/test/test_cross.hpp +551 -0
- package/numkong/test/test_cross.py +467 -0
- package/numkong/test/test_cross_amx.cpp +50 -0
- package/numkong/test/test_cross_arm.cpp +225 -0
- package/numkong/test/test_cross_blas.cpp +354 -0
- package/numkong/test/test_cross_loongarch.cpp +90 -0
- package/numkong/test/test_cross_power.cpp +73 -0
- package/numkong/test/test_cross_rvv.cpp +109 -0
- package/numkong/test/test_cross_serial.cpp +181 -0
- package/numkong/test/test_cross_sme.cpp +148 -0
- package/numkong/test/test_cross_wasm.cpp +141 -0
- package/numkong/test/test_cross_x86.cpp +442 -0
- package/numkong/test/test_curved.cpp +182 -0
- package/numkong/test/test_curved.py +207 -0
- package/numkong/test/test_dot.cpp +353 -0
- package/numkong/test/test_dot.py +296 -0
- package/numkong/test/test_each.cpp +437 -0
- package/numkong/test/test_each.py +756 -0
- package/numkong/test/test_geospatial.cpp +155 -0
- package/numkong/test/test_geospatial.py +241 -0
- package/numkong/test/test_maxsim.cpp +138 -0
- package/numkong/test/test_maxsim.py +209 -0
- package/numkong/test/test_mesh.cpp +203 -0
- package/numkong/test/test_mesh.py +394 -0
- package/numkong/test/test_probability.cpp +136 -0
- package/numkong/test/test_probability.py +166 -0
- package/numkong/test/test_reduce.cpp +563 -0
- package/numkong/test/test_reduce.py +781 -0
- package/numkong/test/test_set.cpp +151 -0
- package/numkong/test/test_set.py +120 -0
- package/numkong/test/test_similarities.py +579 -0
- package/numkong/test/test_sparse.cpp +121 -0
- package/numkong/test/test_sparse.py +118 -0
- package/numkong/test/test_spatial.cpp +518 -0
- package/numkong/test/test_spatial.py +269 -0
- package/numkong/test/test_tensor.cpp +884 -0
- package/numkong/test/test_tensor.py +1417 -0
- package/numkong/test/test_trigonometry.cpp +156 -0
- package/numkong/test/test_trigonometry.py +180 -0
- package/package.json +7 -8
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/simsimd/.git-blame-ignore-revs +0 -2
- package/simsimd/CONTRIBUTING.md +0 -321
- package/simsimd/Cargo.toml +0 -49
- package/simsimd/MANIFEST.in +0 -4
- package/simsimd/Package.swift +0 -28
- package/simsimd/README.md +0 -1493
- package/simsimd/binding.gyp +0 -35
- package/simsimd/build.rs +0 -70
- package/simsimd/c/lib.c +0 -374
- package/simsimd/include/module.modulemap +0 -4
- package/simsimd/include/simsimd/binary.h +0 -486
- package/simsimd/include/simsimd/curved.h +0 -1541
- package/simsimd/include/simsimd/dot.h +0 -1853
- package/simsimd/include/simsimd/elementwise.h +0 -2465
- package/simsimd/include/simsimd/geospatial.h +0 -43
- package/simsimd/include/simsimd/mesh.h +0 -69
- package/simsimd/include/simsimd/probability.h +0 -606
- package/simsimd/include/simsimd/simsimd.h +0 -2567
- package/simsimd/include/simsimd/sparse.h +0 -1384
- package/simsimd/include/simsimd/spatial.h +0 -2335
- package/simsimd/include/simsimd/types.h +0 -666
- package/simsimd/javascript/fallback.ts +0 -233
- package/simsimd/javascript/lib.c +0 -125
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -179
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/package-lock.json +0 -1761
- package/simsimd/package.json +0 -61
- package/simsimd/rust/lib.rs +0 -1821
- package/simsimd/scripts/bench.cxx +0 -1170
- package/simsimd/scripts/bench.ipynb +0 -187
- package/simsimd/scripts/bench.js +0 -127
- package/simsimd/scripts/bench.rs +0 -876
- package/simsimd/scripts/bench_vectors.py +0 -734
- package/simsimd/scripts/bench_vectors_live.py +0 -213
- package/simsimd/scripts/test.c +0 -187
- package/simsimd/scripts/test.mjs +0 -220
- package/simsimd/scripts/test.py +0 -1505
- package/simsimd/swift/SimSIMD.swift +0 -149
- package/simsimd/swift/Test.swift +0 -128
- /package/{simsimd → numkong}/CODE_OF_CONDUCT.md +0 -0
- /package/{simsimd → numkong}/LICENSE +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-cjs.json +0 -0
- /package/{simsimd → numkong}/javascript/dist-package-esm.json +0 -0
|
@@ -0,0 +1,1821 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @brief SIMD-accelerated Point Cloud Alignment for NEON.
|
|
3
|
+
* @file include/numkong/mesh/neon.h
|
|
4
|
+
* @author Ash Vardanian
|
|
5
|
+
* @date December 27, 2025
|
|
6
|
+
*
|
|
7
|
+
* @sa include/numkong/mesh.h
|
|
8
|
+
*
|
|
9
|
+
* @section neon_mesh_instructions Key NEON Mesh Instructions
|
|
10
|
+
*
|
|
11
|
+
* Point cloud operations use these ARM NEON instructions:
|
|
12
|
+
*
|
|
13
|
+
* Intrinsic Instruction A76 M5
|
|
14
|
+
* vfmaq_f32 FMLA (V.4S, V.4S, V.4S) 4cy @ 2p 3cy @ 4p
|
|
15
|
+
* vmulq_n_f32 FMUL (V.4S, V.4S, V.S[0]) 3cy @ 2p 3cy @ 4p
|
|
16
|
+
* vsubq_f32 FSUB (V.4S, V.4S, V.4S) 2cy @ 2p 2cy @ 4p
|
|
17
|
+
* vaddvq_f32 FADDP+FADDP (reduce) 5cy @ 1p 8cy @ 1p
|
|
18
|
+
* vld3q_f32 LD3 ({Vt.4S, Vt2.4S, Vt3.4S}) 4cy @ 1p 4cy @ 1p
|
|
19
|
+
*
|
|
20
|
+
* LD3 provides hardware stride-3 deinterleaving for XYZ point data. The 6cy latency and
|
|
21
|
+
* 1/cy throughput make it the memory bottleneck regardless of core microarchitecture.
|
|
22
|
+
*
|
|
23
|
+
* FMA throughput doubles on 4-pipe cores (Apple M4+, Graviton3+, Oryon). Using 2x loop
|
|
24
|
+
* unrolling with independent accumulators hides FMA latency and saturates 2 FP pipes on
|
|
25
|
+
* A76-class cores; 4x unrolling may further benefit 4-pipe cores.
|
|
26
|
+
*/
|
|
27
|
+
#ifndef NK_MESH_NEON_H
|
|
28
|
+
#define NK_MESH_NEON_H
|
|
29
|
+
|
|
30
|
+
#if NK_TARGET_ARM64_
|
|
31
|
+
#if NK_TARGET_NEON
|
|
32
|
+
|
|
33
|
+
#include "numkong/types.h"
|
|
34
|
+
#include "numkong/dot/neon.h"
|
|
35
|
+
#include "numkong/mesh/serial.h"
|
|
36
|
+
#include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`, `nk_f64_sqrt_neon`
|
|
37
|
+
|
|
38
|
+
#if defined(__cplusplus)
|
|
39
|
+
extern "C" {
|
|
40
|
+
#endif
|
|
41
|
+
|
|
42
|
+
#if defined(__clang__)
|
|
43
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8-a+simd"))), apply_to = function)
|
|
44
|
+
#elif defined(__GNUC__)
|
|
45
|
+
#pragma GCC push_options
|
|
46
|
+
#pragma GCC target("arch=armv8-a+simd")
|
|
47
|
+
#endif
|
|
48
|
+
|
|
49
|
+
NK_INTERNAL void nk_deinterleave_f32x4_neon_(nk_f32_t const *ptr, float32x4_t *x_out, float32x4_t *y_out,
|
|
50
|
+
float32x4_t *z_out) {
|
|
51
|
+
// Deinterleave 12 floats (4 xyz triplets) into separate x, y, z vectors.
|
|
52
|
+
// Uses NEON vld3q for efficient stride-3 deinterleaving.
|
|
53
|
+
//
|
|
54
|
+
// Input: 12 contiguous floats [x0,y0,z0, x1,y1,z1, x2,y2,z2, x3,y3,z3]
|
|
55
|
+
// Output: x[4], y[4], z[4] vectors
|
|
56
|
+
float32x4x3_t xyz_f32x4x3 = vld3q_f32(ptr);
|
|
57
|
+
*x_out = xyz_f32x4x3.val[0];
|
|
58
|
+
*y_out = xyz_f32x4x3.val[1];
|
|
59
|
+
*z_out = xyz_f32x4x3.val[2];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
NK_INTERNAL void nk_deinterleave_f64x2_neon_(nk_f64_t const *ptr, float64x2_t *x_out, float64x2_t *y_out,
|
|
63
|
+
float64x2_t *z_out) {
|
|
64
|
+
// Deinterleave 6 f64 values (2 xyz triplets) into separate x, y, z vectors.
|
|
65
|
+
//
|
|
66
|
+
// Input: 6 contiguous f64 [x0,y0,z0, x1,y1,z1]
|
|
67
|
+
// Output: x[2], y[2], z[2] vectors
|
|
68
|
+
// NEON doesn't have vld3q_f64, so we use vcombine to avoid stack round-trips
|
|
69
|
+
// Load 2 xyz triplets: [x0,y0,z0, x1,y1,z1]
|
|
70
|
+
*x_out = vcombine_f64(vld1_f64(&ptr[0]), vld1_f64(&ptr[3]));
|
|
71
|
+
*y_out = vcombine_f64(vld1_f64(&ptr[1]), vld1_f64(&ptr[4]));
|
|
72
|
+
*z_out = vcombine_f64(vld1_f64(&ptr[2]), vld1_f64(&ptr[5]));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
NK_INTERNAL nk_f64_t nk_reduce_stable_f64x2_neon_(float64x2_t values_f64x2) {
|
|
76
|
+
nk_b128_vec_t values;
|
|
77
|
+
values.f64x2 = values_f64x2;
|
|
78
|
+
nk_f64_t sum = 0.0, compensation = 0.0;
|
|
79
|
+
nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[0]);
|
|
80
|
+
nk_accumulate_sum_f64_(&sum, &compensation, values.f64s[1]);
|
|
81
|
+
return sum + compensation;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
NK_INTERNAL void nk_rotation_from_svd_f64_neon_(nk_f64_t const *svd_u, nk_f64_t const *svd_v, nk_f64_t *rotation) {
|
|
85
|
+
nk_rotation_from_svd_f64_serial_(svd_u, svd_v, rotation);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
NK_INTERNAL void nk_accumulate_square_f64x2_neon_(float64x2_t *sum_f64x2, float64x2_t *compensation_f64x2,
|
|
89
|
+
float64x2_t values_f64x2) {
|
|
90
|
+
float64x2_t product_f64x2 = vmulq_f64(values_f64x2, values_f64x2);
|
|
91
|
+
float64x2_t product_error_f64x2 = vfmaq_f64(vnegq_f64(product_f64x2), values_f64x2, values_f64x2);
|
|
92
|
+
float64x2_t tentative_sum_f64x2 = vaddq_f64(*sum_f64x2, product_f64x2);
|
|
93
|
+
float64x2_t virtual_addend_f64x2 = vsubq_f64(tentative_sum_f64x2, *sum_f64x2);
|
|
94
|
+
float64x2_t sum_error_f64x2 = vaddq_f64(vsubq_f64(*sum_f64x2, vsubq_f64(tentative_sum_f64x2, virtual_addend_f64x2)),
|
|
95
|
+
vsubq_f64(product_f64x2, virtual_addend_f64x2));
|
|
96
|
+
*sum_f64x2 = tentative_sum_f64x2;
|
|
97
|
+
*compensation_f64x2 = vaddq_f64(*compensation_f64x2, vaddq_f64(sum_error_f64x2, product_error_f64x2));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
NK_INTERNAL nk_f64_t nk_transformed_ssd_f32_neon_( //
|
|
101
|
+
nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t const *r, nk_f64_t scale, nk_f64_t centroid_a_x,
|
|
102
|
+
nk_f64_t centroid_a_y, nk_f64_t centroid_a_z, nk_f64_t centroid_b_x, nk_f64_t centroid_b_y, nk_f64_t centroid_b_z) {
|
|
103
|
+
float64x2_t scaled_rotation_x_x_f64x2 = vdupq_n_f64(scale * r[0]);
|
|
104
|
+
float64x2_t scaled_rotation_x_y_f64x2 = vdupq_n_f64(scale * r[1]);
|
|
105
|
+
float64x2_t scaled_rotation_x_z_f64x2 = vdupq_n_f64(scale * r[2]);
|
|
106
|
+
float64x2_t scaled_rotation_y_x_f64x2 = vdupq_n_f64(scale * r[3]);
|
|
107
|
+
float64x2_t scaled_rotation_y_y_f64x2 = vdupq_n_f64(scale * r[4]);
|
|
108
|
+
float64x2_t scaled_rotation_y_z_f64x2 = vdupq_n_f64(scale * r[5]);
|
|
109
|
+
float64x2_t scaled_rotation_z_x_f64x2 = vdupq_n_f64(scale * r[6]);
|
|
110
|
+
float64x2_t scaled_rotation_z_y_f64x2 = vdupq_n_f64(scale * r[7]);
|
|
111
|
+
float64x2_t scaled_rotation_z_z_f64x2 = vdupq_n_f64(scale * r[8]);
|
|
112
|
+
float64x2_t centroid_a_x_f64x2 = vdupq_n_f64(centroid_a_x), centroid_a_y_f64x2 = vdupq_n_f64(centroid_a_y);
|
|
113
|
+
float64x2_t centroid_a_z_f64x2 = vdupq_n_f64(centroid_a_z), centroid_b_x_f64x2 = vdupq_n_f64(centroid_b_x);
|
|
114
|
+
float64x2_t centroid_b_y_f64x2 = vdupq_n_f64(centroid_b_y), centroid_b_z_f64x2 = vdupq_n_f64(centroid_b_z);
|
|
115
|
+
float64x2_t sum_squared_low_f64x2 = vdupq_n_f64(0.0), sum_squared_high_f64x2 = vdupq_n_f64(0.0);
|
|
116
|
+
nk_size_t index = 0;
|
|
117
|
+
|
|
118
|
+
for (; index + 4 <= n; index += 4) {
|
|
119
|
+
float32x4_t a_x_f32x4, a_y_f32x4, a_z_f32x4, b_x_f32x4, b_y_f32x4, b_z_f32x4;
|
|
120
|
+
nk_deinterleave_f32x4_neon_(a + index * 3, &a_x_f32x4, &a_y_f32x4, &a_z_f32x4),
|
|
121
|
+
nk_deinterleave_f32x4_neon_(b + index * 3, &b_x_f32x4, &b_y_f32x4, &b_z_f32x4);
|
|
122
|
+
|
|
123
|
+
float64x2_t centered_a_x_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(a_x_f32x4)), centroid_a_x_f64x2);
|
|
124
|
+
float64x2_t centered_a_x_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(a_x_f32x4), centroid_a_x_f64x2);
|
|
125
|
+
float64x2_t centered_a_y_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(a_y_f32x4)), centroid_a_y_f64x2);
|
|
126
|
+
float64x2_t centered_a_y_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(a_y_f32x4), centroid_a_y_f64x2);
|
|
127
|
+
float64x2_t centered_a_z_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(a_z_f32x4)), centroid_a_z_f64x2);
|
|
128
|
+
float64x2_t centered_a_z_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(a_z_f32x4), centroid_a_z_f64x2);
|
|
129
|
+
float64x2_t centered_b_x_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(b_x_f32x4)), centroid_b_x_f64x2);
|
|
130
|
+
float64x2_t centered_b_x_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(b_x_f32x4), centroid_b_x_f64x2);
|
|
131
|
+
float64x2_t centered_b_y_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(b_y_f32x4)), centroid_b_y_f64x2);
|
|
132
|
+
float64x2_t centered_b_y_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(b_y_f32x4), centroid_b_y_f64x2);
|
|
133
|
+
float64x2_t centered_b_z_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(b_z_f32x4)), centroid_b_z_f64x2);
|
|
134
|
+
float64x2_t centered_b_z_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(b_z_f32x4), centroid_b_z_f64x2);
|
|
135
|
+
|
|
136
|
+
float64x2_t rotated_a_x_low_f64x2 = vfmaq_f64(
|
|
137
|
+
vfmaq_f64(vmulq_f64(scaled_rotation_x_x_f64x2, centered_a_x_low_f64x2), scaled_rotation_x_y_f64x2,
|
|
138
|
+
centered_a_y_low_f64x2),
|
|
139
|
+
scaled_rotation_x_z_f64x2, centered_a_z_low_f64x2);
|
|
140
|
+
float64x2_t rotated_a_x_high_f64x2 = vfmaq_f64(
|
|
141
|
+
vfmaq_f64(vmulq_f64(scaled_rotation_x_x_f64x2, centered_a_x_high_f64x2), scaled_rotation_x_y_f64x2,
|
|
142
|
+
centered_a_y_high_f64x2),
|
|
143
|
+
scaled_rotation_x_z_f64x2, centered_a_z_high_f64x2);
|
|
144
|
+
float64x2_t rotated_a_y_low_f64x2 = vfmaq_f64(
|
|
145
|
+
vfmaq_f64(vmulq_f64(scaled_rotation_y_x_f64x2, centered_a_x_low_f64x2), scaled_rotation_y_y_f64x2,
|
|
146
|
+
centered_a_y_low_f64x2),
|
|
147
|
+
scaled_rotation_y_z_f64x2, centered_a_z_low_f64x2);
|
|
148
|
+
float64x2_t rotated_a_y_high_f64x2 = vfmaq_f64(
|
|
149
|
+
vfmaq_f64(vmulq_f64(scaled_rotation_y_x_f64x2, centered_a_x_high_f64x2), scaled_rotation_y_y_f64x2,
|
|
150
|
+
centered_a_y_high_f64x2),
|
|
151
|
+
scaled_rotation_y_z_f64x2, centered_a_z_high_f64x2);
|
|
152
|
+
float64x2_t rotated_a_z_low_f64x2 = vfmaq_f64(
|
|
153
|
+
vfmaq_f64(vmulq_f64(scaled_rotation_z_x_f64x2, centered_a_x_low_f64x2), scaled_rotation_z_y_f64x2,
|
|
154
|
+
centered_a_y_low_f64x2),
|
|
155
|
+
scaled_rotation_z_z_f64x2, centered_a_z_low_f64x2);
|
|
156
|
+
float64x2_t rotated_a_z_high_f64x2 = vfmaq_f64(
|
|
157
|
+
vfmaq_f64(vmulq_f64(scaled_rotation_z_x_f64x2, centered_a_x_high_f64x2), scaled_rotation_z_y_f64x2,
|
|
158
|
+
centered_a_y_high_f64x2),
|
|
159
|
+
scaled_rotation_z_z_f64x2, centered_a_z_high_f64x2);
|
|
160
|
+
|
|
161
|
+
float64x2_t delta_x_low_f64x2 = vsubq_f64(rotated_a_x_low_f64x2, centered_b_x_low_f64x2);
|
|
162
|
+
float64x2_t delta_x_high_f64x2 = vsubq_f64(rotated_a_x_high_f64x2, centered_b_x_high_f64x2);
|
|
163
|
+
float64x2_t delta_y_low_f64x2 = vsubq_f64(rotated_a_y_low_f64x2, centered_b_y_low_f64x2);
|
|
164
|
+
float64x2_t delta_y_high_f64x2 = vsubq_f64(rotated_a_y_high_f64x2, centered_b_y_high_f64x2);
|
|
165
|
+
float64x2_t delta_z_low_f64x2 = vsubq_f64(rotated_a_z_low_f64x2, centered_b_z_low_f64x2);
|
|
166
|
+
float64x2_t delta_z_high_f64x2 = vsubq_f64(rotated_a_z_high_f64x2, centered_b_z_high_f64x2);
|
|
167
|
+
|
|
168
|
+
sum_squared_low_f64x2 = vfmaq_f64(sum_squared_low_f64x2, delta_x_low_f64x2, delta_x_low_f64x2),
|
|
169
|
+
sum_squared_high_f64x2 = vfmaq_f64(sum_squared_high_f64x2, delta_x_high_f64x2, delta_x_high_f64x2);
|
|
170
|
+
sum_squared_low_f64x2 = vfmaq_f64(sum_squared_low_f64x2, delta_y_low_f64x2, delta_y_low_f64x2),
|
|
171
|
+
sum_squared_high_f64x2 = vfmaq_f64(sum_squared_high_f64x2, delta_y_high_f64x2, delta_y_high_f64x2);
|
|
172
|
+
sum_squared_low_f64x2 = vfmaq_f64(sum_squared_low_f64x2, delta_z_low_f64x2, delta_z_low_f64x2),
|
|
173
|
+
sum_squared_high_f64x2 = vfmaq_f64(sum_squared_high_f64x2, delta_z_high_f64x2, delta_z_high_f64x2);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
nk_f64_t sum_squared = vaddvq_f64(vaddq_f64(sum_squared_low_f64x2, sum_squared_high_f64x2));
|
|
177
|
+
for (; index < n; ++index) {
|
|
178
|
+
nk_f64_t centered_a_x = (nk_f64_t)a[index * 3 + 0] - centroid_a_x,
|
|
179
|
+
centered_a_y = (nk_f64_t)a[index * 3 + 1] - centroid_a_y,
|
|
180
|
+
centered_a_z = (nk_f64_t)a[index * 3 + 2] - centroid_a_z;
|
|
181
|
+
nk_f64_t centered_b_x = (nk_f64_t)b[index * 3 + 0] - centroid_b_x,
|
|
182
|
+
centered_b_y = (nk_f64_t)b[index * 3 + 1] - centroid_b_y,
|
|
183
|
+
centered_b_z = (nk_f64_t)b[index * 3 + 2] - centroid_b_z;
|
|
184
|
+
nk_f64_t rotated_a_x = scale * (r[0] * centered_a_x + r[1] * centered_a_y + r[2] * centered_a_z),
|
|
185
|
+
rotated_a_y = scale * (r[3] * centered_a_x + r[4] * centered_a_y + r[5] * centered_a_z),
|
|
186
|
+
rotated_a_z = scale * (r[6] * centered_a_x + r[7] * centered_a_y + r[8] * centered_a_z);
|
|
187
|
+
nk_f64_t delta_x = rotated_a_x - centered_b_x, delta_y = rotated_a_y - centered_b_y,
|
|
188
|
+
delta_z = rotated_a_z - centered_b_z;
|
|
189
|
+
sum_squared += delta_x * delta_x + delta_y * delta_y + delta_z * delta_z;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return sum_squared;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/* Compute sum of squared distances for f64 after applying rotation (and optional scale).
|
|
196
|
+
*
|
|
197
|
+
* Optimization: 2x loop unrolling with multiple accumulators hides FMA latency (3-7 cycles).
|
|
198
|
+
*/
|
|
199
|
+
NK_INTERNAL nk_f64_t nk_transformed_ssd_f64_neon_(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t const *r,
|
|
200
|
+
nk_f64_t scale, nk_f64_t centroid_a_x, nk_f64_t centroid_a_y,
|
|
201
|
+
nk_f64_t centroid_a_z, nk_f64_t centroid_b_x, nk_f64_t centroid_b_y,
|
|
202
|
+
nk_f64_t centroid_b_z) {
|
|
203
|
+
// Broadcast scaled rotation matrix elements
|
|
204
|
+
float64x2_t scaled_rotation_x_x_f64x2 = vdupq_n_f64(scale * r[0]);
|
|
205
|
+
float64x2_t scaled_rotation_x_y_f64x2 = vdupq_n_f64(scale * r[1]);
|
|
206
|
+
float64x2_t scaled_rotation_x_z_f64x2 = vdupq_n_f64(scale * r[2]);
|
|
207
|
+
float64x2_t scaled_rotation_y_x_f64x2 = vdupq_n_f64(scale * r[3]);
|
|
208
|
+
float64x2_t scaled_rotation_y_y_f64x2 = vdupq_n_f64(scale * r[4]);
|
|
209
|
+
float64x2_t scaled_rotation_y_z_f64x2 = vdupq_n_f64(scale * r[5]);
|
|
210
|
+
float64x2_t scaled_rotation_z_x_f64x2 = vdupq_n_f64(scale * r[6]);
|
|
211
|
+
float64x2_t scaled_rotation_z_y_f64x2 = vdupq_n_f64(scale * r[7]);
|
|
212
|
+
float64x2_t scaled_rotation_z_z_f64x2 = vdupq_n_f64(scale * r[8]);
|
|
213
|
+
|
|
214
|
+
// Broadcast centroids
|
|
215
|
+
float64x2_t centroid_a_x_f64x2 = vdupq_n_f64(centroid_a_x);
|
|
216
|
+
float64x2_t centroid_a_y_f64x2 = vdupq_n_f64(centroid_a_y);
|
|
217
|
+
float64x2_t centroid_a_z_f64x2 = vdupq_n_f64(centroid_a_z);
|
|
218
|
+
float64x2_t centroid_b_x_f64x2 = vdupq_n_f64(centroid_b_x);
|
|
219
|
+
float64x2_t centroid_b_y_f64x2 = vdupq_n_f64(centroid_b_y);
|
|
220
|
+
float64x2_t centroid_b_z_f64x2 = vdupq_n_f64(centroid_b_z);
|
|
221
|
+
|
|
222
|
+
// Two independent accumulators to hide FMA latency
|
|
223
|
+
float64x2_t sum_squared_a_f64x2 = vdupq_n_f64(0), sum_squared_a_compensation_f64x2 = vdupq_n_f64(0);
|
|
224
|
+
float64x2_t sum_squared_b_f64x2 = vdupq_n_f64(0), sum_squared_b_compensation_f64x2 = vdupq_n_f64(0);
|
|
225
|
+
nk_size_t j = 0;
|
|
226
|
+
|
|
227
|
+
// Main loop: process 4 points per iteration (2x unrolled, 2 points per batch)
|
|
228
|
+
for (; j + 4 <= n; j += 4) {
|
|
229
|
+
// First batch of 2 points
|
|
230
|
+
float64x2_t a1_x_f64x2, a1_y_f64x2, a1_z_f64x2, b1_x_f64x2, b1_y_f64x2, b1_z_f64x2;
|
|
231
|
+
nk_deinterleave_f64x2_neon_(a + j * 3, &a1_x_f64x2, &a1_y_f64x2, &a1_z_f64x2);
|
|
232
|
+
nk_deinterleave_f64x2_neon_(b + j * 3, &b1_x_f64x2, &b1_y_f64x2, &b1_z_f64x2);
|
|
233
|
+
|
|
234
|
+
// Second batch of 2 points
|
|
235
|
+
float64x2_t a2_x_f64x2, a2_y_f64x2, a2_z_f64x2, b2_x_f64x2, b2_y_f64x2, b2_z_f64x2;
|
|
236
|
+
nk_deinterleave_f64x2_neon_(a + (j + 2) * 3, &a2_x_f64x2, &a2_y_f64x2, &a2_z_f64x2);
|
|
237
|
+
nk_deinterleave_f64x2_neon_(b + (j + 2) * 3, &b2_x_f64x2, &b2_y_f64x2, &b2_z_f64x2);
|
|
238
|
+
|
|
239
|
+
// Center first batch
|
|
240
|
+
float64x2_t centered_a1_x_f64x2 = vsubq_f64(a1_x_f64x2, centroid_a_x_f64x2);
|
|
241
|
+
float64x2_t centered_a1_y_f64x2 = vsubq_f64(a1_y_f64x2, centroid_a_y_f64x2);
|
|
242
|
+
float64x2_t centered_a1_z_f64x2 = vsubq_f64(a1_z_f64x2, centroid_a_z_f64x2);
|
|
243
|
+
float64x2_t centered_b1_x_f64x2 = vsubq_f64(b1_x_f64x2, centroid_b_x_f64x2);
|
|
244
|
+
float64x2_t centered_b1_y_f64x2 = vsubq_f64(b1_y_f64x2, centroid_b_y_f64x2);
|
|
245
|
+
float64x2_t centered_b1_z_f64x2 = vsubq_f64(b1_z_f64x2, centroid_b_z_f64x2);
|
|
246
|
+
|
|
247
|
+
// Center second batch
|
|
248
|
+
float64x2_t centered_a2_x_f64x2 = vsubq_f64(a2_x_f64x2, centroid_a_x_f64x2);
|
|
249
|
+
float64x2_t centered_a2_y_f64x2 = vsubq_f64(a2_y_f64x2, centroid_a_y_f64x2);
|
|
250
|
+
float64x2_t centered_a2_z_f64x2 = vsubq_f64(a2_z_f64x2, centroid_a_z_f64x2);
|
|
251
|
+
float64x2_t centered_b2_x_f64x2 = vsubq_f64(b2_x_f64x2, centroid_b_x_f64x2);
|
|
252
|
+
float64x2_t centered_b2_y_f64x2 = vsubq_f64(b2_y_f64x2, centroid_b_y_f64x2);
|
|
253
|
+
float64x2_t centered_b2_z_f64x2 = vsubq_f64(b2_z_f64x2, centroid_b_z_f64x2);
|
|
254
|
+
|
|
255
|
+
// Rotate and scale first batch
|
|
256
|
+
float64x2_t rotated_a1_x_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_x_x_f64x2, centered_a1_x_f64x2),
|
|
257
|
+
scaled_rotation_x_y_f64x2, centered_a1_y_f64x2),
|
|
258
|
+
scaled_rotation_x_z_f64x2, centered_a1_z_f64x2);
|
|
259
|
+
float64x2_t rotated_a1_y_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_y_x_f64x2, centered_a1_x_f64x2),
|
|
260
|
+
scaled_rotation_y_y_f64x2, centered_a1_y_f64x2),
|
|
261
|
+
scaled_rotation_y_z_f64x2, centered_a1_z_f64x2);
|
|
262
|
+
float64x2_t rotated_a1_z_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_z_x_f64x2, centered_a1_x_f64x2),
|
|
263
|
+
scaled_rotation_z_y_f64x2, centered_a1_y_f64x2),
|
|
264
|
+
scaled_rotation_z_z_f64x2, centered_a1_z_f64x2);
|
|
265
|
+
|
|
266
|
+
// Rotate and scale second batch
|
|
267
|
+
float64x2_t rotated_a2_x_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_x_x_f64x2, centered_a2_x_f64x2),
|
|
268
|
+
scaled_rotation_x_y_f64x2, centered_a2_y_f64x2),
|
|
269
|
+
scaled_rotation_x_z_f64x2, centered_a2_z_f64x2);
|
|
270
|
+
float64x2_t rotated_a2_y_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_y_x_f64x2, centered_a2_x_f64x2),
|
|
271
|
+
scaled_rotation_y_y_f64x2, centered_a2_y_f64x2),
|
|
272
|
+
scaled_rotation_y_z_f64x2, centered_a2_z_f64x2);
|
|
273
|
+
float64x2_t rotated_a2_z_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_z_x_f64x2, centered_a2_x_f64x2),
|
|
274
|
+
scaled_rotation_z_y_f64x2, centered_a2_y_f64x2),
|
|
275
|
+
scaled_rotation_z_z_f64x2, centered_a2_z_f64x2);
|
|
276
|
+
|
|
277
|
+
// Deltas
|
|
278
|
+
float64x2_t delta1_x_f64x2 = vsubq_f64(rotated_a1_x_f64x2, centered_b1_x_f64x2);
|
|
279
|
+
float64x2_t delta1_y_f64x2 = vsubq_f64(rotated_a1_y_f64x2, centered_b1_y_f64x2);
|
|
280
|
+
float64x2_t delta1_z_f64x2 = vsubq_f64(rotated_a1_z_f64x2, centered_b1_z_f64x2);
|
|
281
|
+
float64x2_t delta2_x_f64x2 = vsubq_f64(rotated_a2_x_f64x2, centered_b2_x_f64x2);
|
|
282
|
+
float64x2_t delta2_y_f64x2 = vsubq_f64(rotated_a2_y_f64x2, centered_b2_y_f64x2);
|
|
283
|
+
float64x2_t delta2_z_f64x2 = vsubq_f64(rotated_a2_z_f64x2, centered_b2_z_f64x2);
|
|
284
|
+
|
|
285
|
+
// Accumulate to independent accumulators (interleaved for latency hiding)
|
|
286
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_a_f64x2, &sum_squared_a_compensation_f64x2, delta1_x_f64x2);
|
|
287
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_b_f64x2, &sum_squared_b_compensation_f64x2, delta2_x_f64x2);
|
|
288
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_a_f64x2, &sum_squared_a_compensation_f64x2, delta1_y_f64x2);
|
|
289
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_b_f64x2, &sum_squared_b_compensation_f64x2, delta2_y_f64x2);
|
|
290
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_a_f64x2, &sum_squared_a_compensation_f64x2, delta1_z_f64x2);
|
|
291
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_b_f64x2, &sum_squared_b_compensation_f64x2, delta2_z_f64x2);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// Handle remaining 2 points
|
|
295
|
+
if (j + 2 <= n) {
|
|
296
|
+
float64x2_t a_x_f64x2, a_y_f64x2, a_z_f64x2, b_x_f64x2, b_y_f64x2, b_z_f64x2;
|
|
297
|
+
nk_deinterleave_f64x2_neon_(a + j * 3, &a_x_f64x2, &a_y_f64x2, &a_z_f64x2);
|
|
298
|
+
nk_deinterleave_f64x2_neon_(b + j * 3, &b_x_f64x2, &b_y_f64x2, &b_z_f64x2);
|
|
299
|
+
|
|
300
|
+
float64x2_t centered_a_x_f64x2 = vsubq_f64(a_x_f64x2, centroid_a_x_f64x2);
|
|
301
|
+
float64x2_t centered_a_y_f64x2 = vsubq_f64(a_y_f64x2, centroid_a_y_f64x2);
|
|
302
|
+
float64x2_t centered_a_z_f64x2 = vsubq_f64(a_z_f64x2, centroid_a_z_f64x2);
|
|
303
|
+
float64x2_t centered_b_x_f64x2 = vsubq_f64(b_x_f64x2, centroid_b_x_f64x2);
|
|
304
|
+
float64x2_t centered_b_y_f64x2 = vsubq_f64(b_y_f64x2, centroid_b_y_f64x2);
|
|
305
|
+
float64x2_t centered_b_z_f64x2 = vsubq_f64(b_z_f64x2, centroid_b_z_f64x2);
|
|
306
|
+
|
|
307
|
+
float64x2_t rotated_a_x_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_x_x_f64x2, centered_a_x_f64x2),
|
|
308
|
+
scaled_rotation_x_y_f64x2, centered_a_y_f64x2),
|
|
309
|
+
scaled_rotation_x_z_f64x2, centered_a_z_f64x2);
|
|
310
|
+
float64x2_t rotated_a_y_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_y_x_f64x2, centered_a_x_f64x2),
|
|
311
|
+
scaled_rotation_y_y_f64x2, centered_a_y_f64x2),
|
|
312
|
+
scaled_rotation_y_z_f64x2, centered_a_z_f64x2);
|
|
313
|
+
float64x2_t rotated_a_z_f64x2 = vfmaq_f64(vfmaq_f64(vmulq_f64(scaled_rotation_z_x_f64x2, centered_a_x_f64x2),
|
|
314
|
+
scaled_rotation_z_y_f64x2, centered_a_y_f64x2),
|
|
315
|
+
scaled_rotation_z_z_f64x2, centered_a_z_f64x2);
|
|
316
|
+
|
|
317
|
+
float64x2_t delta_x_f64x2 = vsubq_f64(rotated_a_x_f64x2, centered_b_x_f64x2);
|
|
318
|
+
float64x2_t delta_y_f64x2 = vsubq_f64(rotated_a_y_f64x2, centered_b_y_f64x2);
|
|
319
|
+
float64x2_t delta_z_f64x2 = vsubq_f64(rotated_a_z_f64x2, centered_b_z_f64x2);
|
|
320
|
+
|
|
321
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_a_f64x2, &sum_squared_a_compensation_f64x2, delta_x_f64x2);
|
|
322
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_a_f64x2, &sum_squared_a_compensation_f64x2, delta_y_f64x2);
|
|
323
|
+
nk_accumulate_square_f64x2_neon_(&sum_squared_a_f64x2, &sum_squared_a_compensation_f64x2, delta_z_f64x2);
|
|
324
|
+
j += 2;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// Combine accumulators and reduce
|
|
328
|
+
float64x2_t sum_squared_f64x2 = vaddq_f64(sum_squared_a_f64x2, sum_squared_b_f64x2);
|
|
329
|
+
float64x2_t sum_squared_compensation_f64x2 = vaddq_f64(sum_squared_a_compensation_f64x2,
|
|
330
|
+
sum_squared_b_compensation_f64x2);
|
|
331
|
+
nk_f64_t sum_squared = nk_dot_stable_sum_f64x2_neon_(sum_squared_f64x2, sum_squared_compensation_f64x2);
|
|
332
|
+
nk_f64_t sum_squared_compensation = 0.0;
|
|
333
|
+
|
|
334
|
+
// Scalar tail
|
|
335
|
+
for (; j < n; ++j) {
|
|
336
|
+
nk_f64_t pa_x = a[j * 3 + 0] - centroid_a_x, pa_y = a[j * 3 + 1] - centroid_a_y,
|
|
337
|
+
pa_z = a[j * 3 + 2] - centroid_a_z;
|
|
338
|
+
nk_f64_t pb_x = b[j * 3 + 0] - centroid_b_x, pb_y = b[j * 3 + 1] - centroid_b_y,
|
|
339
|
+
pb_z = b[j * 3 + 2] - centroid_b_z;
|
|
340
|
+
|
|
341
|
+
nk_f64_t ra_x = scale * (r[0] * pa_x + r[1] * pa_y + r[2] * pa_z),
|
|
342
|
+
ra_y = scale * (r[3] * pa_x + r[4] * pa_y + r[5] * pa_z),
|
|
343
|
+
ra_z = scale * (r[6] * pa_x + r[7] * pa_y + r[8] * pa_z);
|
|
344
|
+
|
|
345
|
+
nk_f64_t delta_x = ra_x - pb_x, delta_y = ra_y - pb_y, delta_z = ra_z - pb_z;
|
|
346
|
+
nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_x);
|
|
347
|
+
nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_y);
|
|
348
|
+
nk_accumulate_square_f64_(&sum_squared, &sum_squared_compensation, delta_z);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return sum_squared + sum_squared_compensation;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
NK_PUBLIC void nk_rmsd_f32_neon(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
355
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
|
|
356
|
+
if (a_centroid) a_centroid[0] = 0, a_centroid[1] = 0, a_centroid[2] = 0;
|
|
357
|
+
if (b_centroid) b_centroid[0] = 0, b_centroid[1] = 0, b_centroid[2] = 0;
|
|
358
|
+
if (rotation)
|
|
359
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
360
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
361
|
+
if (scale) *scale = 1.0f;
|
|
362
|
+
|
|
363
|
+
float64x2_t zero_f64x2 = vdupq_n_f64(0.0);
|
|
364
|
+
float64x2_t sum_squared_x_low_f64x2 = zero_f64x2, sum_squared_x_high_f64x2 = zero_f64x2;
|
|
365
|
+
float64x2_t sum_squared_y_low_f64x2 = zero_f64x2, sum_squared_y_high_f64x2 = zero_f64x2;
|
|
366
|
+
float64x2_t sum_squared_z_low_f64x2 = zero_f64x2, sum_squared_z_high_f64x2 = zero_f64x2;
|
|
367
|
+
nk_size_t index = 0;
|
|
368
|
+
|
|
369
|
+
for (; index + 4 <= n; index += 4) {
|
|
370
|
+
float32x4_t a_x_f32x4, a_y_f32x4, a_z_f32x4, b_x_f32x4, b_y_f32x4, b_z_f32x4;
|
|
371
|
+
nk_deinterleave_f32x4_neon_(a + index * 3, &a_x_f32x4, &a_y_f32x4, &a_z_f32x4),
|
|
372
|
+
nk_deinterleave_f32x4_neon_(b + index * 3, &b_x_f32x4, &b_y_f32x4, &b_z_f32x4);
|
|
373
|
+
|
|
374
|
+
float64x2_t delta_x_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(a_x_f32x4)),
|
|
375
|
+
vcvt_f64_f32(vget_low_f32(b_x_f32x4)));
|
|
376
|
+
float64x2_t delta_x_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(a_x_f32x4), vcvt_high_f64_f32(b_x_f32x4));
|
|
377
|
+
float64x2_t delta_y_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(a_y_f32x4)),
|
|
378
|
+
vcvt_f64_f32(vget_low_f32(b_y_f32x4)));
|
|
379
|
+
float64x2_t delta_y_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(a_y_f32x4), vcvt_high_f64_f32(b_y_f32x4));
|
|
380
|
+
float64x2_t delta_z_low_f64x2 = vsubq_f64(vcvt_f64_f32(vget_low_f32(a_z_f32x4)),
|
|
381
|
+
vcvt_f64_f32(vget_low_f32(b_z_f32x4)));
|
|
382
|
+
float64x2_t delta_z_high_f64x2 = vsubq_f64(vcvt_high_f64_f32(a_z_f32x4), vcvt_high_f64_f32(b_z_f32x4));
|
|
383
|
+
|
|
384
|
+
sum_squared_x_low_f64x2 = vfmaq_f64(sum_squared_x_low_f64x2, delta_x_low_f64x2, delta_x_low_f64x2),
|
|
385
|
+
sum_squared_x_high_f64x2 = vfmaq_f64(sum_squared_x_high_f64x2, delta_x_high_f64x2, delta_x_high_f64x2);
|
|
386
|
+
sum_squared_y_low_f64x2 = vfmaq_f64(sum_squared_y_low_f64x2, delta_y_low_f64x2, delta_y_low_f64x2),
|
|
387
|
+
sum_squared_y_high_f64x2 = vfmaq_f64(sum_squared_y_high_f64x2, delta_y_high_f64x2, delta_y_high_f64x2);
|
|
388
|
+
sum_squared_z_low_f64x2 = vfmaq_f64(sum_squared_z_low_f64x2, delta_z_low_f64x2, delta_z_low_f64x2),
|
|
389
|
+
sum_squared_z_high_f64x2 = vfmaq_f64(sum_squared_z_high_f64x2, delta_z_high_f64x2, delta_z_high_f64x2);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
nk_f64_t sum_squared_x = vaddvq_f64(vaddq_f64(sum_squared_x_low_f64x2, sum_squared_x_high_f64x2));
|
|
393
|
+
nk_f64_t sum_squared_y = vaddvq_f64(vaddq_f64(sum_squared_y_low_f64x2, sum_squared_y_high_f64x2));
|
|
394
|
+
nk_f64_t sum_squared_z = vaddvq_f64(vaddq_f64(sum_squared_z_low_f64x2, sum_squared_z_high_f64x2));
|
|
395
|
+
|
|
396
|
+
for (; index < n; ++index) {
|
|
397
|
+
nk_f64_t delta_x = (nk_f64_t)a[index * 3 + 0] - (nk_f64_t)b[index * 3 + 0];
|
|
398
|
+
nk_f64_t delta_y = (nk_f64_t)a[index * 3 + 1] - (nk_f64_t)b[index * 3 + 1];
|
|
399
|
+
nk_f64_t delta_z = (nk_f64_t)a[index * 3 + 2] - (nk_f64_t)b[index * 3 + 2];
|
|
400
|
+
sum_squared_x += delta_x * delta_x, sum_squared_y += delta_y * delta_y, sum_squared_z += delta_z * delta_z;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
*result = nk_f64_sqrt_neon((sum_squared_x + sum_squared_y + sum_squared_z) / (nk_f64_t)n);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
NK_PUBLIC void nk_rmsd_f64_neon(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
|
|
407
|
+
nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
|
|
408
|
+
if (a_centroid) a_centroid[0] = 0, a_centroid[1] = 0, a_centroid[2] = 0;
|
|
409
|
+
if (b_centroid) b_centroid[0] = 0, b_centroid[1] = 0, b_centroid[2] = 0;
|
|
410
|
+
if (rotation)
|
|
411
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
412
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
413
|
+
if (scale) *scale = 1.0;
|
|
414
|
+
|
|
415
|
+
float64x2_t const zeros_f64x2 = vdupq_n_f64(0);
|
|
416
|
+
float64x2_t sum_squared_x_f64x2 = zeros_f64x2, sum_squared_y_f64x2 = zeros_f64x2, sum_squared_z_f64x2 = zeros_f64x2;
|
|
417
|
+
|
|
418
|
+
float64x2_t a_x_f64x2, a_y_f64x2, a_z_f64x2, b_x_f64x2, b_y_f64x2, b_z_f64x2;
|
|
419
|
+
nk_size_t i = 0;
|
|
420
|
+
|
|
421
|
+
for (; i + 2 <= n; i += 2) {
|
|
422
|
+
nk_deinterleave_f64x2_neon_(a + i * 3, &a_x_f64x2, &a_y_f64x2, &a_z_f64x2);
|
|
423
|
+
nk_deinterleave_f64x2_neon_(b + i * 3, &b_x_f64x2, &b_y_f64x2, &b_z_f64x2);
|
|
424
|
+
|
|
425
|
+
float64x2_t delta_x_f64x2 = vsubq_f64(a_x_f64x2, b_x_f64x2);
|
|
426
|
+
float64x2_t delta_y_f64x2 = vsubq_f64(a_y_f64x2, b_y_f64x2);
|
|
427
|
+
float64x2_t delta_z_f64x2 = vsubq_f64(a_z_f64x2, b_z_f64x2);
|
|
428
|
+
|
|
429
|
+
sum_squared_x_f64x2 = vfmaq_f64(sum_squared_x_f64x2, delta_x_f64x2, delta_x_f64x2);
|
|
430
|
+
sum_squared_y_f64x2 = vfmaq_f64(sum_squared_y_f64x2, delta_y_f64x2, delta_y_f64x2);
|
|
431
|
+
sum_squared_z_f64x2 = vfmaq_f64(sum_squared_z_f64x2, delta_z_f64x2, delta_z_f64x2);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
nk_f64_t total_squared_x = nk_reduce_stable_f64x2_neon_(sum_squared_x_f64x2), total_squared_x_compensation = 0.0;
|
|
435
|
+
nk_f64_t total_squared_y = nk_reduce_stable_f64x2_neon_(sum_squared_y_f64x2), total_squared_y_compensation = 0.0;
|
|
436
|
+
nk_f64_t total_squared_z = nk_reduce_stable_f64x2_neon_(sum_squared_z_f64x2), total_squared_z_compensation = 0.0;
|
|
437
|
+
|
|
438
|
+
for (; i < n; ++i) {
|
|
439
|
+
nk_f64_t delta_x = a[i * 3 + 0] - b[i * 3 + 0];
|
|
440
|
+
nk_f64_t delta_y = a[i * 3 + 1] - b[i * 3 + 1];
|
|
441
|
+
nk_f64_t delta_z = a[i * 3 + 2] - b[i * 3 + 2];
|
|
442
|
+
nk_accumulate_square_f64_(&total_squared_x, &total_squared_x_compensation, delta_x);
|
|
443
|
+
nk_accumulate_square_f64_(&total_squared_y, &total_squared_y_compensation, delta_y);
|
|
444
|
+
nk_accumulate_square_f64_(&total_squared_z, &total_squared_z_compensation, delta_z);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
total_squared_x += total_squared_x_compensation, total_squared_y += total_squared_y_compensation,
|
|
448
|
+
total_squared_z += total_squared_z_compensation;
|
|
449
|
+
|
|
450
|
+
*result = nk_f64_sqrt_neon((total_squared_x + total_squared_y + total_squared_z) / (nk_f64_t)n);
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
NK_PUBLIC void nk_kabsch_f32_neon(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
454
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
|
|
455
|
+
float64x2_t zero_f64x2 = vdupq_n_f64(0.0);
|
|
456
|
+
|
|
457
|
+
// Centroid accumulators (f64, lower/upper halves of f32x4)
|
|
458
|
+
float64x2_t sum_a_x_low_f64x2 = zero_f64x2, sum_a_x_high_f64x2 = zero_f64x2;
|
|
459
|
+
float64x2_t sum_a_y_low_f64x2 = zero_f64x2, sum_a_y_high_f64x2 = zero_f64x2;
|
|
460
|
+
float64x2_t sum_a_z_low_f64x2 = zero_f64x2, sum_a_z_high_f64x2 = zero_f64x2;
|
|
461
|
+
float64x2_t sum_b_x_low_f64x2 = zero_f64x2, sum_b_x_high_f64x2 = zero_f64x2;
|
|
462
|
+
float64x2_t sum_b_y_low_f64x2 = zero_f64x2, sum_b_y_high_f64x2 = zero_f64x2;
|
|
463
|
+
float64x2_t sum_b_z_low_f64x2 = zero_f64x2, sum_b_z_high_f64x2 = zero_f64x2;
|
|
464
|
+
|
|
465
|
+
// Covariance accumulators (f64, lower/upper halves)
|
|
466
|
+
float64x2_t cov_xx_low_f64x2 = zero_f64x2, cov_xx_high_f64x2 = zero_f64x2;
|
|
467
|
+
float64x2_t cov_xy_low_f64x2 = zero_f64x2, cov_xy_high_f64x2 = zero_f64x2;
|
|
468
|
+
float64x2_t cov_xz_low_f64x2 = zero_f64x2, cov_xz_high_f64x2 = zero_f64x2;
|
|
469
|
+
float64x2_t cov_yx_low_f64x2 = zero_f64x2, cov_yx_high_f64x2 = zero_f64x2;
|
|
470
|
+
float64x2_t cov_yy_low_f64x2 = zero_f64x2, cov_yy_high_f64x2 = zero_f64x2;
|
|
471
|
+
float64x2_t cov_yz_low_f64x2 = zero_f64x2, cov_yz_high_f64x2 = zero_f64x2;
|
|
472
|
+
float64x2_t cov_zx_low_f64x2 = zero_f64x2, cov_zx_high_f64x2 = zero_f64x2;
|
|
473
|
+
float64x2_t cov_zy_low_f64x2 = zero_f64x2, cov_zy_high_f64x2 = zero_f64x2;
|
|
474
|
+
float64x2_t cov_zz_low_f64x2 = zero_f64x2, cov_zz_high_f64x2 = zero_f64x2;
|
|
475
|
+
|
|
476
|
+
nk_size_t index = 0;
|
|
477
|
+
for (; index + 4 <= n; index += 4) {
|
|
478
|
+
float32x4_t a_x_f32x4, a_y_f32x4, a_z_f32x4, b_x_f32x4, b_y_f32x4, b_z_f32x4;
|
|
479
|
+
nk_deinterleave_f32x4_neon_(a + index * 3, &a_x_f32x4, &a_y_f32x4, &a_z_f32x4),
|
|
480
|
+
nk_deinterleave_f32x4_neon_(b + index * 3, &b_x_f32x4, &b_y_f32x4, &b_z_f32x4);
|
|
481
|
+
|
|
482
|
+
float64x2_t a_x_low_f64x2 = vcvt_f64_f32(vget_low_f32(a_x_f32x4));
|
|
483
|
+
float64x2_t a_x_high_f64x2 = vcvt_high_f64_f32(a_x_f32x4);
|
|
484
|
+
float64x2_t a_y_low_f64x2 = vcvt_f64_f32(vget_low_f32(a_y_f32x4));
|
|
485
|
+
float64x2_t a_y_high_f64x2 = vcvt_high_f64_f32(a_y_f32x4);
|
|
486
|
+
float64x2_t a_z_low_f64x2 = vcvt_f64_f32(vget_low_f32(a_z_f32x4));
|
|
487
|
+
float64x2_t a_z_high_f64x2 = vcvt_high_f64_f32(a_z_f32x4);
|
|
488
|
+
float64x2_t b_x_low_f64x2 = vcvt_f64_f32(vget_low_f32(b_x_f32x4));
|
|
489
|
+
float64x2_t b_x_high_f64x2 = vcvt_high_f64_f32(b_x_f32x4);
|
|
490
|
+
float64x2_t b_y_low_f64x2 = vcvt_f64_f32(vget_low_f32(b_y_f32x4));
|
|
491
|
+
float64x2_t b_y_high_f64x2 = vcvt_high_f64_f32(b_y_f32x4);
|
|
492
|
+
float64x2_t b_z_low_f64x2 = vcvt_f64_f32(vget_low_f32(b_z_f32x4));
|
|
493
|
+
float64x2_t b_z_high_f64x2 = vcvt_high_f64_f32(b_z_f32x4);
|
|
494
|
+
|
|
495
|
+
// Accumulate centroids
|
|
496
|
+
sum_a_x_low_f64x2 = vaddq_f64(sum_a_x_low_f64x2, a_x_low_f64x2),
|
|
497
|
+
sum_a_x_high_f64x2 = vaddq_f64(sum_a_x_high_f64x2, a_x_high_f64x2);
|
|
498
|
+
sum_a_y_low_f64x2 = vaddq_f64(sum_a_y_low_f64x2, a_y_low_f64x2),
|
|
499
|
+
sum_a_y_high_f64x2 = vaddq_f64(sum_a_y_high_f64x2, a_y_high_f64x2);
|
|
500
|
+
sum_a_z_low_f64x2 = vaddq_f64(sum_a_z_low_f64x2, a_z_low_f64x2),
|
|
501
|
+
sum_a_z_high_f64x2 = vaddq_f64(sum_a_z_high_f64x2, a_z_high_f64x2);
|
|
502
|
+
sum_b_x_low_f64x2 = vaddq_f64(sum_b_x_low_f64x2, b_x_low_f64x2),
|
|
503
|
+
sum_b_x_high_f64x2 = vaddq_f64(sum_b_x_high_f64x2, b_x_high_f64x2);
|
|
504
|
+
sum_b_y_low_f64x2 = vaddq_f64(sum_b_y_low_f64x2, b_y_low_f64x2),
|
|
505
|
+
sum_b_y_high_f64x2 = vaddq_f64(sum_b_y_high_f64x2, b_y_high_f64x2);
|
|
506
|
+
sum_b_z_low_f64x2 = vaddq_f64(sum_b_z_low_f64x2, b_z_low_f64x2),
|
|
507
|
+
sum_b_z_high_f64x2 = vaddq_f64(sum_b_z_high_f64x2, b_z_high_f64x2);
|
|
508
|
+
|
|
509
|
+
// Accumulate raw outer products (uncentered)
|
|
510
|
+
cov_xx_low_f64x2 = vfmaq_f64(cov_xx_low_f64x2, a_x_low_f64x2, b_x_low_f64x2),
|
|
511
|
+
cov_xx_high_f64x2 = vfmaq_f64(cov_xx_high_f64x2, a_x_high_f64x2, b_x_high_f64x2);
|
|
512
|
+
cov_xy_low_f64x2 = vfmaq_f64(cov_xy_low_f64x2, a_x_low_f64x2, b_y_low_f64x2),
|
|
513
|
+
cov_xy_high_f64x2 = vfmaq_f64(cov_xy_high_f64x2, a_x_high_f64x2, b_y_high_f64x2);
|
|
514
|
+
cov_xz_low_f64x2 = vfmaq_f64(cov_xz_low_f64x2, a_x_low_f64x2, b_z_low_f64x2),
|
|
515
|
+
cov_xz_high_f64x2 = vfmaq_f64(cov_xz_high_f64x2, a_x_high_f64x2, b_z_high_f64x2);
|
|
516
|
+
cov_yx_low_f64x2 = vfmaq_f64(cov_yx_low_f64x2, a_y_low_f64x2, b_x_low_f64x2),
|
|
517
|
+
cov_yx_high_f64x2 = vfmaq_f64(cov_yx_high_f64x2, a_y_high_f64x2, b_x_high_f64x2);
|
|
518
|
+
cov_yy_low_f64x2 = vfmaq_f64(cov_yy_low_f64x2, a_y_low_f64x2, b_y_low_f64x2),
|
|
519
|
+
cov_yy_high_f64x2 = vfmaq_f64(cov_yy_high_f64x2, a_y_high_f64x2, b_y_high_f64x2);
|
|
520
|
+
cov_yz_low_f64x2 = vfmaq_f64(cov_yz_low_f64x2, a_y_low_f64x2, b_z_low_f64x2),
|
|
521
|
+
cov_yz_high_f64x2 = vfmaq_f64(cov_yz_high_f64x2, a_y_high_f64x2, b_z_high_f64x2);
|
|
522
|
+
cov_zx_low_f64x2 = vfmaq_f64(cov_zx_low_f64x2, a_z_low_f64x2, b_x_low_f64x2),
|
|
523
|
+
cov_zx_high_f64x2 = vfmaq_f64(cov_zx_high_f64x2, a_z_high_f64x2, b_x_high_f64x2);
|
|
524
|
+
cov_zy_low_f64x2 = vfmaq_f64(cov_zy_low_f64x2, a_z_low_f64x2, b_y_low_f64x2),
|
|
525
|
+
cov_zy_high_f64x2 = vfmaq_f64(cov_zy_high_f64x2, a_z_high_f64x2, b_y_high_f64x2);
|
|
526
|
+
cov_zz_low_f64x2 = vfmaq_f64(cov_zz_low_f64x2, a_z_low_f64x2, b_z_low_f64x2),
|
|
527
|
+
cov_zz_high_f64x2 = vfmaq_f64(cov_zz_high_f64x2, a_z_high_f64x2, b_z_high_f64x2);
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
// Reduce centroid accumulators
|
|
531
|
+
nk_f64_t sum_a_x = vaddvq_f64(vaddq_f64(sum_a_x_low_f64x2, sum_a_x_high_f64x2));
|
|
532
|
+
nk_f64_t sum_a_y = vaddvq_f64(vaddq_f64(sum_a_y_low_f64x2, sum_a_y_high_f64x2));
|
|
533
|
+
nk_f64_t sum_a_z = vaddvq_f64(vaddq_f64(sum_a_z_low_f64x2, sum_a_z_high_f64x2));
|
|
534
|
+
nk_f64_t sum_b_x = vaddvq_f64(vaddq_f64(sum_b_x_low_f64x2, sum_b_x_high_f64x2));
|
|
535
|
+
nk_f64_t sum_b_y = vaddvq_f64(vaddq_f64(sum_b_y_low_f64x2, sum_b_y_high_f64x2));
|
|
536
|
+
nk_f64_t sum_b_z = vaddvq_f64(vaddq_f64(sum_b_z_low_f64x2, sum_b_z_high_f64x2));
|
|
537
|
+
|
|
538
|
+
// Reduce covariance accumulators
|
|
539
|
+
nk_f64_t covariance_x_x = vaddvq_f64(vaddq_f64(cov_xx_low_f64x2, cov_xx_high_f64x2));
|
|
540
|
+
nk_f64_t covariance_x_y = vaddvq_f64(vaddq_f64(cov_xy_low_f64x2, cov_xy_high_f64x2));
|
|
541
|
+
nk_f64_t covariance_x_z = vaddvq_f64(vaddq_f64(cov_xz_low_f64x2, cov_xz_high_f64x2));
|
|
542
|
+
nk_f64_t covariance_y_x = vaddvq_f64(vaddq_f64(cov_yx_low_f64x2, cov_yx_high_f64x2));
|
|
543
|
+
nk_f64_t covariance_y_y = vaddvq_f64(vaddq_f64(cov_yy_low_f64x2, cov_yy_high_f64x2));
|
|
544
|
+
nk_f64_t covariance_y_z = vaddvq_f64(vaddq_f64(cov_yz_low_f64x2, cov_yz_high_f64x2));
|
|
545
|
+
nk_f64_t covariance_z_x = vaddvq_f64(vaddq_f64(cov_zx_low_f64x2, cov_zx_high_f64x2));
|
|
546
|
+
nk_f64_t covariance_z_y = vaddvq_f64(vaddq_f64(cov_zy_low_f64x2, cov_zy_high_f64x2));
|
|
547
|
+
nk_f64_t covariance_z_z = vaddvq_f64(vaddq_f64(cov_zz_low_f64x2, cov_zz_high_f64x2));
|
|
548
|
+
|
|
549
|
+
// Scalar tail
|
|
550
|
+
for (; index < n; ++index) {
|
|
551
|
+
nk_f64_t ax = (nk_f64_t)a[index * 3 + 0], ay = (nk_f64_t)a[index * 3 + 1], az = (nk_f64_t)a[index * 3 + 2];
|
|
552
|
+
nk_f64_t bx = (nk_f64_t)b[index * 3 + 0], by = (nk_f64_t)b[index * 3 + 1], bz = (nk_f64_t)b[index * 3 + 2];
|
|
553
|
+
sum_a_x += ax, sum_a_y += ay, sum_a_z += az;
|
|
554
|
+
sum_b_x += bx, sum_b_y += by, sum_b_z += bz;
|
|
555
|
+
covariance_x_x += ax * bx, covariance_x_y += ax * by, covariance_x_z += ax * bz;
|
|
556
|
+
covariance_y_x += ay * bx, covariance_y_y += ay * by, covariance_y_z += ay * bz;
|
|
557
|
+
covariance_z_x += az * bx, covariance_z_y += az * by, covariance_z_z += az * bz;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// Compute centroids
|
|
561
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
562
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
563
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
564
|
+
if (a_centroid)
|
|
565
|
+
a_centroid[0] = (nk_f32_t)centroid_a_x, a_centroid[1] = (nk_f32_t)centroid_a_y,
|
|
566
|
+
a_centroid[2] = (nk_f32_t)centroid_a_z;
|
|
567
|
+
if (b_centroid)
|
|
568
|
+
b_centroid[0] = (nk_f32_t)centroid_b_x, b_centroid[1] = (nk_f32_t)centroid_b_y,
|
|
569
|
+
b_centroid[2] = (nk_f32_t)centroid_b_z;
|
|
570
|
+
|
|
571
|
+
// Apply centering correction: H_centered = sum(a * bᵀ) - n * centroid_a * centroid_bᵀ
|
|
572
|
+
nk_f64_t h[9];
|
|
573
|
+
h[0] = covariance_x_x - (nk_f64_t)n * centroid_a_x * centroid_b_x;
|
|
574
|
+
h[1] = covariance_x_y - (nk_f64_t)n * centroid_a_x * centroid_b_y;
|
|
575
|
+
h[2] = covariance_x_z - (nk_f64_t)n * centroid_a_x * centroid_b_z;
|
|
576
|
+
h[3] = covariance_y_x - (nk_f64_t)n * centroid_a_y * centroid_b_x;
|
|
577
|
+
h[4] = covariance_y_y - (nk_f64_t)n * centroid_a_y * centroid_b_y;
|
|
578
|
+
h[5] = covariance_y_z - (nk_f64_t)n * centroid_a_y * centroid_b_z;
|
|
579
|
+
h[6] = covariance_z_x - (nk_f64_t)n * centroid_a_z * centroid_b_x;
|
|
580
|
+
h[7] = covariance_z_y - (nk_f64_t)n * centroid_a_z * centroid_b_y;
|
|
581
|
+
h[8] = covariance_z_z - (nk_f64_t)n * centroid_a_z * centroid_b_z;
|
|
582
|
+
|
|
583
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
584
|
+
nk_svd3x3_f64_(h, svd_u, svd_s, svd_v);
|
|
585
|
+
|
|
586
|
+
nk_f64_t r[9];
|
|
587
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
588
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
589
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
590
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
591
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
592
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
593
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
594
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
595
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
596
|
+
|
|
597
|
+
if (nk_det3x3_f64_(r) < 0) {
|
|
598
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
599
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
600
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
601
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
602
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
603
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
604
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
605
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
606
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
607
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
if (rotation)
|
|
611
|
+
for (int j = 0; j != 9; ++j) rotation[j] = (nk_f32_t)r[j];
|
|
612
|
+
if (scale) *scale = 1.0f;
|
|
613
|
+
*result = nk_f64_sqrt_neon(nk_transformed_ssd_f32_neon_(a, b, n, r, 1.0, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
614
|
+
centroid_b_x, centroid_b_y, centroid_b_z) /
|
|
615
|
+
(nk_f64_t)n);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
NK_PUBLIC void nk_kabsch_f64_neon(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
|
|
619
|
+
nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
|
|
620
|
+
float64x2_t const zeros_f64x2 = vdupq_n_f64(0);
|
|
621
|
+
|
|
622
|
+
// 2x unrolling with dual accumulators to hide FMA latency.
|
|
623
|
+
float64x2_t sum_a_x_a_f64x2 = zeros_f64x2, sum_a_y_a_f64x2 = zeros_f64x2, sum_a_z_a_f64x2 = zeros_f64x2;
|
|
624
|
+
float64x2_t sum_b_x_a_f64x2 = zeros_f64x2, sum_b_y_a_f64x2 = zeros_f64x2, sum_b_z_a_f64x2 = zeros_f64x2;
|
|
625
|
+
float64x2_t sum_a_x_b_f64x2 = zeros_f64x2, sum_a_y_b_f64x2 = zeros_f64x2, sum_a_z_b_f64x2 = zeros_f64x2;
|
|
626
|
+
float64x2_t sum_b_x_b_f64x2 = zeros_f64x2, sum_b_y_b_f64x2 = zeros_f64x2, sum_b_z_b_f64x2 = zeros_f64x2;
|
|
627
|
+
|
|
628
|
+
float64x2_t cov_xx_a_f64x2 = zeros_f64x2, cov_xy_a_f64x2 = zeros_f64x2, cov_xz_a_f64x2 = zeros_f64x2;
|
|
629
|
+
float64x2_t cov_yx_a_f64x2 = zeros_f64x2, cov_yy_a_f64x2 = zeros_f64x2, cov_yz_a_f64x2 = zeros_f64x2;
|
|
630
|
+
float64x2_t cov_zx_a_f64x2 = zeros_f64x2, cov_zy_a_f64x2 = zeros_f64x2, cov_zz_a_f64x2 = zeros_f64x2;
|
|
631
|
+
float64x2_t cov_xx_b_f64x2 = zeros_f64x2, cov_xy_b_f64x2 = zeros_f64x2, cov_xz_b_f64x2 = zeros_f64x2;
|
|
632
|
+
float64x2_t cov_yx_b_f64x2 = zeros_f64x2, cov_yy_b_f64x2 = zeros_f64x2, cov_yz_b_f64x2 = zeros_f64x2;
|
|
633
|
+
float64x2_t cov_zx_b_f64x2 = zeros_f64x2, cov_zy_b_f64x2 = zeros_f64x2, cov_zz_b_f64x2 = zeros_f64x2;
|
|
634
|
+
|
|
635
|
+
nk_size_t i = 0;
|
|
636
|
+
float64x2_t a1_x_f64x2, a1_y_f64x2, a1_z_f64x2, b1_x_f64x2, b1_y_f64x2, b1_z_f64x2;
|
|
637
|
+
float64x2_t a2_x_f64x2, a2_y_f64x2, a2_z_f64x2, b2_x_f64x2, b2_y_f64x2, b2_z_f64x2;
|
|
638
|
+
|
|
639
|
+
// Main loop: 4 points per iteration (2x unrolled)
|
|
640
|
+
for (; i + 4 <= n; i += 4) {
|
|
641
|
+
nk_deinterleave_f64x2_neon_(a + i * 3, &a1_x_f64x2, &a1_y_f64x2, &a1_z_f64x2);
|
|
642
|
+
nk_deinterleave_f64x2_neon_(b + i * 3, &b1_x_f64x2, &b1_y_f64x2, &b1_z_f64x2);
|
|
643
|
+
nk_deinterleave_f64x2_neon_(a + (i + 2) * 3, &a2_x_f64x2, &a2_y_f64x2, &a2_z_f64x2);
|
|
644
|
+
nk_deinterleave_f64x2_neon_(b + (i + 2) * 3, &b2_x_f64x2, &b2_y_f64x2, &b2_z_f64x2);
|
|
645
|
+
|
|
646
|
+
// Interleaved accumulation
|
|
647
|
+
sum_a_x_a_f64x2 = vaddq_f64(sum_a_x_a_f64x2, a1_x_f64x2);
|
|
648
|
+
sum_a_x_b_f64x2 = vaddq_f64(sum_a_x_b_f64x2, a2_x_f64x2);
|
|
649
|
+
sum_a_y_a_f64x2 = vaddq_f64(sum_a_y_a_f64x2, a1_y_f64x2);
|
|
650
|
+
sum_a_y_b_f64x2 = vaddq_f64(sum_a_y_b_f64x2, a2_y_f64x2);
|
|
651
|
+
sum_a_z_a_f64x2 = vaddq_f64(sum_a_z_a_f64x2, a1_z_f64x2);
|
|
652
|
+
sum_a_z_b_f64x2 = vaddq_f64(sum_a_z_b_f64x2, a2_z_f64x2);
|
|
653
|
+
sum_b_x_a_f64x2 = vaddq_f64(sum_b_x_a_f64x2, b1_x_f64x2);
|
|
654
|
+
sum_b_x_b_f64x2 = vaddq_f64(sum_b_x_b_f64x2, b2_x_f64x2);
|
|
655
|
+
sum_b_y_a_f64x2 = vaddq_f64(sum_b_y_a_f64x2, b1_y_f64x2);
|
|
656
|
+
sum_b_y_b_f64x2 = vaddq_f64(sum_b_y_b_f64x2, b2_y_f64x2);
|
|
657
|
+
sum_b_z_a_f64x2 = vaddq_f64(sum_b_z_a_f64x2, b1_z_f64x2);
|
|
658
|
+
sum_b_z_b_f64x2 = vaddq_f64(sum_b_z_b_f64x2, b2_z_f64x2);
|
|
659
|
+
|
|
660
|
+
cov_xx_a_f64x2 = vfmaq_f64(cov_xx_a_f64x2, a1_x_f64x2, b1_x_f64x2);
|
|
661
|
+
cov_xx_b_f64x2 = vfmaq_f64(cov_xx_b_f64x2, a2_x_f64x2, b2_x_f64x2);
|
|
662
|
+
cov_xy_a_f64x2 = vfmaq_f64(cov_xy_a_f64x2, a1_x_f64x2, b1_y_f64x2);
|
|
663
|
+
cov_xy_b_f64x2 = vfmaq_f64(cov_xy_b_f64x2, a2_x_f64x2, b2_y_f64x2);
|
|
664
|
+
cov_xz_a_f64x2 = vfmaq_f64(cov_xz_a_f64x2, a1_x_f64x2, b1_z_f64x2);
|
|
665
|
+
cov_xz_b_f64x2 = vfmaq_f64(cov_xz_b_f64x2, a2_x_f64x2, b2_z_f64x2);
|
|
666
|
+
cov_yx_a_f64x2 = vfmaq_f64(cov_yx_a_f64x2, a1_y_f64x2, b1_x_f64x2);
|
|
667
|
+
cov_yx_b_f64x2 = vfmaq_f64(cov_yx_b_f64x2, a2_y_f64x2, b2_x_f64x2);
|
|
668
|
+
cov_yy_a_f64x2 = vfmaq_f64(cov_yy_a_f64x2, a1_y_f64x2, b1_y_f64x2);
|
|
669
|
+
cov_yy_b_f64x2 = vfmaq_f64(cov_yy_b_f64x2, a2_y_f64x2, b2_y_f64x2);
|
|
670
|
+
cov_yz_a_f64x2 = vfmaq_f64(cov_yz_a_f64x2, a1_y_f64x2, b1_z_f64x2);
|
|
671
|
+
cov_yz_b_f64x2 = vfmaq_f64(cov_yz_b_f64x2, a2_y_f64x2, b2_z_f64x2);
|
|
672
|
+
cov_zx_a_f64x2 = vfmaq_f64(cov_zx_a_f64x2, a1_z_f64x2, b1_x_f64x2);
|
|
673
|
+
cov_zx_b_f64x2 = vfmaq_f64(cov_zx_b_f64x2, a2_z_f64x2, b2_x_f64x2);
|
|
674
|
+
cov_zy_a_f64x2 = vfmaq_f64(cov_zy_a_f64x2, a1_z_f64x2, b1_y_f64x2);
|
|
675
|
+
cov_zy_b_f64x2 = vfmaq_f64(cov_zy_b_f64x2, a2_z_f64x2, b2_y_f64x2);
|
|
676
|
+
cov_zz_a_f64x2 = vfmaq_f64(cov_zz_a_f64x2, a1_z_f64x2, b1_z_f64x2);
|
|
677
|
+
cov_zz_b_f64x2 = vfmaq_f64(cov_zz_b_f64x2, a2_z_f64x2, b2_z_f64x2);
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// 2-point tail
|
|
681
|
+
for (; i + 2 <= n; i += 2) {
|
|
682
|
+
nk_deinterleave_f64x2_neon_(a + i * 3, &a1_x_f64x2, &a1_y_f64x2, &a1_z_f64x2);
|
|
683
|
+
nk_deinterleave_f64x2_neon_(b + i * 3, &b1_x_f64x2, &b1_y_f64x2, &b1_z_f64x2);
|
|
684
|
+
sum_a_x_a_f64x2 = vaddq_f64(sum_a_x_a_f64x2, a1_x_f64x2);
|
|
685
|
+
sum_a_y_a_f64x2 = vaddq_f64(sum_a_y_a_f64x2, a1_y_f64x2);
|
|
686
|
+
sum_a_z_a_f64x2 = vaddq_f64(sum_a_z_a_f64x2, a1_z_f64x2);
|
|
687
|
+
sum_b_x_a_f64x2 = vaddq_f64(sum_b_x_a_f64x2, b1_x_f64x2);
|
|
688
|
+
sum_b_y_a_f64x2 = vaddq_f64(sum_b_y_a_f64x2, b1_y_f64x2);
|
|
689
|
+
sum_b_z_a_f64x2 = vaddq_f64(sum_b_z_a_f64x2, b1_z_f64x2);
|
|
690
|
+
cov_xx_a_f64x2 = vfmaq_f64(cov_xx_a_f64x2, a1_x_f64x2, b1_x_f64x2);
|
|
691
|
+
cov_xy_a_f64x2 = vfmaq_f64(cov_xy_a_f64x2, a1_x_f64x2, b1_y_f64x2);
|
|
692
|
+
cov_xz_a_f64x2 = vfmaq_f64(cov_xz_a_f64x2, a1_x_f64x2, b1_z_f64x2);
|
|
693
|
+
cov_yx_a_f64x2 = vfmaq_f64(cov_yx_a_f64x2, a1_y_f64x2, b1_x_f64x2);
|
|
694
|
+
cov_yy_a_f64x2 = vfmaq_f64(cov_yy_a_f64x2, a1_y_f64x2, b1_y_f64x2);
|
|
695
|
+
cov_yz_a_f64x2 = vfmaq_f64(cov_yz_a_f64x2, a1_y_f64x2, b1_z_f64x2);
|
|
696
|
+
cov_zx_a_f64x2 = vfmaq_f64(cov_zx_a_f64x2, a1_z_f64x2, b1_x_f64x2);
|
|
697
|
+
cov_zy_a_f64x2 = vfmaq_f64(cov_zy_a_f64x2, a1_z_f64x2, b1_y_f64x2);
|
|
698
|
+
cov_zz_a_f64x2 = vfmaq_f64(cov_zz_a_f64x2, a1_z_f64x2, b1_z_f64x2);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// Combine dual accumulators
|
|
702
|
+
float64x2_t sum_a_x_f64x2 = vaddq_f64(sum_a_x_a_f64x2, sum_a_x_b_f64x2);
|
|
703
|
+
float64x2_t sum_a_y_f64x2 = vaddq_f64(sum_a_y_a_f64x2, sum_a_y_b_f64x2);
|
|
704
|
+
float64x2_t sum_a_z_f64x2 = vaddq_f64(sum_a_z_a_f64x2, sum_a_z_b_f64x2);
|
|
705
|
+
float64x2_t sum_b_x_f64x2 = vaddq_f64(sum_b_x_a_f64x2, sum_b_x_b_f64x2);
|
|
706
|
+
float64x2_t sum_b_y_f64x2 = vaddq_f64(sum_b_y_a_f64x2, sum_b_y_b_f64x2);
|
|
707
|
+
float64x2_t sum_b_z_f64x2 = vaddq_f64(sum_b_z_a_f64x2, sum_b_z_b_f64x2);
|
|
708
|
+
float64x2_t cov_xx_f64x2 = vaddq_f64(cov_xx_a_f64x2, cov_xx_b_f64x2);
|
|
709
|
+
float64x2_t cov_xy_f64x2 = vaddq_f64(cov_xy_a_f64x2, cov_xy_b_f64x2);
|
|
710
|
+
float64x2_t cov_xz_f64x2 = vaddq_f64(cov_xz_a_f64x2, cov_xz_b_f64x2);
|
|
711
|
+
float64x2_t cov_yx_f64x2 = vaddq_f64(cov_yx_a_f64x2, cov_yx_b_f64x2);
|
|
712
|
+
float64x2_t cov_yy_f64x2 = vaddq_f64(cov_yy_a_f64x2, cov_yy_b_f64x2);
|
|
713
|
+
float64x2_t cov_yz_f64x2 = vaddq_f64(cov_yz_a_f64x2, cov_yz_b_f64x2);
|
|
714
|
+
float64x2_t cov_zx_f64x2 = vaddq_f64(cov_zx_a_f64x2, cov_zx_b_f64x2);
|
|
715
|
+
float64x2_t cov_zy_f64x2 = vaddq_f64(cov_zy_a_f64x2, cov_zy_b_f64x2);
|
|
716
|
+
float64x2_t cov_zz_f64x2 = vaddq_f64(cov_zz_a_f64x2, cov_zz_b_f64x2);
|
|
717
|
+
|
|
718
|
+
// Reduce vector accumulators.
|
|
719
|
+
nk_f64_t sum_a_x = nk_reduce_stable_f64x2_neon_(sum_a_x_f64x2), sum_a_x_compensation = 0.0;
|
|
720
|
+
nk_f64_t sum_a_y = nk_reduce_stable_f64x2_neon_(sum_a_y_f64x2), sum_a_y_compensation = 0.0;
|
|
721
|
+
nk_f64_t sum_a_z = nk_reduce_stable_f64x2_neon_(sum_a_z_f64x2), sum_a_z_compensation = 0.0;
|
|
722
|
+
nk_f64_t sum_b_x = nk_reduce_stable_f64x2_neon_(sum_b_x_f64x2), sum_b_x_compensation = 0.0;
|
|
723
|
+
nk_f64_t sum_b_y = nk_reduce_stable_f64x2_neon_(sum_b_y_f64x2), sum_b_y_compensation = 0.0;
|
|
724
|
+
nk_f64_t sum_b_z = nk_reduce_stable_f64x2_neon_(sum_b_z_f64x2), sum_b_z_compensation = 0.0;
|
|
725
|
+
|
|
726
|
+
nk_f64_t covariance_x_x = nk_reduce_stable_f64x2_neon_(cov_xx_f64x2), covariance_x_x_compensation = 0.0;
|
|
727
|
+
nk_f64_t covariance_x_y = nk_reduce_stable_f64x2_neon_(cov_xy_f64x2), covariance_x_y_compensation = 0.0;
|
|
728
|
+
nk_f64_t covariance_x_z = nk_reduce_stable_f64x2_neon_(cov_xz_f64x2), covariance_x_z_compensation = 0.0;
|
|
729
|
+
nk_f64_t covariance_y_x = nk_reduce_stable_f64x2_neon_(cov_yx_f64x2), covariance_y_x_compensation = 0.0;
|
|
730
|
+
nk_f64_t covariance_y_y = nk_reduce_stable_f64x2_neon_(cov_yy_f64x2), covariance_y_y_compensation = 0.0;
|
|
731
|
+
nk_f64_t covariance_y_z = nk_reduce_stable_f64x2_neon_(cov_yz_f64x2), covariance_y_z_compensation = 0.0;
|
|
732
|
+
nk_f64_t covariance_z_x = nk_reduce_stable_f64x2_neon_(cov_zx_f64x2), covariance_z_x_compensation = 0.0;
|
|
733
|
+
nk_f64_t covariance_z_y = nk_reduce_stable_f64x2_neon_(cov_zy_f64x2), covariance_z_y_compensation = 0.0;
|
|
734
|
+
nk_f64_t covariance_z_z = nk_reduce_stable_f64x2_neon_(cov_zz_f64x2), covariance_z_z_compensation = 0.0;
|
|
735
|
+
|
|
736
|
+
// Scalar tail
|
|
737
|
+
for (; i < n; ++i) {
|
|
738
|
+
nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
|
|
739
|
+
nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
|
|
740
|
+
nk_accumulate_sum_f64_(&sum_a_x, &sum_a_x_compensation, ax),
|
|
741
|
+
nk_accumulate_sum_f64_(&sum_a_y, &sum_a_y_compensation, ay),
|
|
742
|
+
nk_accumulate_sum_f64_(&sum_a_z, &sum_a_z_compensation, az);
|
|
743
|
+
nk_accumulate_sum_f64_(&sum_b_x, &sum_b_x_compensation, bx),
|
|
744
|
+
nk_accumulate_sum_f64_(&sum_b_y, &sum_b_y_compensation, by),
|
|
745
|
+
nk_accumulate_sum_f64_(&sum_b_z, &sum_b_z_compensation, bz);
|
|
746
|
+
nk_accumulate_product_f64_(&covariance_x_x, &covariance_x_x_compensation, ax, bx),
|
|
747
|
+
nk_accumulate_product_f64_(&covariance_x_y, &covariance_x_y_compensation, ax, by),
|
|
748
|
+
nk_accumulate_product_f64_(&covariance_x_z, &covariance_x_z_compensation, ax, bz);
|
|
749
|
+
nk_accumulate_product_f64_(&covariance_y_x, &covariance_y_x_compensation, ay, bx),
|
|
750
|
+
nk_accumulate_product_f64_(&covariance_y_y, &covariance_y_y_compensation, ay, by),
|
|
751
|
+
nk_accumulate_product_f64_(&covariance_y_z, &covariance_y_z_compensation, ay, bz);
|
|
752
|
+
nk_accumulate_product_f64_(&covariance_z_x, &covariance_z_x_compensation, az, bx),
|
|
753
|
+
nk_accumulate_product_f64_(&covariance_z_y, &covariance_z_y_compensation, az, by),
|
|
754
|
+
nk_accumulate_product_f64_(&covariance_z_z, &covariance_z_z_compensation, az, bz);
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
sum_a_x += sum_a_x_compensation, sum_a_y += sum_a_y_compensation, sum_a_z += sum_a_z_compensation;
|
|
758
|
+
sum_b_x += sum_b_x_compensation, sum_b_y += sum_b_y_compensation, sum_b_z += sum_b_z_compensation;
|
|
759
|
+
covariance_x_x += covariance_x_x_compensation, covariance_x_y += covariance_x_y_compensation,
|
|
760
|
+
covariance_x_z += covariance_x_z_compensation;
|
|
761
|
+
covariance_y_x += covariance_y_x_compensation, covariance_y_y += covariance_y_y_compensation,
|
|
762
|
+
covariance_y_z += covariance_y_z_compensation;
|
|
763
|
+
covariance_z_x += covariance_z_x_compensation, covariance_z_y += covariance_z_y_compensation,
|
|
764
|
+
covariance_z_z += covariance_z_z_compensation;
|
|
765
|
+
|
|
766
|
+
// Compute centroids
|
|
767
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
768
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
769
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
770
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
771
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
772
|
+
|
|
773
|
+
// Apply centering correction: H_centered = H - n * centroid_a * centroid_bᵀ
|
|
774
|
+
covariance_x_x -= (nk_f64_t)n * centroid_a_x * centroid_b_x;
|
|
775
|
+
covariance_x_y -= (nk_f64_t)n * centroid_a_x * centroid_b_y;
|
|
776
|
+
covariance_x_z -= (nk_f64_t)n * centroid_a_x * centroid_b_z;
|
|
777
|
+
covariance_y_x -= (nk_f64_t)n * centroid_a_y * centroid_b_x;
|
|
778
|
+
covariance_y_y -= (nk_f64_t)n * centroid_a_y * centroid_b_y;
|
|
779
|
+
covariance_y_z -= (nk_f64_t)n * centroid_a_y * centroid_b_z;
|
|
780
|
+
covariance_z_x -= (nk_f64_t)n * centroid_a_z * centroid_b_x;
|
|
781
|
+
covariance_z_y -= (nk_f64_t)n * centroid_a_z * centroid_b_y;
|
|
782
|
+
covariance_z_z -= (nk_f64_t)n * centroid_a_z * centroid_b_z;
|
|
783
|
+
|
|
784
|
+
// Compute SVD and optimal rotation
|
|
785
|
+
nk_f64_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
|
|
786
|
+
covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
|
|
787
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
788
|
+
nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
|
|
789
|
+
|
|
790
|
+
nk_f64_t r[9];
|
|
791
|
+
nk_rotation_from_svd_f64_neon_(svd_u, svd_v, r);
|
|
792
|
+
|
|
793
|
+
// Handle reflection: if det(R) < 0, negate third column of V and recompute R
|
|
794
|
+
if (nk_det3x3_f64_(r) < 0) {
|
|
795
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
796
|
+
nk_rotation_from_svd_f64_neon_(svd_u, svd_v, r);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
// Output rotation matrix and scale=1.0.
|
|
800
|
+
if (rotation)
|
|
801
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
802
|
+
|
|
803
|
+
if (scale) *scale = 1.0;
|
|
804
|
+
|
|
805
|
+
// Compute RMSD after optimal rotation
|
|
806
|
+
nk_f64_t sum_squared = nk_transformed_ssd_f64_neon_(a, b, n, r, 1.0, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
807
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
808
|
+
*result = nk_f64_sqrt_neon(sum_squared * inv_n);
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
NK_PUBLIC void nk_umeyama_f32_neon(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
812
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f64_t *result) {
|
|
813
|
+
float64x2_t zero_f64x2 = vdupq_n_f64(0.0);
|
|
814
|
+
|
|
815
|
+
// Centroid accumulators (f64, lower/upper halves of f32x4)
|
|
816
|
+
float64x2_t sum_a_x_low_f64x2 = zero_f64x2, sum_a_x_high_f64x2 = zero_f64x2;
|
|
817
|
+
float64x2_t sum_a_y_low_f64x2 = zero_f64x2, sum_a_y_high_f64x2 = zero_f64x2;
|
|
818
|
+
float64x2_t sum_a_z_low_f64x2 = zero_f64x2, sum_a_z_high_f64x2 = zero_f64x2;
|
|
819
|
+
float64x2_t sum_b_x_low_f64x2 = zero_f64x2, sum_b_x_high_f64x2 = zero_f64x2;
|
|
820
|
+
float64x2_t sum_b_y_low_f64x2 = zero_f64x2, sum_b_y_high_f64x2 = zero_f64x2;
|
|
821
|
+
float64x2_t sum_b_z_low_f64x2 = zero_f64x2, sum_b_z_high_f64x2 = zero_f64x2;
|
|
822
|
+
|
|
823
|
+
// Covariance accumulators (f64, lower/upper halves)
|
|
824
|
+
float64x2_t cov_xx_low_f64x2 = zero_f64x2, cov_xx_high_f64x2 = zero_f64x2;
|
|
825
|
+
float64x2_t cov_xy_low_f64x2 = zero_f64x2, cov_xy_high_f64x2 = zero_f64x2;
|
|
826
|
+
float64x2_t cov_xz_low_f64x2 = zero_f64x2, cov_xz_high_f64x2 = zero_f64x2;
|
|
827
|
+
float64x2_t cov_yx_low_f64x2 = zero_f64x2, cov_yx_high_f64x2 = zero_f64x2;
|
|
828
|
+
float64x2_t cov_yy_low_f64x2 = zero_f64x2, cov_yy_high_f64x2 = zero_f64x2;
|
|
829
|
+
float64x2_t cov_yz_low_f64x2 = zero_f64x2, cov_yz_high_f64x2 = zero_f64x2;
|
|
830
|
+
float64x2_t cov_zx_low_f64x2 = zero_f64x2, cov_zx_high_f64x2 = zero_f64x2;
|
|
831
|
+
float64x2_t cov_zy_low_f64x2 = zero_f64x2, cov_zy_high_f64x2 = zero_f64x2;
|
|
832
|
+
float64x2_t cov_zz_low_f64x2 = zero_f64x2, cov_zz_high_f64x2 = zero_f64x2;
|
|
833
|
+
|
|
834
|
+
// Variance of A accumulator
|
|
835
|
+
float64x2_t variance_low_f64x2 = zero_f64x2, variance_high_f64x2 = zero_f64x2;
|
|
836
|
+
|
|
837
|
+
nk_size_t index = 0;
|
|
838
|
+
for (; index + 4 <= n; index += 4) {
|
|
839
|
+
float32x4_t a_x_f32x4, a_y_f32x4, a_z_f32x4, b_x_f32x4, b_y_f32x4, b_z_f32x4;
|
|
840
|
+
nk_deinterleave_f32x4_neon_(a + index * 3, &a_x_f32x4, &a_y_f32x4, &a_z_f32x4),
|
|
841
|
+
nk_deinterleave_f32x4_neon_(b + index * 3, &b_x_f32x4, &b_y_f32x4, &b_z_f32x4);
|
|
842
|
+
|
|
843
|
+
float64x2_t a_x_low_f64x2 = vcvt_f64_f32(vget_low_f32(a_x_f32x4));
|
|
844
|
+
float64x2_t a_x_high_f64x2 = vcvt_high_f64_f32(a_x_f32x4);
|
|
845
|
+
float64x2_t a_y_low_f64x2 = vcvt_f64_f32(vget_low_f32(a_y_f32x4));
|
|
846
|
+
float64x2_t a_y_high_f64x2 = vcvt_high_f64_f32(a_y_f32x4);
|
|
847
|
+
float64x2_t a_z_low_f64x2 = vcvt_f64_f32(vget_low_f32(a_z_f32x4));
|
|
848
|
+
float64x2_t a_z_high_f64x2 = vcvt_high_f64_f32(a_z_f32x4);
|
|
849
|
+
float64x2_t b_x_low_f64x2 = vcvt_f64_f32(vget_low_f32(b_x_f32x4));
|
|
850
|
+
float64x2_t b_x_high_f64x2 = vcvt_high_f64_f32(b_x_f32x4);
|
|
851
|
+
float64x2_t b_y_low_f64x2 = vcvt_f64_f32(vget_low_f32(b_y_f32x4));
|
|
852
|
+
float64x2_t b_y_high_f64x2 = vcvt_high_f64_f32(b_y_f32x4);
|
|
853
|
+
float64x2_t b_z_low_f64x2 = vcvt_f64_f32(vget_low_f32(b_z_f32x4));
|
|
854
|
+
float64x2_t b_z_high_f64x2 = vcvt_high_f64_f32(b_z_f32x4);
|
|
855
|
+
|
|
856
|
+
// Accumulate centroids
|
|
857
|
+
sum_a_x_low_f64x2 = vaddq_f64(sum_a_x_low_f64x2, a_x_low_f64x2),
|
|
858
|
+
sum_a_x_high_f64x2 = vaddq_f64(sum_a_x_high_f64x2, a_x_high_f64x2);
|
|
859
|
+
sum_a_y_low_f64x2 = vaddq_f64(sum_a_y_low_f64x2, a_y_low_f64x2),
|
|
860
|
+
sum_a_y_high_f64x2 = vaddq_f64(sum_a_y_high_f64x2, a_y_high_f64x2);
|
|
861
|
+
sum_a_z_low_f64x2 = vaddq_f64(sum_a_z_low_f64x2, a_z_low_f64x2),
|
|
862
|
+
sum_a_z_high_f64x2 = vaddq_f64(sum_a_z_high_f64x2, a_z_high_f64x2);
|
|
863
|
+
sum_b_x_low_f64x2 = vaddq_f64(sum_b_x_low_f64x2, b_x_low_f64x2),
|
|
864
|
+
sum_b_x_high_f64x2 = vaddq_f64(sum_b_x_high_f64x2, b_x_high_f64x2);
|
|
865
|
+
sum_b_y_low_f64x2 = vaddq_f64(sum_b_y_low_f64x2, b_y_low_f64x2),
|
|
866
|
+
sum_b_y_high_f64x2 = vaddq_f64(sum_b_y_high_f64x2, b_y_high_f64x2);
|
|
867
|
+
sum_b_z_low_f64x2 = vaddq_f64(sum_b_z_low_f64x2, b_z_low_f64x2),
|
|
868
|
+
sum_b_z_high_f64x2 = vaddq_f64(sum_b_z_high_f64x2, b_z_high_f64x2);
|
|
869
|
+
|
|
870
|
+
// Accumulate raw outer products (uncentered)
|
|
871
|
+
cov_xx_low_f64x2 = vfmaq_f64(cov_xx_low_f64x2, a_x_low_f64x2, b_x_low_f64x2),
|
|
872
|
+
cov_xx_high_f64x2 = vfmaq_f64(cov_xx_high_f64x2, a_x_high_f64x2, b_x_high_f64x2);
|
|
873
|
+
cov_xy_low_f64x2 = vfmaq_f64(cov_xy_low_f64x2, a_x_low_f64x2, b_y_low_f64x2),
|
|
874
|
+
cov_xy_high_f64x2 = vfmaq_f64(cov_xy_high_f64x2, a_x_high_f64x2, b_y_high_f64x2);
|
|
875
|
+
cov_xz_low_f64x2 = vfmaq_f64(cov_xz_low_f64x2, a_x_low_f64x2, b_z_low_f64x2),
|
|
876
|
+
cov_xz_high_f64x2 = vfmaq_f64(cov_xz_high_f64x2, a_x_high_f64x2, b_z_high_f64x2);
|
|
877
|
+
cov_yx_low_f64x2 = vfmaq_f64(cov_yx_low_f64x2, a_y_low_f64x2, b_x_low_f64x2),
|
|
878
|
+
cov_yx_high_f64x2 = vfmaq_f64(cov_yx_high_f64x2, a_y_high_f64x2, b_x_high_f64x2);
|
|
879
|
+
cov_yy_low_f64x2 = vfmaq_f64(cov_yy_low_f64x2, a_y_low_f64x2, b_y_low_f64x2),
|
|
880
|
+
cov_yy_high_f64x2 = vfmaq_f64(cov_yy_high_f64x2, a_y_high_f64x2, b_y_high_f64x2);
|
|
881
|
+
cov_yz_low_f64x2 = vfmaq_f64(cov_yz_low_f64x2, a_y_low_f64x2, b_z_low_f64x2),
|
|
882
|
+
cov_yz_high_f64x2 = vfmaq_f64(cov_yz_high_f64x2, a_y_high_f64x2, b_z_high_f64x2);
|
|
883
|
+
cov_zx_low_f64x2 = vfmaq_f64(cov_zx_low_f64x2, a_z_low_f64x2, b_x_low_f64x2),
|
|
884
|
+
cov_zx_high_f64x2 = vfmaq_f64(cov_zx_high_f64x2, a_z_high_f64x2, b_x_high_f64x2);
|
|
885
|
+
cov_zy_low_f64x2 = vfmaq_f64(cov_zy_low_f64x2, a_z_low_f64x2, b_y_low_f64x2),
|
|
886
|
+
cov_zy_high_f64x2 = vfmaq_f64(cov_zy_high_f64x2, a_z_high_f64x2, b_y_high_f64x2);
|
|
887
|
+
cov_zz_low_f64x2 = vfmaq_f64(cov_zz_low_f64x2, a_z_low_f64x2, b_z_low_f64x2),
|
|
888
|
+
cov_zz_high_f64x2 = vfmaq_f64(cov_zz_high_f64x2, a_z_high_f64x2, b_z_high_f64x2);
|
|
889
|
+
|
|
890
|
+
// Accumulate variance of A (sum of squared coordinates)
|
|
891
|
+
variance_low_f64x2 = vfmaq_f64(variance_low_f64x2, a_x_low_f64x2, a_x_low_f64x2),
|
|
892
|
+
variance_high_f64x2 = vfmaq_f64(variance_high_f64x2, a_x_high_f64x2, a_x_high_f64x2);
|
|
893
|
+
variance_low_f64x2 = vfmaq_f64(variance_low_f64x2, a_y_low_f64x2, a_y_low_f64x2),
|
|
894
|
+
variance_high_f64x2 = vfmaq_f64(variance_high_f64x2, a_y_high_f64x2, a_y_high_f64x2);
|
|
895
|
+
variance_low_f64x2 = vfmaq_f64(variance_low_f64x2, a_z_low_f64x2, a_z_low_f64x2),
|
|
896
|
+
variance_high_f64x2 = vfmaq_f64(variance_high_f64x2, a_z_high_f64x2, a_z_high_f64x2);
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
// Reduce centroid accumulators
|
|
900
|
+
nk_f64_t sum_a_x = vaddvq_f64(vaddq_f64(sum_a_x_low_f64x2, sum_a_x_high_f64x2));
|
|
901
|
+
nk_f64_t sum_a_y = vaddvq_f64(vaddq_f64(sum_a_y_low_f64x2, sum_a_y_high_f64x2));
|
|
902
|
+
nk_f64_t sum_a_z = vaddvq_f64(vaddq_f64(sum_a_z_low_f64x2, sum_a_z_high_f64x2));
|
|
903
|
+
nk_f64_t sum_b_x = vaddvq_f64(vaddq_f64(sum_b_x_low_f64x2, sum_b_x_high_f64x2));
|
|
904
|
+
nk_f64_t sum_b_y = vaddvq_f64(vaddq_f64(sum_b_y_low_f64x2, sum_b_y_high_f64x2));
|
|
905
|
+
nk_f64_t sum_b_z = vaddvq_f64(vaddq_f64(sum_b_z_low_f64x2, sum_b_z_high_f64x2));
|
|
906
|
+
|
|
907
|
+
// Reduce covariance accumulators
|
|
908
|
+
nk_f64_t covariance_x_x = vaddvq_f64(vaddq_f64(cov_xx_low_f64x2, cov_xx_high_f64x2));
|
|
909
|
+
nk_f64_t covariance_x_y = vaddvq_f64(vaddq_f64(cov_xy_low_f64x2, cov_xy_high_f64x2));
|
|
910
|
+
nk_f64_t covariance_x_z = vaddvq_f64(vaddq_f64(cov_xz_low_f64x2, cov_xz_high_f64x2));
|
|
911
|
+
nk_f64_t covariance_y_x = vaddvq_f64(vaddq_f64(cov_yx_low_f64x2, cov_yx_high_f64x2));
|
|
912
|
+
nk_f64_t covariance_y_y = vaddvq_f64(vaddq_f64(cov_yy_low_f64x2, cov_yy_high_f64x2));
|
|
913
|
+
nk_f64_t covariance_y_z = vaddvq_f64(vaddq_f64(cov_yz_low_f64x2, cov_yz_high_f64x2));
|
|
914
|
+
nk_f64_t covariance_z_x = vaddvq_f64(vaddq_f64(cov_zx_low_f64x2, cov_zx_high_f64x2));
|
|
915
|
+
nk_f64_t covariance_z_y = vaddvq_f64(vaddq_f64(cov_zy_low_f64x2, cov_zy_high_f64x2));
|
|
916
|
+
nk_f64_t covariance_z_z = vaddvq_f64(vaddq_f64(cov_zz_low_f64x2, cov_zz_high_f64x2));
|
|
917
|
+
nk_f64_t sum_sq_a = vaddvq_f64(vaddq_f64(variance_low_f64x2, variance_high_f64x2));
|
|
918
|
+
|
|
919
|
+
// Scalar tail
|
|
920
|
+
for (; index < n; ++index) {
|
|
921
|
+
nk_f64_t ax = (nk_f64_t)a[index * 3 + 0], ay = (nk_f64_t)a[index * 3 + 1], az = (nk_f64_t)a[index * 3 + 2];
|
|
922
|
+
nk_f64_t bx = (nk_f64_t)b[index * 3 + 0], by = (nk_f64_t)b[index * 3 + 1], bz = (nk_f64_t)b[index * 3 + 2];
|
|
923
|
+
sum_a_x += ax, sum_a_y += ay, sum_a_z += az;
|
|
924
|
+
sum_b_x += bx, sum_b_y += by, sum_b_z += bz;
|
|
925
|
+
covariance_x_x += ax * bx, covariance_x_y += ax * by, covariance_x_z += ax * bz;
|
|
926
|
+
covariance_y_x += ay * bx, covariance_y_y += ay * by, covariance_y_z += ay * bz;
|
|
927
|
+
covariance_z_x += az * bx, covariance_z_y += az * by, covariance_z_z += az * bz;
|
|
928
|
+
sum_sq_a += ax * ax + ay * ay + az * az;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
// Compute centroids
|
|
932
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
933
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
934
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
935
|
+
if (a_centroid)
|
|
936
|
+
a_centroid[0] = (nk_f32_t)centroid_a_x, a_centroid[1] = (nk_f32_t)centroid_a_y,
|
|
937
|
+
a_centroid[2] = (nk_f32_t)centroid_a_z;
|
|
938
|
+
if (b_centroid)
|
|
939
|
+
b_centroid[0] = (nk_f32_t)centroid_b_x, b_centroid[1] = (nk_f32_t)centroid_b_y,
|
|
940
|
+
b_centroid[2] = (nk_f32_t)centroid_b_z;
|
|
941
|
+
|
|
942
|
+
// Compute variance of A (centered): var = sum(a^2)/n - centroid^2
|
|
943
|
+
nk_f64_t variance_a = sum_sq_a * inv_n -
|
|
944
|
+
(centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
|
|
945
|
+
|
|
946
|
+
// Apply centering correction: H_centered = sum(a * bᵀ) - n * centroid_a * centroid_bᵀ
|
|
947
|
+
nk_f64_t h[9];
|
|
948
|
+
h[0] = covariance_x_x - (nk_f64_t)n * centroid_a_x * centroid_b_x;
|
|
949
|
+
h[1] = covariance_x_y - (nk_f64_t)n * centroid_a_x * centroid_b_y;
|
|
950
|
+
h[2] = covariance_x_z - (nk_f64_t)n * centroid_a_x * centroid_b_z;
|
|
951
|
+
h[3] = covariance_y_x - (nk_f64_t)n * centroid_a_y * centroid_b_x;
|
|
952
|
+
h[4] = covariance_y_y - (nk_f64_t)n * centroid_a_y * centroid_b_y;
|
|
953
|
+
h[5] = covariance_y_z - (nk_f64_t)n * centroid_a_y * centroid_b_z;
|
|
954
|
+
h[6] = covariance_z_x - (nk_f64_t)n * centroid_a_z * centroid_b_x;
|
|
955
|
+
h[7] = covariance_z_y - (nk_f64_t)n * centroid_a_z * centroid_b_y;
|
|
956
|
+
h[8] = covariance_z_z - (nk_f64_t)n * centroid_a_z * centroid_b_z;
|
|
957
|
+
|
|
958
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
959
|
+
nk_svd3x3_f64_(h, svd_u, svd_s, svd_v);
|
|
960
|
+
|
|
961
|
+
nk_f64_t r[9];
|
|
962
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
963
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
964
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
965
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
966
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
967
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
968
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
969
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
970
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
971
|
+
|
|
972
|
+
nk_f64_t det = nk_det3x3_f64_(r), sign_correction = det < 0 ? -1.0 : 1.0;
|
|
973
|
+
if (det < 0) {
|
|
974
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
975
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
976
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
977
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
978
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
979
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
980
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
981
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
982
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
983
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
nk_f64_t applied_scale = (svd_s[0] + svd_s[4] + sign_correction * svd_s[8]) / ((nk_f64_t)n * variance_a);
|
|
987
|
+
if (rotation)
|
|
988
|
+
for (int j = 0; j != 9; ++j) rotation[j] = (nk_f32_t)r[j];
|
|
989
|
+
if (scale) *scale = (nk_f32_t)applied_scale;
|
|
990
|
+
*result = nk_f64_sqrt_neon(nk_transformed_ssd_f32_neon_(a, b, n, r, applied_scale, centroid_a_x, centroid_a_y,
|
|
991
|
+
centroid_a_z, centroid_b_x, centroid_b_y, centroid_b_z) /
|
|
992
|
+
(nk_f64_t)n);
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
NK_PUBLIC void nk_umeyama_f64_neon(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *a_centroid,
|
|
996
|
+
nk_f64_t *b_centroid, nk_f64_t *rotation, nk_f64_t *scale, nk_f64_t *result) {
|
|
997
|
+
float64x2_t const zeros_f64x2 = vdupq_n_f64(0);
|
|
998
|
+
|
|
999
|
+
// 2x unrolling with dual accumulators to hide FMA latency.
|
|
1000
|
+
float64x2_t sum_a_x_a_f64x2 = zeros_f64x2, sum_a_y_a_f64x2 = zeros_f64x2, sum_a_z_a_f64x2 = zeros_f64x2;
|
|
1001
|
+
float64x2_t sum_b_x_a_f64x2 = zeros_f64x2, sum_b_y_a_f64x2 = zeros_f64x2, sum_b_z_a_f64x2 = zeros_f64x2;
|
|
1002
|
+
float64x2_t sum_a_x_b_f64x2 = zeros_f64x2, sum_a_y_b_f64x2 = zeros_f64x2, sum_a_z_b_f64x2 = zeros_f64x2;
|
|
1003
|
+
float64x2_t sum_b_x_b_f64x2 = zeros_f64x2, sum_b_y_b_f64x2 = zeros_f64x2, sum_b_z_b_f64x2 = zeros_f64x2;
|
|
1004
|
+
|
|
1005
|
+
float64x2_t cov_xx_a_f64x2 = zeros_f64x2, cov_xy_a_f64x2 = zeros_f64x2, cov_xz_a_f64x2 = zeros_f64x2;
|
|
1006
|
+
float64x2_t cov_yx_a_f64x2 = zeros_f64x2, cov_yy_a_f64x2 = zeros_f64x2, cov_yz_a_f64x2 = zeros_f64x2;
|
|
1007
|
+
float64x2_t cov_zx_a_f64x2 = zeros_f64x2, cov_zy_a_f64x2 = zeros_f64x2, cov_zz_a_f64x2 = zeros_f64x2;
|
|
1008
|
+
float64x2_t cov_xx_b_f64x2 = zeros_f64x2, cov_xy_b_f64x2 = zeros_f64x2, cov_xz_b_f64x2 = zeros_f64x2;
|
|
1009
|
+
float64x2_t cov_yx_b_f64x2 = zeros_f64x2, cov_yy_b_f64x2 = zeros_f64x2, cov_yz_b_f64x2 = zeros_f64x2;
|
|
1010
|
+
float64x2_t cov_zx_b_f64x2 = zeros_f64x2, cov_zy_b_f64x2 = zeros_f64x2, cov_zz_b_f64x2 = zeros_f64x2;
|
|
1011
|
+
float64x2_t variance_a_a_f64x2 = zeros_f64x2, variance_a_b_f64x2 = zeros_f64x2;
|
|
1012
|
+
|
|
1013
|
+
nk_size_t i = 0;
|
|
1014
|
+
float64x2_t a1_x_f64x2, a1_y_f64x2, a1_z_f64x2, b1_x_f64x2, b1_y_f64x2, b1_z_f64x2;
|
|
1015
|
+
float64x2_t a2_x_f64x2, a2_y_f64x2, a2_z_f64x2, b2_x_f64x2, b2_y_f64x2, b2_z_f64x2;
|
|
1016
|
+
|
|
1017
|
+
// Main loop: 4 points per iteration (2x unrolled)
|
|
1018
|
+
for (; i + 4 <= n; i += 4) {
|
|
1019
|
+
nk_deinterleave_f64x2_neon_(a + i * 3, &a1_x_f64x2, &a1_y_f64x2, &a1_z_f64x2);
|
|
1020
|
+
nk_deinterleave_f64x2_neon_(b + i * 3, &b1_x_f64x2, &b1_y_f64x2, &b1_z_f64x2);
|
|
1021
|
+
nk_deinterleave_f64x2_neon_(a + (i + 2) * 3, &a2_x_f64x2, &a2_y_f64x2, &a2_z_f64x2);
|
|
1022
|
+
nk_deinterleave_f64x2_neon_(b + (i + 2) * 3, &b2_x_f64x2, &b2_y_f64x2, &b2_z_f64x2);
|
|
1023
|
+
|
|
1024
|
+
// Interleaved accumulation
|
|
1025
|
+
sum_a_x_a_f64x2 = vaddq_f64(sum_a_x_a_f64x2, a1_x_f64x2);
|
|
1026
|
+
sum_a_x_b_f64x2 = vaddq_f64(sum_a_x_b_f64x2, a2_x_f64x2);
|
|
1027
|
+
sum_a_y_a_f64x2 = vaddq_f64(sum_a_y_a_f64x2, a1_y_f64x2);
|
|
1028
|
+
sum_a_y_b_f64x2 = vaddq_f64(sum_a_y_b_f64x2, a2_y_f64x2);
|
|
1029
|
+
sum_a_z_a_f64x2 = vaddq_f64(sum_a_z_a_f64x2, a1_z_f64x2);
|
|
1030
|
+
sum_a_z_b_f64x2 = vaddq_f64(sum_a_z_b_f64x2, a2_z_f64x2);
|
|
1031
|
+
sum_b_x_a_f64x2 = vaddq_f64(sum_b_x_a_f64x2, b1_x_f64x2);
|
|
1032
|
+
sum_b_x_b_f64x2 = vaddq_f64(sum_b_x_b_f64x2, b2_x_f64x2);
|
|
1033
|
+
sum_b_y_a_f64x2 = vaddq_f64(sum_b_y_a_f64x2, b1_y_f64x2);
|
|
1034
|
+
sum_b_y_b_f64x2 = vaddq_f64(sum_b_y_b_f64x2, b2_y_f64x2);
|
|
1035
|
+
sum_b_z_a_f64x2 = vaddq_f64(sum_b_z_a_f64x2, b1_z_f64x2);
|
|
1036
|
+
sum_b_z_b_f64x2 = vaddq_f64(sum_b_z_b_f64x2, b2_z_f64x2);
|
|
1037
|
+
|
|
1038
|
+
cov_xx_a_f64x2 = vfmaq_f64(cov_xx_a_f64x2, a1_x_f64x2, b1_x_f64x2);
|
|
1039
|
+
cov_xx_b_f64x2 = vfmaq_f64(cov_xx_b_f64x2, a2_x_f64x2, b2_x_f64x2);
|
|
1040
|
+
cov_xy_a_f64x2 = vfmaq_f64(cov_xy_a_f64x2, a1_x_f64x2, b1_y_f64x2);
|
|
1041
|
+
cov_xy_b_f64x2 = vfmaq_f64(cov_xy_b_f64x2, a2_x_f64x2, b2_y_f64x2);
|
|
1042
|
+
cov_xz_a_f64x2 = vfmaq_f64(cov_xz_a_f64x2, a1_x_f64x2, b1_z_f64x2);
|
|
1043
|
+
cov_xz_b_f64x2 = vfmaq_f64(cov_xz_b_f64x2, a2_x_f64x2, b2_z_f64x2);
|
|
1044
|
+
cov_yx_a_f64x2 = vfmaq_f64(cov_yx_a_f64x2, a1_y_f64x2, b1_x_f64x2);
|
|
1045
|
+
cov_yx_b_f64x2 = vfmaq_f64(cov_yx_b_f64x2, a2_y_f64x2, b2_x_f64x2);
|
|
1046
|
+
cov_yy_a_f64x2 = vfmaq_f64(cov_yy_a_f64x2, a1_y_f64x2, b1_y_f64x2);
|
|
1047
|
+
cov_yy_b_f64x2 = vfmaq_f64(cov_yy_b_f64x2, a2_y_f64x2, b2_y_f64x2);
|
|
1048
|
+
cov_yz_a_f64x2 = vfmaq_f64(cov_yz_a_f64x2, a1_y_f64x2, b1_z_f64x2);
|
|
1049
|
+
cov_yz_b_f64x2 = vfmaq_f64(cov_yz_b_f64x2, a2_y_f64x2, b2_z_f64x2);
|
|
1050
|
+
cov_zx_a_f64x2 = vfmaq_f64(cov_zx_a_f64x2, a1_z_f64x2, b1_x_f64x2);
|
|
1051
|
+
cov_zx_b_f64x2 = vfmaq_f64(cov_zx_b_f64x2, a2_z_f64x2, b2_x_f64x2);
|
|
1052
|
+
cov_zy_a_f64x2 = vfmaq_f64(cov_zy_a_f64x2, a1_z_f64x2, b1_y_f64x2);
|
|
1053
|
+
cov_zy_b_f64x2 = vfmaq_f64(cov_zy_b_f64x2, a2_z_f64x2, b2_y_f64x2);
|
|
1054
|
+
cov_zz_a_f64x2 = vfmaq_f64(cov_zz_a_f64x2, a1_z_f64x2, b1_z_f64x2);
|
|
1055
|
+
cov_zz_b_f64x2 = vfmaq_f64(cov_zz_b_f64x2, a2_z_f64x2, b2_z_f64x2);
|
|
1056
|
+
|
|
1057
|
+
variance_a_a_f64x2 = vfmaq_f64(variance_a_a_f64x2, a1_x_f64x2, a1_x_f64x2);
|
|
1058
|
+
variance_a_b_f64x2 = vfmaq_f64(variance_a_b_f64x2, a2_x_f64x2, a2_x_f64x2);
|
|
1059
|
+
variance_a_a_f64x2 = vfmaq_f64(variance_a_a_f64x2, a1_y_f64x2, a1_y_f64x2);
|
|
1060
|
+
variance_a_b_f64x2 = vfmaq_f64(variance_a_b_f64x2, a2_y_f64x2, a2_y_f64x2);
|
|
1061
|
+
variance_a_a_f64x2 = vfmaq_f64(variance_a_a_f64x2, a1_z_f64x2, a1_z_f64x2);
|
|
1062
|
+
variance_a_b_f64x2 = vfmaq_f64(variance_a_b_f64x2, a2_z_f64x2, a2_z_f64x2);
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// 2-point tail
|
|
1066
|
+
for (; i + 2 <= n; i += 2) {
|
|
1067
|
+
nk_deinterleave_f64x2_neon_(a + i * 3, &a1_x_f64x2, &a1_y_f64x2, &a1_z_f64x2);
|
|
1068
|
+
nk_deinterleave_f64x2_neon_(b + i * 3, &b1_x_f64x2, &b1_y_f64x2, &b1_z_f64x2);
|
|
1069
|
+
sum_a_x_a_f64x2 = vaddq_f64(sum_a_x_a_f64x2, a1_x_f64x2);
|
|
1070
|
+
sum_a_y_a_f64x2 = vaddq_f64(sum_a_y_a_f64x2, a1_y_f64x2);
|
|
1071
|
+
sum_a_z_a_f64x2 = vaddq_f64(sum_a_z_a_f64x2, a1_z_f64x2);
|
|
1072
|
+
sum_b_x_a_f64x2 = vaddq_f64(sum_b_x_a_f64x2, b1_x_f64x2);
|
|
1073
|
+
sum_b_y_a_f64x2 = vaddq_f64(sum_b_y_a_f64x2, b1_y_f64x2);
|
|
1074
|
+
sum_b_z_a_f64x2 = vaddq_f64(sum_b_z_a_f64x2, b1_z_f64x2);
|
|
1075
|
+
cov_xx_a_f64x2 = vfmaq_f64(cov_xx_a_f64x2, a1_x_f64x2, b1_x_f64x2);
|
|
1076
|
+
cov_xy_a_f64x2 = vfmaq_f64(cov_xy_a_f64x2, a1_x_f64x2, b1_y_f64x2);
|
|
1077
|
+
cov_xz_a_f64x2 = vfmaq_f64(cov_xz_a_f64x2, a1_x_f64x2, b1_z_f64x2);
|
|
1078
|
+
cov_yx_a_f64x2 = vfmaq_f64(cov_yx_a_f64x2, a1_y_f64x2, b1_x_f64x2);
|
|
1079
|
+
cov_yy_a_f64x2 = vfmaq_f64(cov_yy_a_f64x2, a1_y_f64x2, b1_y_f64x2);
|
|
1080
|
+
cov_yz_a_f64x2 = vfmaq_f64(cov_yz_a_f64x2, a1_y_f64x2, b1_z_f64x2);
|
|
1081
|
+
cov_zx_a_f64x2 = vfmaq_f64(cov_zx_a_f64x2, a1_z_f64x2, b1_x_f64x2);
|
|
1082
|
+
cov_zy_a_f64x2 = vfmaq_f64(cov_zy_a_f64x2, a1_z_f64x2, b1_y_f64x2);
|
|
1083
|
+
cov_zz_a_f64x2 = vfmaq_f64(cov_zz_a_f64x2, a1_z_f64x2, b1_z_f64x2);
|
|
1084
|
+
variance_a_a_f64x2 = vfmaq_f64(variance_a_a_f64x2, a1_x_f64x2, a1_x_f64x2);
|
|
1085
|
+
variance_a_a_f64x2 = vfmaq_f64(variance_a_a_f64x2, a1_y_f64x2, a1_y_f64x2);
|
|
1086
|
+
variance_a_a_f64x2 = vfmaq_f64(variance_a_a_f64x2, a1_z_f64x2, a1_z_f64x2);
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// Combine dual accumulators
|
|
1090
|
+
float64x2_t sum_a_x_f64x2 = vaddq_f64(sum_a_x_a_f64x2, sum_a_x_b_f64x2);
|
|
1091
|
+
float64x2_t sum_a_y_f64x2 = vaddq_f64(sum_a_y_a_f64x2, sum_a_y_b_f64x2);
|
|
1092
|
+
float64x2_t sum_a_z_f64x2 = vaddq_f64(sum_a_z_a_f64x2, sum_a_z_b_f64x2);
|
|
1093
|
+
float64x2_t sum_b_x_f64x2 = vaddq_f64(sum_b_x_a_f64x2, sum_b_x_b_f64x2);
|
|
1094
|
+
float64x2_t sum_b_y_f64x2 = vaddq_f64(sum_b_y_a_f64x2, sum_b_y_b_f64x2);
|
|
1095
|
+
float64x2_t sum_b_z_f64x2 = vaddq_f64(sum_b_z_a_f64x2, sum_b_z_b_f64x2);
|
|
1096
|
+
float64x2_t cov_xx_f64x2 = vaddq_f64(cov_xx_a_f64x2, cov_xx_b_f64x2);
|
|
1097
|
+
float64x2_t cov_xy_f64x2 = vaddq_f64(cov_xy_a_f64x2, cov_xy_b_f64x2);
|
|
1098
|
+
float64x2_t cov_xz_f64x2 = vaddq_f64(cov_xz_a_f64x2, cov_xz_b_f64x2);
|
|
1099
|
+
float64x2_t cov_yx_f64x2 = vaddq_f64(cov_yx_a_f64x2, cov_yx_b_f64x2);
|
|
1100
|
+
float64x2_t cov_yy_f64x2 = vaddq_f64(cov_yy_a_f64x2, cov_yy_b_f64x2);
|
|
1101
|
+
float64x2_t cov_yz_f64x2 = vaddq_f64(cov_yz_a_f64x2, cov_yz_b_f64x2);
|
|
1102
|
+
float64x2_t cov_zx_f64x2 = vaddq_f64(cov_zx_a_f64x2, cov_zx_b_f64x2);
|
|
1103
|
+
float64x2_t cov_zy_f64x2 = vaddq_f64(cov_zy_a_f64x2, cov_zy_b_f64x2);
|
|
1104
|
+
float64x2_t cov_zz_f64x2 = vaddq_f64(cov_zz_a_f64x2, cov_zz_b_f64x2);
|
|
1105
|
+
float64x2_t variance_a_f64x2 = vaddq_f64(variance_a_a_f64x2, variance_a_b_f64x2);
|
|
1106
|
+
|
|
1107
|
+
// Reduce vector accumulators.
|
|
1108
|
+
nk_f64_t sum_a_x = nk_reduce_stable_f64x2_neon_(sum_a_x_f64x2), sum_a_x_compensation = 0.0;
|
|
1109
|
+
nk_f64_t sum_a_y = nk_reduce_stable_f64x2_neon_(sum_a_y_f64x2), sum_a_y_compensation = 0.0;
|
|
1110
|
+
nk_f64_t sum_a_z = nk_reduce_stable_f64x2_neon_(sum_a_z_f64x2), sum_a_z_compensation = 0.0;
|
|
1111
|
+
nk_f64_t sum_b_x = nk_reduce_stable_f64x2_neon_(sum_b_x_f64x2), sum_b_x_compensation = 0.0;
|
|
1112
|
+
nk_f64_t sum_b_y = nk_reduce_stable_f64x2_neon_(sum_b_y_f64x2), sum_b_y_compensation = 0.0;
|
|
1113
|
+
nk_f64_t sum_b_z = nk_reduce_stable_f64x2_neon_(sum_b_z_f64x2), sum_b_z_compensation = 0.0;
|
|
1114
|
+
nk_f64_t covariance_x_x = nk_reduce_stable_f64x2_neon_(cov_xx_f64x2), covariance_x_x_compensation = 0.0;
|
|
1115
|
+
nk_f64_t covariance_x_y = nk_reduce_stable_f64x2_neon_(cov_xy_f64x2), covariance_x_y_compensation = 0.0;
|
|
1116
|
+
nk_f64_t covariance_x_z = nk_reduce_stable_f64x2_neon_(cov_xz_f64x2), covariance_x_z_compensation = 0.0;
|
|
1117
|
+
nk_f64_t covariance_y_x = nk_reduce_stable_f64x2_neon_(cov_yx_f64x2), covariance_y_x_compensation = 0.0;
|
|
1118
|
+
nk_f64_t covariance_y_y = nk_reduce_stable_f64x2_neon_(cov_yy_f64x2), covariance_y_y_compensation = 0.0;
|
|
1119
|
+
nk_f64_t covariance_y_z = nk_reduce_stable_f64x2_neon_(cov_yz_f64x2), covariance_y_z_compensation = 0.0;
|
|
1120
|
+
nk_f64_t covariance_z_x = nk_reduce_stable_f64x2_neon_(cov_zx_f64x2), covariance_z_x_compensation = 0.0;
|
|
1121
|
+
nk_f64_t covariance_z_y = nk_reduce_stable_f64x2_neon_(cov_zy_f64x2), covariance_z_y_compensation = 0.0;
|
|
1122
|
+
nk_f64_t covariance_z_z = nk_reduce_stable_f64x2_neon_(cov_zz_f64x2), covariance_z_z_compensation = 0.0;
|
|
1123
|
+
nk_f64_t sum_sq_a = nk_reduce_stable_f64x2_neon_(variance_a_f64x2), sum_sq_a_compensation = 0.0;
|
|
1124
|
+
|
|
1125
|
+
// Scalar tail
|
|
1126
|
+
for (; i < n; ++i) {
|
|
1127
|
+
nk_f64_t ax = a[i * 3 + 0], ay = a[i * 3 + 1], az = a[i * 3 + 2];
|
|
1128
|
+
nk_f64_t bx = b[i * 3 + 0], by = b[i * 3 + 1], bz = b[i * 3 + 2];
|
|
1129
|
+
nk_accumulate_sum_f64_(&sum_a_x, &sum_a_x_compensation, ax),
|
|
1130
|
+
nk_accumulate_sum_f64_(&sum_a_y, &sum_a_y_compensation, ay),
|
|
1131
|
+
nk_accumulate_sum_f64_(&sum_a_z, &sum_a_z_compensation, az);
|
|
1132
|
+
nk_accumulate_sum_f64_(&sum_b_x, &sum_b_x_compensation, bx),
|
|
1133
|
+
nk_accumulate_sum_f64_(&sum_b_y, &sum_b_y_compensation, by),
|
|
1134
|
+
nk_accumulate_sum_f64_(&sum_b_z, &sum_b_z_compensation, bz);
|
|
1135
|
+
nk_accumulate_product_f64_(&covariance_x_x, &covariance_x_x_compensation, ax, bx),
|
|
1136
|
+
nk_accumulate_product_f64_(&covariance_x_y, &covariance_x_y_compensation, ax, by),
|
|
1137
|
+
nk_accumulate_product_f64_(&covariance_x_z, &covariance_x_z_compensation, ax, bz);
|
|
1138
|
+
nk_accumulate_product_f64_(&covariance_y_x, &covariance_y_x_compensation, ay, bx),
|
|
1139
|
+
nk_accumulate_product_f64_(&covariance_y_y, &covariance_y_y_compensation, ay, by),
|
|
1140
|
+
nk_accumulate_product_f64_(&covariance_y_z, &covariance_y_z_compensation, ay, bz);
|
|
1141
|
+
nk_accumulate_product_f64_(&covariance_z_x, &covariance_z_x_compensation, az, bx),
|
|
1142
|
+
nk_accumulate_product_f64_(&covariance_z_y, &covariance_z_y_compensation, az, by),
|
|
1143
|
+
nk_accumulate_product_f64_(&covariance_z_z, &covariance_z_z_compensation, az, bz);
|
|
1144
|
+
nk_accumulate_square_f64_(&sum_sq_a, &sum_sq_a_compensation, ax),
|
|
1145
|
+
nk_accumulate_square_f64_(&sum_sq_a, &sum_sq_a_compensation, ay),
|
|
1146
|
+
nk_accumulate_square_f64_(&sum_sq_a, &sum_sq_a_compensation, az);
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
sum_a_x += sum_a_x_compensation, sum_a_y += sum_a_y_compensation, sum_a_z += sum_a_z_compensation;
|
|
1150
|
+
sum_b_x += sum_b_x_compensation, sum_b_y += sum_b_y_compensation, sum_b_z += sum_b_z_compensation;
|
|
1151
|
+
covariance_x_x += covariance_x_x_compensation, covariance_x_y += covariance_x_y_compensation,
|
|
1152
|
+
covariance_x_z += covariance_x_z_compensation;
|
|
1153
|
+
covariance_y_x += covariance_y_x_compensation, covariance_y_y += covariance_y_y_compensation,
|
|
1154
|
+
covariance_y_z += covariance_y_z_compensation;
|
|
1155
|
+
covariance_z_x += covariance_z_x_compensation, covariance_z_y += covariance_z_y_compensation,
|
|
1156
|
+
covariance_z_z += covariance_z_z_compensation;
|
|
1157
|
+
sum_sq_a += sum_sq_a_compensation;
|
|
1158
|
+
|
|
1159
|
+
// Compute centroids
|
|
1160
|
+
nk_f64_t inv_n = 1.0 / (nk_f64_t)n;
|
|
1161
|
+
nk_f64_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1162
|
+
nk_f64_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1163
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1164
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1165
|
+
|
|
1166
|
+
// Compute variance of A (centered)
|
|
1167
|
+
nk_f64_t centroid_sq = centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z;
|
|
1168
|
+
nk_f64_t var_a = sum_sq_a * inv_n - centroid_sq;
|
|
1169
|
+
|
|
1170
|
+
// Apply centering correction: H_centered = H - n * centroid_a * centroid_bᵀ
|
|
1171
|
+
covariance_x_x -= (nk_f64_t)n * centroid_a_x * centroid_b_x;
|
|
1172
|
+
covariance_x_y -= (nk_f64_t)n * centroid_a_x * centroid_b_y;
|
|
1173
|
+
covariance_x_z -= (nk_f64_t)n * centroid_a_x * centroid_b_z;
|
|
1174
|
+
covariance_y_x -= (nk_f64_t)n * centroid_a_y * centroid_b_x;
|
|
1175
|
+
covariance_y_y -= (nk_f64_t)n * centroid_a_y * centroid_b_y;
|
|
1176
|
+
covariance_y_z -= (nk_f64_t)n * centroid_a_y * centroid_b_z;
|
|
1177
|
+
covariance_z_x -= (nk_f64_t)n * centroid_a_z * centroid_b_x;
|
|
1178
|
+
covariance_z_y -= (nk_f64_t)n * centroid_a_z * centroid_b_y;
|
|
1179
|
+
covariance_z_z -= (nk_f64_t)n * centroid_a_z * centroid_b_z;
|
|
1180
|
+
|
|
1181
|
+
// Compute SVD
|
|
1182
|
+
nk_f64_t cross_covariance[9] = {covariance_x_x, covariance_x_y, covariance_x_z, covariance_y_x, covariance_y_y,
|
|
1183
|
+
covariance_y_z, covariance_z_x, covariance_z_y, covariance_z_z};
|
|
1184
|
+
nk_f64_t svd_u[9], svd_s[9], svd_v[9];
|
|
1185
|
+
nk_svd3x3_f64_(cross_covariance, svd_u, svd_s, svd_v);
|
|
1186
|
+
|
|
1187
|
+
nk_f64_t r[9];
|
|
1188
|
+
nk_rotation_from_svd_f64_neon_(svd_u, svd_v, r);
|
|
1189
|
+
|
|
1190
|
+
// Handle reflection and compute scale
|
|
1191
|
+
nk_f64_t det = nk_det3x3_f64_(r);
|
|
1192
|
+
nk_f64_t trace_d_s = svd_s[0] + svd_s[4] + (det < 0 ? -svd_s[8] : svd_s[8]);
|
|
1193
|
+
nk_f64_t computed_scale = trace_d_s / ((nk_f64_t)n * var_a);
|
|
1194
|
+
|
|
1195
|
+
if (det < 0) {
|
|
1196
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1197
|
+
nk_rotation_from_svd_f64_neon_(svd_u, svd_v, r);
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
if (rotation)
|
|
1201
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
1202
|
+
if (scale) *scale = computed_scale;
|
|
1203
|
+
|
|
1204
|
+
// Compute RMSD after transformation
|
|
1205
|
+
nk_f64_t sum_squared = nk_transformed_ssd_f64_neon_(a, b, n, r, computed_scale, centroid_a_x, centroid_a_y,
|
|
1206
|
+
centroid_a_z, centroid_b_x, centroid_b_y, centroid_b_z);
|
|
1207
|
+
*result = nk_f64_sqrt_neon(sum_squared * inv_n);
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
NK_INTERNAL void nk_deinterleave_f16x8_to_f32x4x2_neon_(nk_f16_t const *ptr, //
|
|
1211
|
+
float32x4_t *x_low_out, float32x4_t *x_high_out, //
|
|
1212
|
+
float32x4_t *y_low_out, float32x4_t *y_high_out, //
|
|
1213
|
+
float32x4_t *z_low_out, float32x4_t *z_high_out) {
|
|
1214
|
+
// Deinterleave 24 f16 values (8 xyz triplets) into separate x, y, z vectors.
|
|
1215
|
+
// Uses NEON vld3q_u16 for efficient stride-3 deinterleaving, then converts to f32.
|
|
1216
|
+
// Avoids vld3q_f16 which is unavailable on MSVC for ARM.
|
|
1217
|
+
//
|
|
1218
|
+
// Input: 24 contiguous f16 values [x0,y0,z0, ..., x7,y7,z7]
|
|
1219
|
+
// Output: x_low[4]+x_high[4], y_low[4]+y_high[4], z_low[4]+z_high[4] vectors in f32
|
|
1220
|
+
uint16x8x3_t xyz_u16x8x3 = vld3q_u16((nk_u16_t const *)ptr);
|
|
1221
|
+
float16x8_t x_f16x8 = vreinterpretq_f16_u16(xyz_u16x8x3.val[0]);
|
|
1222
|
+
float16x8_t y_f16x8 = vreinterpretq_f16_u16(xyz_u16x8x3.val[1]);
|
|
1223
|
+
float16x8_t z_f16x8 = vreinterpretq_f16_u16(xyz_u16x8x3.val[2]);
|
|
1224
|
+
*x_low_out = vcvt_f32_f16(vget_low_f16(x_f16x8));
|
|
1225
|
+
*x_high_out = vcvt_high_f32_f16(x_f16x8);
|
|
1226
|
+
*y_low_out = vcvt_f32_f16(vget_low_f16(y_f16x8));
|
|
1227
|
+
*y_high_out = vcvt_high_f32_f16(y_f16x8);
|
|
1228
|
+
*z_low_out = vcvt_f32_f16(vget_low_f16(z_f16x8));
|
|
1229
|
+
*z_high_out = vcvt_high_f32_f16(z_f16x8);
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
NK_INTERNAL void nk_partial_deinterleave_f16_to_f32x4x2_neon_(nk_f16_t const *ptr, nk_size_t n_points, //
|
|
1233
|
+
float32x4_t *x_low_out, float32x4_t *x_high_out, //
|
|
1234
|
+
float32x4_t *y_low_out, float32x4_t *y_high_out, //
|
|
1235
|
+
float32x4_t *z_low_out, float32x4_t *z_high_out) {
|
|
1236
|
+
nk_u16_t buf[24] = {0};
|
|
1237
|
+
nk_u16_t const *src = (nk_u16_t const *)ptr;
|
|
1238
|
+
for (nk_size_t k = 0; k < n_points * 3; ++k) buf[k] = src[k];
|
|
1239
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_((nk_f16_t const *)buf, x_low_out, x_high_out, y_low_out, y_high_out,
|
|
1240
|
+
z_low_out, z_high_out);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
NK_INTERNAL nk_f32_t nk_transformed_ssd_f16_neon_(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t const *r,
|
|
1244
|
+
nk_f32_t scale, nk_f32_t centroid_a_x, nk_f32_t centroid_a_y,
|
|
1245
|
+
nk_f32_t centroid_a_z, nk_f32_t centroid_b_x, nk_f32_t centroid_b_y,
|
|
1246
|
+
nk_f32_t centroid_b_z) {
|
|
1247
|
+
// Compute sum of squared differences after rigid transformation.
|
|
1248
|
+
// Used by Kabsch algorithm for RMSD computation after rotation is applied.
|
|
1249
|
+
float32x4_t const centroid_a_x_f32x4 = vdupq_n_f32(centroid_a_x);
|
|
1250
|
+
float32x4_t const centroid_a_y_f32x4 = vdupq_n_f32(centroid_a_y);
|
|
1251
|
+
float32x4_t const centroid_a_z_f32x4 = vdupq_n_f32(centroid_a_z);
|
|
1252
|
+
float32x4_t const centroid_b_x_f32x4 = vdupq_n_f32(centroid_b_x);
|
|
1253
|
+
float32x4_t const centroid_b_y_f32x4 = vdupq_n_f32(centroid_b_y);
|
|
1254
|
+
float32x4_t const centroid_b_z_f32x4 = vdupq_n_f32(centroid_b_z);
|
|
1255
|
+
float32x4_t const scale_f32x4 = vdupq_n_f32(scale);
|
|
1256
|
+
|
|
1257
|
+
// Load rotation matrix elements
|
|
1258
|
+
float32x4_t const r00_f32x4 = vdupq_n_f32(r[0]), r01_f32x4 = vdupq_n_f32(r[1]), r02_f32x4 = vdupq_n_f32(r[2]);
|
|
1259
|
+
float32x4_t const r10_f32x4 = vdupq_n_f32(r[3]), r11_f32x4 = vdupq_n_f32(r[4]), r12_f32x4 = vdupq_n_f32(r[5]);
|
|
1260
|
+
float32x4_t const r20_f32x4 = vdupq_n_f32(r[6]), r21_f32x4 = vdupq_n_f32(r[7]), r22_f32x4 = vdupq_n_f32(r[8]);
|
|
1261
|
+
|
|
1262
|
+
float32x4_t sum_squared_f32x4 = vdupq_n_f32(0);
|
|
1263
|
+
float32x4_t a_x_low_f32x4, a_x_high_f32x4, a_y_low_f32x4, a_y_high_f32x4, a_z_low_f32x4, a_z_high_f32x4;
|
|
1264
|
+
float32x4_t b_x_low_f32x4, b_x_high_f32x4, b_y_low_f32x4, b_y_high_f32x4, b_z_low_f32x4, b_z_high_f32x4;
|
|
1265
|
+
|
|
1266
|
+
nk_size_t j = 0;
|
|
1267
|
+
for (; j + 8 <= n; j += 8) {
|
|
1268
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(a + j * 3, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1269
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1270
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(b + j * 3, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1271
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1272
|
+
|
|
1273
|
+
// Center points → low half
|
|
1274
|
+
float32x4_t pa_x_f32x4 = vsubq_f32(a_x_low_f32x4, centroid_a_x_f32x4);
|
|
1275
|
+
float32x4_t pa_y_f32x4 = vsubq_f32(a_y_low_f32x4, centroid_a_y_f32x4);
|
|
1276
|
+
float32x4_t pa_z_f32x4 = vsubq_f32(a_z_low_f32x4, centroid_a_z_f32x4);
|
|
1277
|
+
float32x4_t pb_x_f32x4 = vsubq_f32(b_x_low_f32x4, centroid_b_x_f32x4);
|
|
1278
|
+
float32x4_t pb_y_f32x4 = vsubq_f32(b_y_low_f32x4, centroid_b_y_f32x4);
|
|
1279
|
+
float32x4_t pb_z_f32x4 = vsubq_f32(b_z_low_f32x4, centroid_b_z_f32x4);
|
|
1280
|
+
float32x4_t ra_x_f32x4 = vmulq_f32(
|
|
1281
|
+
scale_f32x4,
|
|
1282
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r00_f32x4, pa_x_f32x4), r01_f32x4, pa_y_f32x4), r02_f32x4, pa_z_f32x4));
|
|
1283
|
+
float32x4_t ra_y_f32x4 = vmulq_f32(
|
|
1284
|
+
scale_f32x4,
|
|
1285
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r10_f32x4, pa_x_f32x4), r11_f32x4, pa_y_f32x4), r12_f32x4, pa_z_f32x4));
|
|
1286
|
+
float32x4_t ra_z_f32x4 = vmulq_f32(
|
|
1287
|
+
scale_f32x4,
|
|
1288
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r20_f32x4, pa_x_f32x4), r21_f32x4, pa_y_f32x4), r22_f32x4, pa_z_f32x4));
|
|
1289
|
+
float32x4_t delta_x_f32x4 = vsubq_f32(ra_x_f32x4, pb_x_f32x4);
|
|
1290
|
+
float32x4_t delta_y_f32x4 = vsubq_f32(ra_y_f32x4, pb_y_f32x4);
|
|
1291
|
+
float32x4_t delta_z_f32x4 = vsubq_f32(ra_z_f32x4, pb_z_f32x4);
|
|
1292
|
+
sum_squared_f32x4 = vfmaq_f32(sum_squared_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1293
|
+
sum_squared_f32x4 = vfmaq_f32(sum_squared_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1294
|
+
sum_squared_f32x4 = vfmaq_f32(sum_squared_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1295
|
+
|
|
1296
|
+
// Center points → high half
|
|
1297
|
+
pa_x_f32x4 = vsubq_f32(a_x_high_f32x4, centroid_a_x_f32x4);
|
|
1298
|
+
pa_y_f32x4 = vsubq_f32(a_y_high_f32x4, centroid_a_y_f32x4);
|
|
1299
|
+
pa_z_f32x4 = vsubq_f32(a_z_high_f32x4, centroid_a_z_f32x4);
|
|
1300
|
+
pb_x_f32x4 = vsubq_f32(b_x_high_f32x4, centroid_b_x_f32x4);
|
|
1301
|
+
pb_y_f32x4 = vsubq_f32(b_y_high_f32x4, centroid_b_y_f32x4);
|
|
1302
|
+
pb_z_f32x4 = vsubq_f32(b_z_high_f32x4, centroid_b_z_f32x4);
|
|
1303
|
+
ra_x_f32x4 = vmulq_f32(
|
|
1304
|
+
scale_f32x4,
|
|
1305
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r00_f32x4, pa_x_f32x4), r01_f32x4, pa_y_f32x4), r02_f32x4, pa_z_f32x4));
|
|
1306
|
+
ra_y_f32x4 = vmulq_f32(
|
|
1307
|
+
scale_f32x4,
|
|
1308
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r10_f32x4, pa_x_f32x4), r11_f32x4, pa_y_f32x4), r12_f32x4, pa_z_f32x4));
|
|
1309
|
+
ra_z_f32x4 = vmulq_f32(
|
|
1310
|
+
scale_f32x4,
|
|
1311
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r20_f32x4, pa_x_f32x4), r21_f32x4, pa_y_f32x4), r22_f32x4, pa_z_f32x4));
|
|
1312
|
+
delta_x_f32x4 = vsubq_f32(ra_x_f32x4, pb_x_f32x4);
|
|
1313
|
+
delta_y_f32x4 = vsubq_f32(ra_y_f32x4, pb_y_f32x4);
|
|
1314
|
+
delta_z_f32x4 = vsubq_f32(ra_z_f32x4, pb_z_f32x4);
|
|
1315
|
+
sum_squared_f32x4 = vfmaq_f32(sum_squared_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1316
|
+
sum_squared_f32x4 = vfmaq_f32(sum_squared_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1317
|
+
sum_squared_f32x4 = vfmaq_f32(sum_squared_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
// Reduce to scalar
|
|
1321
|
+
nk_f32_t sum_squared = vaddvq_f32(sum_squared_f32x4);
|
|
1322
|
+
|
|
1323
|
+
if (j < n) {
|
|
1324
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(a + j * 3, n - j, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1325
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1326
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(b + j * 3, n - j, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1327
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1328
|
+
|
|
1329
|
+
// Low half
|
|
1330
|
+
float32x4_t pa_x_f32x4 = vsubq_f32(a_x_low_f32x4, centroid_a_x_f32x4);
|
|
1331
|
+
float32x4_t pa_y_f32x4 = vsubq_f32(a_y_low_f32x4, centroid_a_y_f32x4);
|
|
1332
|
+
float32x4_t pa_z_f32x4 = vsubq_f32(a_z_low_f32x4, centroid_a_z_f32x4);
|
|
1333
|
+
float32x4_t pb_x_f32x4 = vsubq_f32(b_x_low_f32x4, centroid_b_x_f32x4);
|
|
1334
|
+
float32x4_t pb_y_f32x4 = vsubq_f32(b_y_low_f32x4, centroid_b_y_f32x4);
|
|
1335
|
+
float32x4_t pb_z_f32x4 = vsubq_f32(b_z_low_f32x4, centroid_b_z_f32x4);
|
|
1336
|
+
float32x4_t ra_x_f32x4 = vmulq_f32(
|
|
1337
|
+
scale_f32x4,
|
|
1338
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r00_f32x4, pa_x_f32x4), r01_f32x4, pa_y_f32x4), r02_f32x4, pa_z_f32x4));
|
|
1339
|
+
float32x4_t ra_y_f32x4 = vmulq_f32(
|
|
1340
|
+
scale_f32x4,
|
|
1341
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r10_f32x4, pa_x_f32x4), r11_f32x4, pa_y_f32x4), r12_f32x4, pa_z_f32x4));
|
|
1342
|
+
float32x4_t ra_z_f32x4 = vmulq_f32(
|
|
1343
|
+
scale_f32x4,
|
|
1344
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r20_f32x4, pa_x_f32x4), r21_f32x4, pa_y_f32x4), r22_f32x4, pa_z_f32x4));
|
|
1345
|
+
float32x4_t delta_x_f32x4 = vsubq_f32(ra_x_f32x4, pb_x_f32x4);
|
|
1346
|
+
float32x4_t delta_y_f32x4 = vsubq_f32(ra_y_f32x4, pb_y_f32x4);
|
|
1347
|
+
float32x4_t delta_z_f32x4 = vsubq_f32(ra_z_f32x4, pb_z_f32x4);
|
|
1348
|
+
float32x4_t tail_sum_f32x4 = vmulq_f32(delta_x_f32x4, delta_x_f32x4);
|
|
1349
|
+
tail_sum_f32x4 = vfmaq_f32(tail_sum_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1350
|
+
tail_sum_f32x4 = vfmaq_f32(tail_sum_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1351
|
+
|
|
1352
|
+
// High half
|
|
1353
|
+
pa_x_f32x4 = vsubq_f32(a_x_high_f32x4, centroid_a_x_f32x4);
|
|
1354
|
+
pa_y_f32x4 = vsubq_f32(a_y_high_f32x4, centroid_a_y_f32x4);
|
|
1355
|
+
pa_z_f32x4 = vsubq_f32(a_z_high_f32x4, centroid_a_z_f32x4);
|
|
1356
|
+
pb_x_f32x4 = vsubq_f32(b_x_high_f32x4, centroid_b_x_f32x4);
|
|
1357
|
+
pb_y_f32x4 = vsubq_f32(b_y_high_f32x4, centroid_b_y_f32x4);
|
|
1358
|
+
pb_z_f32x4 = vsubq_f32(b_z_high_f32x4, centroid_b_z_f32x4);
|
|
1359
|
+
ra_x_f32x4 = vmulq_f32(
|
|
1360
|
+
scale_f32x4,
|
|
1361
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r00_f32x4, pa_x_f32x4), r01_f32x4, pa_y_f32x4), r02_f32x4, pa_z_f32x4));
|
|
1362
|
+
ra_y_f32x4 = vmulq_f32(
|
|
1363
|
+
scale_f32x4,
|
|
1364
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r10_f32x4, pa_x_f32x4), r11_f32x4, pa_y_f32x4), r12_f32x4, pa_z_f32x4));
|
|
1365
|
+
ra_z_f32x4 = vmulq_f32(
|
|
1366
|
+
scale_f32x4,
|
|
1367
|
+
vfmaq_f32(vfmaq_f32(vmulq_f32(r20_f32x4, pa_x_f32x4), r21_f32x4, pa_y_f32x4), r22_f32x4, pa_z_f32x4));
|
|
1368
|
+
delta_x_f32x4 = vsubq_f32(ra_x_f32x4, pb_x_f32x4);
|
|
1369
|
+
delta_y_f32x4 = vsubq_f32(ra_y_f32x4, pb_y_f32x4);
|
|
1370
|
+
delta_z_f32x4 = vsubq_f32(ra_z_f32x4, pb_z_f32x4);
|
|
1371
|
+
tail_sum_f32x4 = vfmaq_f32(tail_sum_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1372
|
+
tail_sum_f32x4 = vfmaq_f32(tail_sum_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1373
|
+
tail_sum_f32x4 = vfmaq_f32(tail_sum_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1374
|
+
sum_squared += vaddvq_f32(tail_sum_f32x4);
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
return sum_squared;
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
/**
|
|
1381
|
+
* @brief RMSD (Root Mean Square Deviation) computation using NEON FP16 with widening to FP32.
|
|
1382
|
+
* Computes the RMS of distances between corresponding points after centroid alignment.
|
|
1383
|
+
*/
|
|
1384
|
+
NK_PUBLIC void nk_rmsd_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1385
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1386
|
+
// RMSD uses identity rotation and scale=1.0
|
|
1387
|
+
if (rotation)
|
|
1388
|
+
rotation[0] = 1, rotation[1] = 0, rotation[2] = 0, rotation[3] = 0, rotation[4] = 1, rotation[5] = 0,
|
|
1389
|
+
rotation[6] = 0, rotation[7] = 0, rotation[8] = 1;
|
|
1390
|
+
if (scale) *scale = 1.0f;
|
|
1391
|
+
|
|
1392
|
+
float32x4_t const zeros_f32x4 = vdupq_n_f32(0);
|
|
1393
|
+
|
|
1394
|
+
// Accumulators for centroids and squared differences (all in f32)
|
|
1395
|
+
float32x4_t sum_a_x_f32x4 = zeros_f32x4, sum_a_y_f32x4 = zeros_f32x4, sum_a_z_f32x4 = zeros_f32x4;
|
|
1396
|
+
float32x4_t sum_b_x_f32x4 = zeros_f32x4, sum_b_y_f32x4 = zeros_f32x4, sum_b_z_f32x4 = zeros_f32x4;
|
|
1397
|
+
float32x4_t sum_squared_x_f32x4 = zeros_f32x4, sum_squared_y_f32x4 = zeros_f32x4, sum_squared_z_f32x4 = zeros_f32x4;
|
|
1398
|
+
|
|
1399
|
+
float32x4_t a_x_low_f32x4, a_x_high_f32x4, a_y_low_f32x4, a_y_high_f32x4, a_z_low_f32x4, a_z_high_f32x4;
|
|
1400
|
+
float32x4_t b_x_low_f32x4, b_x_high_f32x4, b_y_low_f32x4, b_y_high_f32x4, b_z_low_f32x4, b_z_high_f32x4;
|
|
1401
|
+
nk_size_t i = 0;
|
|
1402
|
+
|
|
1403
|
+
for (; i + 8 <= n; i += 8) {
|
|
1404
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(a + i * 3, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1405
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1406
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(b + i * 3, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1407
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1408
|
+
|
|
1409
|
+
sum_a_x_f32x4 = vaddq_f32(vaddq_f32(sum_a_x_f32x4, a_x_low_f32x4), a_x_high_f32x4);
|
|
1410
|
+
sum_a_y_f32x4 = vaddq_f32(vaddq_f32(sum_a_y_f32x4, a_y_low_f32x4), a_y_high_f32x4);
|
|
1411
|
+
sum_a_z_f32x4 = vaddq_f32(vaddq_f32(sum_a_z_f32x4, a_z_low_f32x4), a_z_high_f32x4);
|
|
1412
|
+
sum_b_x_f32x4 = vaddq_f32(vaddq_f32(sum_b_x_f32x4, b_x_low_f32x4), b_x_high_f32x4);
|
|
1413
|
+
sum_b_y_f32x4 = vaddq_f32(vaddq_f32(sum_b_y_f32x4, b_y_low_f32x4), b_y_high_f32x4);
|
|
1414
|
+
sum_b_z_f32x4 = vaddq_f32(vaddq_f32(sum_b_z_f32x4, b_z_low_f32x4), b_z_high_f32x4);
|
|
1415
|
+
|
|
1416
|
+
float32x4_t delta_x_f32x4 = vsubq_f32(a_x_low_f32x4, b_x_low_f32x4);
|
|
1417
|
+
float32x4_t delta_y_f32x4 = vsubq_f32(a_y_low_f32x4, b_y_low_f32x4);
|
|
1418
|
+
float32x4_t delta_z_f32x4 = vsubq_f32(a_z_low_f32x4, b_z_low_f32x4);
|
|
1419
|
+
sum_squared_x_f32x4 = vfmaq_f32(sum_squared_x_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1420
|
+
sum_squared_y_f32x4 = vfmaq_f32(sum_squared_y_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1421
|
+
sum_squared_z_f32x4 = vfmaq_f32(sum_squared_z_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1422
|
+
|
|
1423
|
+
delta_x_f32x4 = vsubq_f32(a_x_high_f32x4, b_x_high_f32x4);
|
|
1424
|
+
delta_y_f32x4 = vsubq_f32(a_y_high_f32x4, b_y_high_f32x4);
|
|
1425
|
+
delta_z_f32x4 = vsubq_f32(a_z_high_f32x4, b_z_high_f32x4);
|
|
1426
|
+
sum_squared_x_f32x4 = vfmaq_f32(sum_squared_x_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1427
|
+
sum_squared_y_f32x4 = vfmaq_f32(sum_squared_y_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1428
|
+
sum_squared_z_f32x4 = vfmaq_f32(sum_squared_z_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
if (i < n) {
|
|
1432
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(a + i * 3, n - i, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1433
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1434
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(b + i * 3, n - i, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1435
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1436
|
+
|
|
1437
|
+
sum_a_x_f32x4 = vaddq_f32(vaddq_f32(sum_a_x_f32x4, a_x_low_f32x4), a_x_high_f32x4);
|
|
1438
|
+
sum_a_y_f32x4 = vaddq_f32(vaddq_f32(sum_a_y_f32x4, a_y_low_f32x4), a_y_high_f32x4);
|
|
1439
|
+
sum_a_z_f32x4 = vaddq_f32(vaddq_f32(sum_a_z_f32x4, a_z_low_f32x4), a_z_high_f32x4);
|
|
1440
|
+
sum_b_x_f32x4 = vaddq_f32(vaddq_f32(sum_b_x_f32x4, b_x_low_f32x4), b_x_high_f32x4);
|
|
1441
|
+
sum_b_y_f32x4 = vaddq_f32(vaddq_f32(sum_b_y_f32x4, b_y_low_f32x4), b_y_high_f32x4);
|
|
1442
|
+
sum_b_z_f32x4 = vaddq_f32(vaddq_f32(sum_b_z_f32x4, b_z_low_f32x4), b_z_high_f32x4);
|
|
1443
|
+
|
|
1444
|
+
float32x4_t delta_x_f32x4 = vsubq_f32(a_x_low_f32x4, b_x_low_f32x4);
|
|
1445
|
+
float32x4_t delta_y_f32x4 = vsubq_f32(a_y_low_f32x4, b_y_low_f32x4);
|
|
1446
|
+
float32x4_t delta_z_f32x4 = vsubq_f32(a_z_low_f32x4, b_z_low_f32x4);
|
|
1447
|
+
sum_squared_x_f32x4 = vfmaq_f32(sum_squared_x_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1448
|
+
sum_squared_y_f32x4 = vfmaq_f32(sum_squared_y_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1449
|
+
sum_squared_z_f32x4 = vfmaq_f32(sum_squared_z_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1450
|
+
|
|
1451
|
+
delta_x_f32x4 = vsubq_f32(a_x_high_f32x4, b_x_high_f32x4);
|
|
1452
|
+
delta_y_f32x4 = vsubq_f32(a_y_high_f32x4, b_y_high_f32x4);
|
|
1453
|
+
delta_z_f32x4 = vsubq_f32(a_z_high_f32x4, b_z_high_f32x4);
|
|
1454
|
+
sum_squared_x_f32x4 = vfmaq_f32(sum_squared_x_f32x4, delta_x_f32x4, delta_x_f32x4);
|
|
1455
|
+
sum_squared_y_f32x4 = vfmaq_f32(sum_squared_y_f32x4, delta_y_f32x4, delta_y_f32x4);
|
|
1456
|
+
sum_squared_z_f32x4 = vfmaq_f32(sum_squared_z_f32x4, delta_z_f32x4, delta_z_f32x4);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
// Reduce vectors to scalars
|
|
1460
|
+
nk_f32_t total_ax = vaddvq_f32(sum_a_x_f32x4);
|
|
1461
|
+
nk_f32_t total_ay = vaddvq_f32(sum_a_y_f32x4);
|
|
1462
|
+
nk_f32_t total_az = vaddvq_f32(sum_a_z_f32x4);
|
|
1463
|
+
nk_f32_t total_bx = vaddvq_f32(sum_b_x_f32x4);
|
|
1464
|
+
nk_f32_t total_by = vaddvq_f32(sum_b_y_f32x4);
|
|
1465
|
+
nk_f32_t total_bz = vaddvq_f32(sum_b_z_f32x4);
|
|
1466
|
+
nk_f32_t total_sq_x = vaddvq_f32(sum_squared_x_f32x4);
|
|
1467
|
+
nk_f32_t total_sq_y = vaddvq_f32(sum_squared_y_f32x4);
|
|
1468
|
+
nk_f32_t total_sq_z = vaddvq_f32(sum_squared_z_f32x4);
|
|
1469
|
+
|
|
1470
|
+
// Compute centroids
|
|
1471
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1472
|
+
nk_f32_t centroid_a_x = total_ax * inv_n;
|
|
1473
|
+
nk_f32_t centroid_a_y = total_ay * inv_n;
|
|
1474
|
+
nk_f32_t centroid_a_z = total_az * inv_n;
|
|
1475
|
+
nk_f32_t centroid_b_x = total_bx * inv_n;
|
|
1476
|
+
nk_f32_t centroid_b_y = total_by * inv_n;
|
|
1477
|
+
nk_f32_t centroid_b_z = total_bz * inv_n;
|
|
1478
|
+
|
|
1479
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1480
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1481
|
+
|
|
1482
|
+
// Compute RMSD
|
|
1483
|
+
nk_f32_t mean_diff_x = centroid_a_x - centroid_b_x;
|
|
1484
|
+
nk_f32_t mean_diff_y = centroid_a_y - centroid_b_y;
|
|
1485
|
+
nk_f32_t mean_diff_z = centroid_a_z - centroid_b_z;
|
|
1486
|
+
nk_f32_t sum_squared = total_sq_x + total_sq_y + total_sq_z;
|
|
1487
|
+
nk_f32_t mean_diff_sq = mean_diff_x * mean_diff_x + mean_diff_y * mean_diff_y + mean_diff_z * mean_diff_z;
|
|
1488
|
+
|
|
1489
|
+
*result = nk_f32_sqrt_neon(sum_squared * inv_n - mean_diff_sq);
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
/**
|
|
1493
|
+
* @brief Kabsch algorithm for optimal rigid body superposition using NEON FP16 with widening to FP32.
|
|
1494
|
+
* Finds the rotation matrix R that minimizes RMSD between two point sets.
|
|
1495
|
+
*/
|
|
1496
|
+
NK_PUBLIC void nk_kabsch_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1497
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1498
|
+
// Fused single-pass: load f16, convert to f32, compute centroids and covariance
|
|
1499
|
+
float32x4_t const zeros_f32x4 = vdupq_n_f32(0);
|
|
1500
|
+
|
|
1501
|
+
// Accumulators for centroids (f32)
|
|
1502
|
+
float32x4_t sum_a_x_f32x4 = zeros_f32x4, sum_a_y_f32x4 = zeros_f32x4, sum_a_z_f32x4 = zeros_f32x4;
|
|
1503
|
+
float32x4_t sum_b_x_f32x4 = zeros_f32x4, sum_b_y_f32x4 = zeros_f32x4, sum_b_z_f32x4 = zeros_f32x4;
|
|
1504
|
+
|
|
1505
|
+
// Accumulators for covariance matrix (sum of outer products)
|
|
1506
|
+
float32x4_t cov_xx_f32x4 = zeros_f32x4, cov_xy_f32x4 = zeros_f32x4, cov_xz_f32x4 = zeros_f32x4;
|
|
1507
|
+
float32x4_t cov_yx_f32x4 = zeros_f32x4, cov_yy_f32x4 = zeros_f32x4, cov_yz_f32x4 = zeros_f32x4;
|
|
1508
|
+
float32x4_t cov_zx_f32x4 = zeros_f32x4, cov_zy_f32x4 = zeros_f32x4, cov_zz_f32x4 = zeros_f32x4;
|
|
1509
|
+
|
|
1510
|
+
nk_size_t i = 0;
|
|
1511
|
+
float32x4_t a_x_low_f32x4, a_x_high_f32x4, a_y_low_f32x4, a_y_high_f32x4, a_z_low_f32x4, a_z_high_f32x4;
|
|
1512
|
+
float32x4_t b_x_low_f32x4, b_x_high_f32x4, b_y_low_f32x4, b_y_high_f32x4, b_z_low_f32x4, b_z_high_f32x4;
|
|
1513
|
+
|
|
1514
|
+
for (; i + 8 <= n; i += 8) {
|
|
1515
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(a + i * 3, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1516
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1517
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(b + i * 3, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1518
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1519
|
+
|
|
1520
|
+
sum_a_x_f32x4 = vaddq_f32(vaddq_f32(sum_a_x_f32x4, a_x_low_f32x4), a_x_high_f32x4);
|
|
1521
|
+
sum_a_y_f32x4 = vaddq_f32(vaddq_f32(sum_a_y_f32x4, a_y_low_f32x4), a_y_high_f32x4);
|
|
1522
|
+
sum_a_z_f32x4 = vaddq_f32(vaddq_f32(sum_a_z_f32x4, a_z_low_f32x4), a_z_high_f32x4);
|
|
1523
|
+
sum_b_x_f32x4 = vaddq_f32(vaddq_f32(sum_b_x_f32x4, b_x_low_f32x4), b_x_high_f32x4);
|
|
1524
|
+
sum_b_y_f32x4 = vaddq_f32(vaddq_f32(sum_b_y_f32x4, b_y_low_f32x4), b_y_high_f32x4);
|
|
1525
|
+
sum_b_z_f32x4 = vaddq_f32(vaddq_f32(sum_b_z_f32x4, b_z_low_f32x4), b_z_high_f32x4);
|
|
1526
|
+
|
|
1527
|
+
cov_xx_f32x4 = vfmaq_f32(vfmaq_f32(cov_xx_f32x4, a_x_low_f32x4, b_x_low_f32x4), a_x_high_f32x4, b_x_high_f32x4);
|
|
1528
|
+
cov_xy_f32x4 = vfmaq_f32(vfmaq_f32(cov_xy_f32x4, a_x_low_f32x4, b_y_low_f32x4), a_x_high_f32x4, b_y_high_f32x4);
|
|
1529
|
+
cov_xz_f32x4 = vfmaq_f32(vfmaq_f32(cov_xz_f32x4, a_x_low_f32x4, b_z_low_f32x4), a_x_high_f32x4, b_z_high_f32x4);
|
|
1530
|
+
cov_yx_f32x4 = vfmaq_f32(vfmaq_f32(cov_yx_f32x4, a_y_low_f32x4, b_x_low_f32x4), a_y_high_f32x4, b_x_high_f32x4);
|
|
1531
|
+
cov_yy_f32x4 = vfmaq_f32(vfmaq_f32(cov_yy_f32x4, a_y_low_f32x4, b_y_low_f32x4), a_y_high_f32x4, b_y_high_f32x4);
|
|
1532
|
+
cov_yz_f32x4 = vfmaq_f32(vfmaq_f32(cov_yz_f32x4, a_y_low_f32x4, b_z_low_f32x4), a_y_high_f32x4, b_z_high_f32x4);
|
|
1533
|
+
cov_zx_f32x4 = vfmaq_f32(vfmaq_f32(cov_zx_f32x4, a_z_low_f32x4, b_x_low_f32x4), a_z_high_f32x4, b_x_high_f32x4);
|
|
1534
|
+
cov_zy_f32x4 = vfmaq_f32(vfmaq_f32(cov_zy_f32x4, a_z_low_f32x4, b_y_low_f32x4), a_z_high_f32x4, b_y_high_f32x4);
|
|
1535
|
+
cov_zz_f32x4 = vfmaq_f32(vfmaq_f32(cov_zz_f32x4, a_z_low_f32x4, b_z_low_f32x4), a_z_high_f32x4, b_z_high_f32x4);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
if (i < n) {
|
|
1539
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(a + i * 3, n - i, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1540
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1541
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(b + i * 3, n - i, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1542
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1543
|
+
|
|
1544
|
+
sum_a_x_f32x4 = vaddq_f32(vaddq_f32(sum_a_x_f32x4, a_x_low_f32x4), a_x_high_f32x4);
|
|
1545
|
+
sum_a_y_f32x4 = vaddq_f32(vaddq_f32(sum_a_y_f32x4, a_y_low_f32x4), a_y_high_f32x4);
|
|
1546
|
+
sum_a_z_f32x4 = vaddq_f32(vaddq_f32(sum_a_z_f32x4, a_z_low_f32x4), a_z_high_f32x4);
|
|
1547
|
+
sum_b_x_f32x4 = vaddq_f32(vaddq_f32(sum_b_x_f32x4, b_x_low_f32x4), b_x_high_f32x4);
|
|
1548
|
+
sum_b_y_f32x4 = vaddq_f32(vaddq_f32(sum_b_y_f32x4, b_y_low_f32x4), b_y_high_f32x4);
|
|
1549
|
+
sum_b_z_f32x4 = vaddq_f32(vaddq_f32(sum_b_z_f32x4, b_z_low_f32x4), b_z_high_f32x4);
|
|
1550
|
+
|
|
1551
|
+
cov_xx_f32x4 = vfmaq_f32(vfmaq_f32(cov_xx_f32x4, a_x_low_f32x4, b_x_low_f32x4), a_x_high_f32x4, b_x_high_f32x4);
|
|
1552
|
+
cov_xy_f32x4 = vfmaq_f32(vfmaq_f32(cov_xy_f32x4, a_x_low_f32x4, b_y_low_f32x4), a_x_high_f32x4, b_y_high_f32x4);
|
|
1553
|
+
cov_xz_f32x4 = vfmaq_f32(vfmaq_f32(cov_xz_f32x4, a_x_low_f32x4, b_z_low_f32x4), a_x_high_f32x4, b_z_high_f32x4);
|
|
1554
|
+
cov_yx_f32x4 = vfmaq_f32(vfmaq_f32(cov_yx_f32x4, a_y_low_f32x4, b_x_low_f32x4), a_y_high_f32x4, b_x_high_f32x4);
|
|
1555
|
+
cov_yy_f32x4 = vfmaq_f32(vfmaq_f32(cov_yy_f32x4, a_y_low_f32x4, b_y_low_f32x4), a_y_high_f32x4, b_y_high_f32x4);
|
|
1556
|
+
cov_yz_f32x4 = vfmaq_f32(vfmaq_f32(cov_yz_f32x4, a_y_low_f32x4, b_z_low_f32x4), a_y_high_f32x4, b_z_high_f32x4);
|
|
1557
|
+
cov_zx_f32x4 = vfmaq_f32(vfmaq_f32(cov_zx_f32x4, a_z_low_f32x4, b_x_low_f32x4), a_z_high_f32x4, b_x_high_f32x4);
|
|
1558
|
+
cov_zy_f32x4 = vfmaq_f32(vfmaq_f32(cov_zy_f32x4, a_z_low_f32x4, b_y_low_f32x4), a_z_high_f32x4, b_y_high_f32x4);
|
|
1559
|
+
cov_zz_f32x4 = vfmaq_f32(vfmaq_f32(cov_zz_f32x4, a_z_low_f32x4, b_z_low_f32x4), a_z_high_f32x4, b_z_high_f32x4);
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
// Reduce vector accumulators
|
|
1563
|
+
nk_f32_t sum_a_x = vaddvq_f32(sum_a_x_f32x4);
|
|
1564
|
+
nk_f32_t sum_a_y = vaddvq_f32(sum_a_y_f32x4);
|
|
1565
|
+
nk_f32_t sum_a_z = vaddvq_f32(sum_a_z_f32x4);
|
|
1566
|
+
nk_f32_t sum_b_x = vaddvq_f32(sum_b_x_f32x4);
|
|
1567
|
+
nk_f32_t sum_b_y = vaddvq_f32(sum_b_y_f32x4);
|
|
1568
|
+
nk_f32_t sum_b_z = vaddvq_f32(sum_b_z_f32x4);
|
|
1569
|
+
|
|
1570
|
+
nk_f32_t covariance_x_x = vaddvq_f32(cov_xx_f32x4);
|
|
1571
|
+
nk_f32_t covariance_x_y = vaddvq_f32(cov_xy_f32x4);
|
|
1572
|
+
nk_f32_t covariance_x_z = vaddvq_f32(cov_xz_f32x4);
|
|
1573
|
+
nk_f32_t covariance_y_x = vaddvq_f32(cov_yx_f32x4);
|
|
1574
|
+
nk_f32_t covariance_y_y = vaddvq_f32(cov_yy_f32x4);
|
|
1575
|
+
nk_f32_t covariance_y_z = vaddvq_f32(cov_yz_f32x4);
|
|
1576
|
+
nk_f32_t covariance_z_x = vaddvq_f32(cov_zx_f32x4);
|
|
1577
|
+
nk_f32_t covariance_z_y = vaddvq_f32(cov_zy_f32x4);
|
|
1578
|
+
nk_f32_t covariance_z_z = vaddvq_f32(cov_zz_f32x4);
|
|
1579
|
+
|
|
1580
|
+
// Compute centroids
|
|
1581
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1582
|
+
nk_f32_t centroid_a_x = sum_a_x * inv_n;
|
|
1583
|
+
nk_f32_t centroid_a_y = sum_a_y * inv_n;
|
|
1584
|
+
nk_f32_t centroid_a_z = sum_a_z * inv_n;
|
|
1585
|
+
nk_f32_t centroid_b_x = sum_b_x * inv_n;
|
|
1586
|
+
nk_f32_t centroid_b_y = sum_b_y * inv_n;
|
|
1587
|
+
nk_f32_t centroid_b_z = sum_b_z * inv_n;
|
|
1588
|
+
|
|
1589
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1590
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1591
|
+
|
|
1592
|
+
// Compute centered covariance: H = (A - centroid_A)ᵀ * (B - centroid_B)
|
|
1593
|
+
// H = sum(a * bᵀ) - n * centroid_a * centroid_bᵀ
|
|
1594
|
+
nk_f32_t h[9];
|
|
1595
|
+
h[0] = covariance_x_x - (nk_f32_t)n * centroid_a_x * centroid_b_x;
|
|
1596
|
+
h[1] = covariance_x_y - (nk_f32_t)n * centroid_a_x * centroid_b_y;
|
|
1597
|
+
h[2] = covariance_x_z - (nk_f32_t)n * centroid_a_x * centroid_b_z;
|
|
1598
|
+
h[3] = covariance_y_x - (nk_f32_t)n * centroid_a_y * centroid_b_x;
|
|
1599
|
+
h[4] = covariance_y_y - (nk_f32_t)n * centroid_a_y * centroid_b_y;
|
|
1600
|
+
h[5] = covariance_y_z - (nk_f32_t)n * centroid_a_y * centroid_b_z;
|
|
1601
|
+
h[6] = covariance_z_x - (nk_f32_t)n * centroid_a_z * centroid_b_x;
|
|
1602
|
+
h[7] = covariance_z_y - (nk_f32_t)n * centroid_a_z * centroid_b_y;
|
|
1603
|
+
h[8] = covariance_z_z - (nk_f32_t)n * centroid_a_z * centroid_b_z;
|
|
1604
|
+
|
|
1605
|
+
// SVD of H = U * S * Vᵀ
|
|
1606
|
+
nk_f32_t svd_u[9], svd_s[9], svd_v[9];
|
|
1607
|
+
nk_svd3x3_f32_(h, svd_u, svd_s, svd_v);
|
|
1608
|
+
|
|
1609
|
+
// R = V * Uᵀ
|
|
1610
|
+
nk_f32_t r[9];
|
|
1611
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
1612
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
1613
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
1614
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
1615
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
1616
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
1617
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
1618
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
1619
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
1620
|
+
|
|
1621
|
+
// Handle reflection: if det(R) < 0, negate third column of V and recompute
|
|
1622
|
+
nk_f32_t det_r = nk_det3x3_f32_(r);
|
|
1623
|
+
if (det_r < 0) {
|
|
1624
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1625
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
1626
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
1627
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
1628
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
1629
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
1630
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
1631
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
1632
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
1633
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
if (rotation)
|
|
1637
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
1638
|
+
if (scale) *scale = 1.0f;
|
|
1639
|
+
|
|
1640
|
+
// Compute RMSD after rotation
|
|
1641
|
+
nk_f32_t sum_squared = nk_transformed_ssd_f16_neon_(a, b, n, r, 1.0f, centroid_a_x, centroid_a_y, centroid_a_z,
|
|
1642
|
+
centroid_b_x, centroid_b_y, centroid_b_z);
|
|
1643
|
+
*result = nk_f32_sqrt_neon(sum_squared * inv_n);
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
NK_PUBLIC void nk_umeyama_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
|
|
1647
|
+
nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
|
|
1648
|
+
// Fused single-pass: load f16, convert to f32, compute centroids, covariance, and variance
|
|
1649
|
+
float32x4_t const zeros_f32x4 = vdupq_n_f32(0);
|
|
1650
|
+
|
|
1651
|
+
float32x4_t sum_a_x_f32x4 = zeros_f32x4, sum_a_y_f32x4 = zeros_f32x4, sum_a_z_f32x4 = zeros_f32x4;
|
|
1652
|
+
float32x4_t sum_b_x_f32x4 = zeros_f32x4, sum_b_y_f32x4 = zeros_f32x4, sum_b_z_f32x4 = zeros_f32x4;
|
|
1653
|
+
float32x4_t cov_xx_f32x4 = zeros_f32x4, cov_xy_f32x4 = zeros_f32x4, cov_xz_f32x4 = zeros_f32x4;
|
|
1654
|
+
float32x4_t cov_yx_f32x4 = zeros_f32x4, cov_yy_f32x4 = zeros_f32x4, cov_yz_f32x4 = zeros_f32x4;
|
|
1655
|
+
float32x4_t cov_zx_f32x4 = zeros_f32x4, cov_zy_f32x4 = zeros_f32x4, cov_zz_f32x4 = zeros_f32x4;
|
|
1656
|
+
float32x4_t variance_a_f32x4 = zeros_f32x4;
|
|
1657
|
+
|
|
1658
|
+
nk_size_t i = 0;
|
|
1659
|
+
float32x4_t a_x_low_f32x4, a_x_high_f32x4, a_y_low_f32x4, a_y_high_f32x4, a_z_low_f32x4, a_z_high_f32x4;
|
|
1660
|
+
float32x4_t b_x_low_f32x4, b_x_high_f32x4, b_y_low_f32x4, b_y_high_f32x4, b_z_low_f32x4, b_z_high_f32x4;
|
|
1661
|
+
|
|
1662
|
+
for (; i + 8 <= n; i += 8) {
|
|
1663
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(a + i * 3, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1664
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1665
|
+
nk_deinterleave_f16x8_to_f32x4x2_neon_(b + i * 3, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1666
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1667
|
+
|
|
1668
|
+
sum_a_x_f32x4 = vaddq_f32(vaddq_f32(sum_a_x_f32x4, a_x_low_f32x4), a_x_high_f32x4);
|
|
1669
|
+
sum_a_y_f32x4 = vaddq_f32(vaddq_f32(sum_a_y_f32x4, a_y_low_f32x4), a_y_high_f32x4);
|
|
1670
|
+
sum_a_z_f32x4 = vaddq_f32(vaddq_f32(sum_a_z_f32x4, a_z_low_f32x4), a_z_high_f32x4);
|
|
1671
|
+
sum_b_x_f32x4 = vaddq_f32(vaddq_f32(sum_b_x_f32x4, b_x_low_f32x4), b_x_high_f32x4);
|
|
1672
|
+
sum_b_y_f32x4 = vaddq_f32(vaddq_f32(sum_b_y_f32x4, b_y_low_f32x4), b_y_high_f32x4);
|
|
1673
|
+
sum_b_z_f32x4 = vaddq_f32(vaddq_f32(sum_b_z_f32x4, b_z_low_f32x4), b_z_high_f32x4);
|
|
1674
|
+
|
|
1675
|
+
cov_xx_f32x4 = vfmaq_f32(vfmaq_f32(cov_xx_f32x4, a_x_low_f32x4, b_x_low_f32x4), a_x_high_f32x4, b_x_high_f32x4);
|
|
1676
|
+
cov_xy_f32x4 = vfmaq_f32(vfmaq_f32(cov_xy_f32x4, a_x_low_f32x4, b_y_low_f32x4), a_x_high_f32x4, b_y_high_f32x4);
|
|
1677
|
+
cov_xz_f32x4 = vfmaq_f32(vfmaq_f32(cov_xz_f32x4, a_x_low_f32x4, b_z_low_f32x4), a_x_high_f32x4, b_z_high_f32x4);
|
|
1678
|
+
cov_yx_f32x4 = vfmaq_f32(vfmaq_f32(cov_yx_f32x4, a_y_low_f32x4, b_x_low_f32x4), a_y_high_f32x4, b_x_high_f32x4);
|
|
1679
|
+
cov_yy_f32x4 = vfmaq_f32(vfmaq_f32(cov_yy_f32x4, a_y_low_f32x4, b_y_low_f32x4), a_y_high_f32x4, b_y_high_f32x4);
|
|
1680
|
+
cov_yz_f32x4 = vfmaq_f32(vfmaq_f32(cov_yz_f32x4, a_y_low_f32x4, b_z_low_f32x4), a_y_high_f32x4, b_z_high_f32x4);
|
|
1681
|
+
cov_zx_f32x4 = vfmaq_f32(vfmaq_f32(cov_zx_f32x4, a_z_low_f32x4, b_x_low_f32x4), a_z_high_f32x4, b_x_high_f32x4);
|
|
1682
|
+
cov_zy_f32x4 = vfmaq_f32(vfmaq_f32(cov_zy_f32x4, a_z_low_f32x4, b_y_low_f32x4), a_z_high_f32x4, b_y_high_f32x4);
|
|
1683
|
+
cov_zz_f32x4 = vfmaq_f32(vfmaq_f32(cov_zz_f32x4, a_z_low_f32x4, b_z_low_f32x4), a_z_high_f32x4, b_z_high_f32x4);
|
|
1684
|
+
|
|
1685
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_x_low_f32x4, a_x_low_f32x4);
|
|
1686
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_y_low_f32x4, a_y_low_f32x4);
|
|
1687
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_z_low_f32x4, a_z_low_f32x4);
|
|
1688
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_x_high_f32x4, a_x_high_f32x4);
|
|
1689
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_y_high_f32x4, a_y_high_f32x4);
|
|
1690
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_z_high_f32x4, a_z_high_f32x4);
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
if (i < n) {
|
|
1694
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(a + i * 3, n - i, &a_x_low_f32x4, &a_x_high_f32x4, &a_y_low_f32x4,
|
|
1695
|
+
&a_y_high_f32x4, &a_z_low_f32x4, &a_z_high_f32x4);
|
|
1696
|
+
nk_partial_deinterleave_f16_to_f32x4x2_neon_(b + i * 3, n - i, &b_x_low_f32x4, &b_x_high_f32x4, &b_y_low_f32x4,
|
|
1697
|
+
&b_y_high_f32x4, &b_z_low_f32x4, &b_z_high_f32x4);
|
|
1698
|
+
|
|
1699
|
+
sum_a_x_f32x4 = vaddq_f32(vaddq_f32(sum_a_x_f32x4, a_x_low_f32x4), a_x_high_f32x4);
|
|
1700
|
+
sum_a_y_f32x4 = vaddq_f32(vaddq_f32(sum_a_y_f32x4, a_y_low_f32x4), a_y_high_f32x4);
|
|
1701
|
+
sum_a_z_f32x4 = vaddq_f32(vaddq_f32(sum_a_z_f32x4, a_z_low_f32x4), a_z_high_f32x4);
|
|
1702
|
+
sum_b_x_f32x4 = vaddq_f32(vaddq_f32(sum_b_x_f32x4, b_x_low_f32x4), b_x_high_f32x4);
|
|
1703
|
+
sum_b_y_f32x4 = vaddq_f32(vaddq_f32(sum_b_y_f32x4, b_y_low_f32x4), b_y_high_f32x4);
|
|
1704
|
+
sum_b_z_f32x4 = vaddq_f32(vaddq_f32(sum_b_z_f32x4, b_z_low_f32x4), b_z_high_f32x4);
|
|
1705
|
+
|
|
1706
|
+
cov_xx_f32x4 = vfmaq_f32(vfmaq_f32(cov_xx_f32x4, a_x_low_f32x4, b_x_low_f32x4), a_x_high_f32x4, b_x_high_f32x4);
|
|
1707
|
+
cov_xy_f32x4 = vfmaq_f32(vfmaq_f32(cov_xy_f32x4, a_x_low_f32x4, b_y_low_f32x4), a_x_high_f32x4, b_y_high_f32x4);
|
|
1708
|
+
cov_xz_f32x4 = vfmaq_f32(vfmaq_f32(cov_xz_f32x4, a_x_low_f32x4, b_z_low_f32x4), a_x_high_f32x4, b_z_high_f32x4);
|
|
1709
|
+
cov_yx_f32x4 = vfmaq_f32(vfmaq_f32(cov_yx_f32x4, a_y_low_f32x4, b_x_low_f32x4), a_y_high_f32x4, b_x_high_f32x4);
|
|
1710
|
+
cov_yy_f32x4 = vfmaq_f32(vfmaq_f32(cov_yy_f32x4, a_y_low_f32x4, b_y_low_f32x4), a_y_high_f32x4, b_y_high_f32x4);
|
|
1711
|
+
cov_yz_f32x4 = vfmaq_f32(vfmaq_f32(cov_yz_f32x4, a_y_low_f32x4, b_z_low_f32x4), a_y_high_f32x4, b_z_high_f32x4);
|
|
1712
|
+
cov_zx_f32x4 = vfmaq_f32(vfmaq_f32(cov_zx_f32x4, a_z_low_f32x4, b_x_low_f32x4), a_z_high_f32x4, b_x_high_f32x4);
|
|
1713
|
+
cov_zy_f32x4 = vfmaq_f32(vfmaq_f32(cov_zy_f32x4, a_z_low_f32x4, b_y_low_f32x4), a_z_high_f32x4, b_y_high_f32x4);
|
|
1714
|
+
cov_zz_f32x4 = vfmaq_f32(vfmaq_f32(cov_zz_f32x4, a_z_low_f32x4, b_z_low_f32x4), a_z_high_f32x4, b_z_high_f32x4);
|
|
1715
|
+
|
|
1716
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_x_low_f32x4, a_x_low_f32x4);
|
|
1717
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_y_low_f32x4, a_y_low_f32x4);
|
|
1718
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_z_low_f32x4, a_z_low_f32x4);
|
|
1719
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_x_high_f32x4, a_x_high_f32x4);
|
|
1720
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_y_high_f32x4, a_y_high_f32x4);
|
|
1721
|
+
variance_a_f32x4 = vfmaq_f32(variance_a_f32x4, a_z_high_f32x4, a_z_high_f32x4);
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
// Reduce vector accumulators
|
|
1725
|
+
nk_f32_t sum_a_x = vaddvq_f32(sum_a_x_f32x4);
|
|
1726
|
+
nk_f32_t sum_a_y = vaddvq_f32(sum_a_y_f32x4);
|
|
1727
|
+
nk_f32_t sum_a_z = vaddvq_f32(sum_a_z_f32x4);
|
|
1728
|
+
nk_f32_t sum_b_x = vaddvq_f32(sum_b_x_f32x4);
|
|
1729
|
+
nk_f32_t sum_b_y = vaddvq_f32(sum_b_y_f32x4);
|
|
1730
|
+
nk_f32_t sum_b_z = vaddvq_f32(sum_b_z_f32x4);
|
|
1731
|
+
nk_f32_t covariance_x_x = vaddvq_f32(cov_xx_f32x4);
|
|
1732
|
+
nk_f32_t covariance_x_y = vaddvq_f32(cov_xy_f32x4);
|
|
1733
|
+
nk_f32_t covariance_x_z = vaddvq_f32(cov_xz_f32x4);
|
|
1734
|
+
nk_f32_t covariance_y_x = vaddvq_f32(cov_yx_f32x4);
|
|
1735
|
+
nk_f32_t covariance_y_y = vaddvq_f32(cov_yy_f32x4);
|
|
1736
|
+
nk_f32_t covariance_y_z = vaddvq_f32(cov_yz_f32x4);
|
|
1737
|
+
nk_f32_t covariance_z_x = vaddvq_f32(cov_zx_f32x4);
|
|
1738
|
+
nk_f32_t covariance_z_y = vaddvq_f32(cov_zy_f32x4);
|
|
1739
|
+
nk_f32_t covariance_z_z = vaddvq_f32(cov_zz_f32x4);
|
|
1740
|
+
nk_f32_t variance_a_sum = vaddvq_f32(variance_a_f32x4);
|
|
1741
|
+
|
|
1742
|
+
// Compute centroids
|
|
1743
|
+
nk_f32_t inv_n = 1.0f / (nk_f32_t)n;
|
|
1744
|
+
nk_f32_t centroid_a_x = sum_a_x * inv_n, centroid_a_y = sum_a_y * inv_n, centroid_a_z = sum_a_z * inv_n;
|
|
1745
|
+
nk_f32_t centroid_b_x = sum_b_x * inv_n, centroid_b_y = sum_b_y * inv_n, centroid_b_z = sum_b_z * inv_n;
|
|
1746
|
+
|
|
1747
|
+
if (a_centroid) a_centroid[0] = centroid_a_x, a_centroid[1] = centroid_a_y, a_centroid[2] = centroid_a_z;
|
|
1748
|
+
if (b_centroid) b_centroid[0] = centroid_b_x, b_centroid[1] = centroid_b_y, b_centroid[2] = centroid_b_z;
|
|
1749
|
+
|
|
1750
|
+
// Compute centered covariance and variance
|
|
1751
|
+
nk_f32_t variance_a = variance_a_sum * inv_n -
|
|
1752
|
+
(centroid_a_x * centroid_a_x + centroid_a_y * centroid_a_y + centroid_a_z * centroid_a_z);
|
|
1753
|
+
|
|
1754
|
+
nk_f32_t h[9];
|
|
1755
|
+
h[0] = covariance_x_x - (nk_f32_t)n * centroid_a_x * centroid_b_x;
|
|
1756
|
+
h[1] = covariance_x_y - (nk_f32_t)n * centroid_a_x * centroid_b_y;
|
|
1757
|
+
h[2] = covariance_x_z - (nk_f32_t)n * centroid_a_x * centroid_b_z;
|
|
1758
|
+
h[3] = covariance_y_x - (nk_f32_t)n * centroid_a_y * centroid_b_x;
|
|
1759
|
+
h[4] = covariance_y_y - (nk_f32_t)n * centroid_a_y * centroid_b_y;
|
|
1760
|
+
h[5] = covariance_y_z - (nk_f32_t)n * centroid_a_y * centroid_b_z;
|
|
1761
|
+
h[6] = covariance_z_x - (nk_f32_t)n * centroid_a_z * centroid_b_x;
|
|
1762
|
+
h[7] = covariance_z_y - (nk_f32_t)n * centroid_a_z * centroid_b_y;
|
|
1763
|
+
h[8] = covariance_z_z - (nk_f32_t)n * centroid_a_z * centroid_b_z;
|
|
1764
|
+
|
|
1765
|
+
// SVD of H = U * S * Vᵀ
|
|
1766
|
+
nk_f32_t svd_u[9], svd_s[9], svd_v[9];
|
|
1767
|
+
nk_svd3x3_f32_(h, svd_u, svd_s, svd_v);
|
|
1768
|
+
|
|
1769
|
+
// R = V * Uᵀ
|
|
1770
|
+
nk_f32_t r[9];
|
|
1771
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
1772
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
1773
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
1774
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
1775
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
1776
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
1777
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
1778
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
1779
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
1780
|
+
|
|
1781
|
+
// Handle reflection and compute scale: c = trace(D × S) / variance(a)
|
|
1782
|
+
nk_f32_t det_r = nk_det3x3_f32_(r);
|
|
1783
|
+
nk_f32_t sign_det = det_r < 0 ? -1.0f : 1.0f;
|
|
1784
|
+
nk_f32_t trace_scaled_s = svd_s[0] + svd_s[4] + sign_det * svd_s[8];
|
|
1785
|
+
nk_f32_t scale_factor = trace_scaled_s / ((nk_f32_t)n * variance_a);
|
|
1786
|
+
if (scale) *scale = scale_factor;
|
|
1787
|
+
|
|
1788
|
+
if (det_r < 0) {
|
|
1789
|
+
svd_v[2] = -svd_v[2], svd_v[5] = -svd_v[5], svd_v[8] = -svd_v[8];
|
|
1790
|
+
r[0] = svd_v[0] * svd_u[0] + svd_v[1] * svd_u[1] + svd_v[2] * svd_u[2];
|
|
1791
|
+
r[1] = svd_v[0] * svd_u[3] + svd_v[1] * svd_u[4] + svd_v[2] * svd_u[5];
|
|
1792
|
+
r[2] = svd_v[0] * svd_u[6] + svd_v[1] * svd_u[7] + svd_v[2] * svd_u[8];
|
|
1793
|
+
r[3] = svd_v[3] * svd_u[0] + svd_v[4] * svd_u[1] + svd_v[5] * svd_u[2];
|
|
1794
|
+
r[4] = svd_v[3] * svd_u[3] + svd_v[4] * svd_u[4] + svd_v[5] * svd_u[5];
|
|
1795
|
+
r[5] = svd_v[3] * svd_u[6] + svd_v[4] * svd_u[7] + svd_v[5] * svd_u[8];
|
|
1796
|
+
r[6] = svd_v[6] * svd_u[0] + svd_v[7] * svd_u[1] + svd_v[8] * svd_u[2];
|
|
1797
|
+
r[7] = svd_v[6] * svd_u[3] + svd_v[7] * svd_u[4] + svd_v[8] * svd_u[5];
|
|
1798
|
+
r[8] = svd_v[6] * svd_u[6] + svd_v[7] * svd_u[7] + svd_v[8] * svd_u[8];
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
if (rotation)
|
|
1802
|
+
for (int j = 0; j < 9; ++j) rotation[j] = r[j];
|
|
1803
|
+
|
|
1804
|
+
// Compute RMSD after similarity transform
|
|
1805
|
+
nk_f32_t sum_squared = nk_transformed_ssd_f16_neon_(a, b, n, r, scale_factor, centroid_a_x, centroid_a_y,
|
|
1806
|
+
centroid_a_z, centroid_b_x, centroid_b_y, centroid_b_z);
|
|
1807
|
+
*result = nk_f32_sqrt_neon(sum_squared * inv_n);
|
|
1808
|
+
}
|
|
1809
|
+
#if defined(__clang__)
|
|
1810
|
+
#pragma clang attribute pop
|
|
1811
|
+
#elif defined(__GNUC__)
|
|
1812
|
+
#pragma GCC pop_options
|
|
1813
|
+
#endif
|
|
1814
|
+
|
|
1815
|
+
#if defined(__cplusplus)
|
|
1816
|
+
} // extern "C"
|
|
1817
|
+
#endif
|
|
1818
|
+
|
|
1819
|
+
#endif // NK_TARGET_NEON
|
|
1820
|
+
#endif // NK_TARGET_ARM64_
|
|
1821
|
+
#endif // NK_MESH_NEON_H
|