usearch 2.26.1 → 2.26.2

package/numkong/include/numkong/mesh/haswell.h

@@ -763,7 +763,7 @@ NK_PUBLIC void nk_umeyama_f32_haswell(nk_f32_t const *a, nk_f32_t const *b, nk_s
         optimal_rotation[3] = 0.0, optimal_rotation[4] = 1.0, optimal_rotation[5] = 0.0;
         optimal_rotation[6] = 0.0, optimal_rotation[7] = 0.0, optimal_rotation[8] = 1.0;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        applied_scale = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        applied_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -792,7 +792,7 @@ NK_PUBLIC void nk_umeyama_f32_haswell(nk_f32_t const *a, nk_f32_t const *b, nk_s
             optimal_rotation[8] = svd_right[6] * svd_left[6] + svd_right[7] * svd_left[7] + svd_right[8] * svd_left[8];
         }
         nk_f64_t trace_ds = svd_diagonal[0] + svd_diagonal[4] + sign_correction * svd_diagonal[8];
-        applied_scale = centered_norm_squared_a > 0.0 ? trace_ds / centered_norm_squared_a : 0.0;
+        applied_scale = trace_ds / centered_norm_squared_a;
         trace_rotation_covariance =
             optimal_rotation[0] * cross_covariance[0] + optimal_rotation[1] * cross_covariance[3] +
             optimal_rotation[2] * cross_covariance[6] + optimal_rotation[3] * cross_covariance[1] +
@@ -962,7 +962,7 @@ NK_PUBLIC void nk_umeyama_f64_haswell(nk_f64_t const *a, nk_f64_t const *b, nk_s
         optimal_rotation[3] = 0.0, optimal_rotation[4] = 1.0, optimal_rotation[5] = 0.0;
         optimal_rotation[6] = 0.0, optimal_rotation[7] = 0.0, optimal_rotation[8] = 1.0;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        c = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        c = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -972,7 +972,7 @@ NK_PUBLIC void nk_umeyama_f64_haswell(nk_f64_t const *a, nk_f64_t const *b, nk_s
         nk_f64_t det = nk_det3x3_f64_(optimal_rotation);
         nk_f64_t d3 = det < 0 ? -1.0 : 1.0;
         nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_diagonal[0], 1.0, svd_diagonal[4], 1.0, svd_diagonal[8], d3);
-        c = centered_norm_squared_a > 0.0 ? trace_ds / centered_norm_squared_a : 0.0;
+        c = trace_ds / centered_norm_squared_a;
 
         if (det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];
@@ -1633,7 +1633,7 @@ NK_PUBLIC void nk_umeyama_f16_haswell(nk_f16_t const *a, nk_f16_t const *b, nk_s
         optimal_rotation[3] = 0.0f, optimal_rotation[4] = 1.0f, optimal_rotation[5] = 0.0f;
         optimal_rotation[6] = 0.0f, optimal_rotation[7] = 0.0f, optimal_rotation[8] = 1.0f;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        applied_scale = centered_norm_squared_a > 0.0f ? trace_rotation_covariance / centered_norm_squared_a : 0.0f;
+        applied_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f32_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -1664,7 +1664,7 @@ NK_PUBLIC void nk_umeyama_f16_haswell(nk_f16_t const *a, nk_f16_t const *b, nk_s
             optimal_rotation[8] = svd_right[6] * svd_left[6] + svd_right[7] * svd_left[7] + svd_right[8] * svd_left[8];
         }
         nk_f32_t trace_ds = svd_diagonal[0] + svd_diagonal[4] + sign_correction * svd_diagonal[8];
-        applied_scale = centered_norm_squared_a > 0.0f ? trace_ds / centered_norm_squared_a : 0.0f;
+        applied_scale = trace_ds / centered_norm_squared_a;
         trace_rotation_covariance =
             optimal_rotation[0] * cross_covariance[0] + optimal_rotation[1] * cross_covariance[3] +
             optimal_rotation[2] * cross_covariance[6] + optimal_rotation[3] * cross_covariance[1] +
@@ -1819,7 +1819,7 @@ NK_PUBLIC void nk_umeyama_bf16_haswell(nk_bf16_t const *a, nk_bf16_t const *b, n
         optimal_rotation[3] = 0.0f, optimal_rotation[4] = 1.0f, optimal_rotation[5] = 0.0f;
         optimal_rotation[6] = 0.0f, optimal_rotation[7] = 0.0f, optimal_rotation[8] = 1.0f;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        applied_scale = centered_norm_squared_a > 0.0f ? trace_rotation_covariance / centered_norm_squared_a : 0.0f;
+        applied_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f32_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -1850,7 +1850,7 @@ NK_PUBLIC void nk_umeyama_bf16_haswell(nk_bf16_t const *a, nk_bf16_t const *b, n
             optimal_rotation[8] = svd_right[6] * svd_left[6] + svd_right[7] * svd_left[7] + svd_right[8] * svd_left[8];
         }
         nk_f32_t trace_ds = svd_diagonal[0] + svd_diagonal[4] + sign_correction * svd_diagonal[8];
-        applied_scale = centered_norm_squared_a > 0.0f ? trace_ds / centered_norm_squared_a : 0.0f;
+        applied_scale = trace_ds / centered_norm_squared_a;
         trace_rotation_covariance =
             optimal_rotation[0] * cross_covariance[0] + optimal_rotation[1] * cross_covariance[3] +
             optimal_rotation[2] * cross_covariance[6] + optimal_rotation[3] * cross_covariance[1] +

package/numkong/include/numkong/mesh/neon.h

@@ -853,7 +853,7 @@ NK_PUBLIC void nk_umeyama_f32_neon(nk_f32_t const *a, nk_f32_t const *b, nk_size
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        applied_scale = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        applied_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -884,7 +884,7 @@ NK_PUBLIC void nk_umeyama_f32_neon(nk_f32_t const *a, nk_f32_t const *b, nk_size
         }
 
         nk_f64_t trace_ds = svd_diagonal[0] + svd_diagonal[4] + sign_correction * svd_diagonal[8];
-        applied_scale = centered_norm_squared_a > 0.0 ? trace_ds / centered_norm_squared_a : 0.0;
+        applied_scale = trace_ds / centered_norm_squared_a;
 
         trace_rotation_covariance =
             optimal_rotation[0] * cross_covariance[0] + optimal_rotation[1] * cross_covariance[3] +
@@ -1139,7 +1139,7 @@ NK_PUBLIC void nk_umeyama_f64_neon(nk_f64_t const *a, nk_f64_t const *b, nk_size
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        computed_scale = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        computed_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -1149,7 +1149,7 @@ NK_PUBLIC void nk_umeyama_f64_neon(nk_f64_t const *a, nk_f64_t const *b, nk_size
         // Handle reflection and compute scale
         nk_f64_t det = nk_det3x3_f64_(optimal_rotation);
         nk_f64_t trace_d_s = svd_diagonal[0] + svd_diagonal[4] + (det < 0 ? -svd_diagonal[8] : svd_diagonal[8]);
-        computed_scale = centered_norm_squared_a > 0.0 ? trace_d_s / centered_norm_squared_a : 0.0;
+        computed_scale = trace_d_s / centered_norm_squared_a;
 
         if (det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];
@@ -1679,7 +1679,7 @@ NK_PUBLIC void nk_umeyama_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_size
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        scale_factor = centered_norm_squared_a > 0.0f ? trace_rotation_covariance / centered_norm_squared_a : 0.0f;
+        scale_factor = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         // SVD of H = U * S * Vᵀ
@@ -1701,7 +1701,7 @@ NK_PUBLIC void nk_umeyama_f16_neon(nk_f16_t const *a, nk_f16_t const *b, nk_size
         nk_f32_t rotation_determinant = nk_det3x3_f32_(optimal_rotation);
         nk_f32_t sign_det = rotation_determinant < 0 ? -1.0f : 1.0f;
         nk_f32_t trace_scaled_s = svd_diagonal[0] + svd_diagonal[4] + sign_det * svd_diagonal[8];
-        scale_factor = centered_norm_squared_a > 0.0f ? trace_scaled_s / centered_norm_squared_a : 0.0f;
+        scale_factor = trace_scaled_s / centered_norm_squared_a;
 
         if (rotation_determinant < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];

package/numkong/include/numkong/mesh/neonbfdot.h

@@ -40,6 +40,7 @@
 #include "numkong/types.h"
 #include "numkong/cast/neon.h"    // `nk_u16x8_splat_`
 #include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
+#include "numkong/mesh/serial.h"  // `nk_det3x3_f32_`, `nk_svd3x3_f32_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -559,7 +560,7 @@ NK_PUBLIC void nk_umeyama_bf16_neonbfdot(nk_bf16_t const *a, nk_bf16_t const *b,
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        c = centered_norm_squared_a > 0.0f ? trace_rotation_covariance / centered_norm_squared_a : 0.0f;
+        c = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f32_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -581,7 +582,7 @@ NK_PUBLIC void nk_umeyama_bf16_neonbfdot(nk_bf16_t const *a, nk_bf16_t const *b,
         nk_f32_t rotation_det = nk_det3x3_f32_(optimal_rotation);
         nk_f32_t sign_det = rotation_det < 0 ? -1.0f : 1.0f;
         nk_f32_t trace_scaled_s = svd_diagonal[0] + svd_diagonal[4] + sign_det * svd_diagonal[8];
-        c = centered_norm_squared_a > 0.0f ? trace_scaled_s / centered_norm_squared_a : 0.0f;
+        c = trace_scaled_s / centered_norm_squared_a;
 
         if (rotation_det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];

package/numkong/include/numkong/mesh/neonfhm.h

@@ -38,6 +38,7 @@
 
 #include "numkong/types.h"
 #include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
+#include "numkong/mesh/serial.h"  // `nk_det3x3_f32_`, `nk_svd3x3_f32_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -604,7 +605,7 @@ NK_PUBLIC void nk_umeyama_f16_neonfhm(nk_f16_t const *a, nk_f16_t const *b, nk_s
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        scale_factor = centered_norm_squared_a > 0.0f ? trace_rotation_covariance / centered_norm_squared_a : 0.0f;
+        scale_factor = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         // SVD of H = U · S · Vᵀ
@@ -626,7 +627,7 @@ NK_PUBLIC void nk_umeyama_f16_neonfhm(nk_f16_t const *a, nk_f16_t const *b, nk_s
         nk_f32_t rotation_determinant = nk_det3x3_f32_(optimal_rotation);
         nk_f32_t sign_det = rotation_determinant < 0 ? -1.0f : 1.0f;
         nk_f32_t trace_scaled_s = svd_diagonal[0] + svd_diagonal[4] + sign_det * svd_diagonal[8];
-        scale_factor = centered_norm_squared_a > 0.0f ? trace_scaled_s / centered_norm_squared_a : 0.0f;
+        scale_factor = trace_scaled_s / centered_norm_squared_a;
 
         if (rotation_determinant < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];

package/numkong/include/numkong/mesh/rvv.h

@@ -884,7 +884,7 @@ NK_PUBLIC void nk_umeyama_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_size_
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        scale_factor = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        scale_factor = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -894,7 +894,7 @@ NK_PUBLIC void nk_umeyama_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_size_
         nk_f64_t sign_det = det < 0 ? -1.0 : 1.0;
         nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_diagonal[0], 1.0, svd_diagonal[4], 1.0, svd_diagonal[8],
                                                        sign_det);
-        scale_factor = centered_norm_squared_a > 0.0 ? trace_ds / centered_norm_squared_a : 0.0;
+        scale_factor = trace_ds / centered_norm_squared_a;
         if (det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];
             nk_rotation_from_svd_f64_serial_(svd_left, svd_right, optimal_rotation);
@@ -944,7 +944,7 @@ NK_PUBLIC void nk_umeyama_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_size_
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        scale_factor = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        scale_factor = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -954,7 +954,7 @@ NK_PUBLIC void nk_umeyama_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_size_
         nk_f64_t sign_det = det < 0 ? -1.0 : 1.0;
         nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_diagonal[0], 1.0, svd_diagonal[4], 1.0, svd_diagonal[8],
                                                        sign_det);
-        scale_factor = centered_norm_squared_a > 0.0 ? trace_ds / centered_norm_squared_a : 0.0;
+        scale_factor = trace_ds / centered_norm_squared_a;
         if (det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];
             nk_rotation_from_svd_f64_serial_(svd_left, svd_right, optimal_rotation);

package/numkong/include/numkong/mesh/serial.h

@@ -272,17 +272,6 @@ extern "C" {
                m[2] * (m[3] * m[7] - m[4] * m[6]);                                       \
     }
 
-/*  Keep the serial instantiations below actually scalar, regardless of build type.
- *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
- *  callers in dispatch_*.c, which wastes binary and breaks the nk_*_serial-as-scalar-oracle
- *  contract that tests and numerical-stability docs rely on. See dots/serial.h. */
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
-#endif
-
 /* Size bias for release. Gated on NDEBUG so Debug builds keep -O0 for stepping. */
 #if defined(NDEBUG)
 #if defined(_MSC_VER)
@@ -701,6 +690,17 @@ nk_define_det3x3_(f64)
         *result = (nk_##result_type##_t)compute_sqrt((sum_squared + sum_squared_compensation) * inv_n);               \
     }
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
+ *  callers in dispatch_*.c, which wastes binary and breaks the nk_*_serial-as-scalar-oracle
+ *  contract that tests and numerical-stability docs rely on. See dots/serial.h. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 nk_define_rmsd_(f64, f64, f64, f64, nk_assign_from_to_, nk_f64_sqrt_serial)         // nk_rmsd_f64_serial
 nk_define_kabsch_(f64, f64, f64, f64, f64, nk_assign_from_to_, nk_f64_sqrt_serial)  // nk_kabsch_f64_serial
 nk_define_umeyama_(f64, f64, f64, f64, f64, nk_assign_from_to_, nk_f64_sqrt_serial) // nk_umeyama_f64_serial
@@ -740,21 +740,21 @@ nk_define_umeyama_(bf16, f32, f32, f32, f32, nk_bf16_to_f32_serial, nk_f32_sqrt_
 #undef nk_define_kabsch_
 #undef nk_define_umeyama_
 
-#if defined(NDEBUG)
-#if defined(_MSC_VER)
-#pragma optimize("", on)
-#elif defined(__clang__)
+#if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)
 #pragma GCC pop_options
 #endif
-#endif
 
-#if defined(__clang__)
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("", on)
+#elif defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)
 #pragma GCC pop_options
 #endif
+#endif
 
 #if defined(__cplusplus)
 } // extern "C"

package/numkong/include/numkong/mesh/skylake.h

@@ -1157,7 +1157,7 @@ NK_PUBLIC void nk_umeyama_f64_skylake(nk_f64_t const *a, nk_f64_t const *b, nk_s
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        c = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        c = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -1168,7 +1168,7 @@ NK_PUBLIC void nk_umeyama_f64_skylake(nk_f64_t const *a, nk_f64_t const *b, nk_s
         nk_f64_t det = nk_det3x3_f64_(optimal_rotation);
         nk_f64_t d3 = det < 0 ? -1.0 : 1.0;
         nk_f64_t trace_ds = nk_sum_three_products_f64_(svd_diagonal[0], 1.0, svd_diagonal[4], 1.0, svd_diagonal[8], d3);
-        c = centered_norm_squared_a > 0.0 ? trace_ds / centered_norm_squared_a : 0.0;
+        c = trace_ds / centered_norm_squared_a;
 
         if (det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];
@@ -1620,7 +1620,7 @@ NK_PUBLIC void nk_umeyama_f16_skylake(nk_f16_t const *a, nk_f16_t const *b, nk_s
     nk_f32_t det = nk_det3x3_f32_(optimal_rotation);
     nk_f32_t d3 = det < 0.0f ? -1.0f : 1.0f;
     nk_f32_t trace_ds = nk_sum_three_products_f32_(svd_diagonal[0], 1.0f, svd_diagonal[4], 1.0f, svd_diagonal[8], d3);
-    nk_f32_t c = centered_norm_squared_a > 0.0f ? trace_ds / centered_norm_squared_a : 0.0f;
+    nk_f32_t c = trace_ds / centered_norm_squared_a;
     if (scale) *scale = c;
 
     if (det < 0.0f) {
@@ -1749,7 +1749,7 @@ NK_PUBLIC void nk_umeyama_bf16_skylake(nk_bf16_t const *a, nk_bf16_t const *b, n
     nk_f32_t det = nk_det3x3_f32_(optimal_rotation);
     nk_f32_t d3 = det < 0.0f ? -1.0f : 1.0f;
     nk_f32_t trace_ds = nk_sum_three_products_f32_(svd_diagonal[0], 1.0f, svd_diagonal[4], 1.0f, svd_diagonal[8], d3);
-    nk_f32_t c = centered_norm_squared_a > 0.0f ? trace_ds / centered_norm_squared_a : 0.0f;
+    nk_f32_t c = trace_ds / centered_norm_squared_a;
     if (scale) *scale = c;
 
     if (det < 0.0f) {

package/numkong/include/numkong/mesh/v128relaxed.h

@@ -842,7 +842,7 @@ NK_PUBLIC void nk_umeyama_f32_v128relaxed(nk_f32_t const *a, nk_f32_t const *b,
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        computed_scale = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        computed_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -874,7 +874,7 @@ NK_PUBLIC void nk_umeyama_f32_v128relaxed(nk_f32_t const *a, nk_f32_t const *b,
 
         nk_f64_t trace_signed_singular_values = svd_diagonal[0] + svd_diagonal[4] +
                                                 (det < 0 ? -svd_diagonal[8] : svd_diagonal[8]);
-        computed_scale = centered_norm_squared_a > 0.0 ? trace_signed_singular_values / centered_norm_squared_a : 0.0;
+        computed_scale = trace_signed_singular_values / centered_norm_squared_a;
 
         trace_rotation_covariance =
             optimal_rotation[0] * cross_covariance[0] + optimal_rotation[1] * cross_covariance[3] +
@@ -1059,7 +1059,7 @@ NK_PUBLIC void nk_umeyama_f64_v128relaxed(nk_f64_t const *a, nk_f64_t const *b,
         optimal_rotation[4] = 1, optimal_rotation[5] = 0, optimal_rotation[6] = 0, optimal_rotation[7] = 0,
         optimal_rotation[8] = 1;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        computed_scale = centered_norm_squared_a > 0.0 ? trace_rotation_covariance / centered_norm_squared_a : 0.0;
+        computed_scale = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f64_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -1069,7 +1069,7 @@ NK_PUBLIC void nk_umeyama_f64_v128relaxed(nk_f64_t const *a, nk_f64_t const *b,
         // Handle reflection and compute scale
         nk_f64_t det = nk_det3x3_f64_(optimal_rotation);
         nk_f64_t trace_d_s = svd_diagonal[0] + svd_diagonal[4] + (det < 0 ? -svd_diagonal[8] : svd_diagonal[8]);
-        computed_scale = centered_norm_squared_a > 0.0 ? trace_d_s / centered_norm_squared_a : 0.0;
+        computed_scale = trace_d_s / centered_norm_squared_a;
 
         if (det < 0) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];

package/numkong/include/numkong/mesh.hpp

@@ -319,8 +319,8 @@ void svd3x3_(scalar_type_ const *a, scalar_type_ *svd_u, scalar_type_ *svd_s, sc
 
 /**
  *  @brief Root Mean Square Deviation between two 3D point clouds (no alignment)
- *  @param[in] a,b Point clouds [d x 3] interleaved (x0,y0,z0, x1,y1,z1, ...)
- *  @param[in] d Number of 3D points
+ *  @param[in] a,b Point clouds [n x 3] interleaved (x0,y0,z0, x1,y1,z1, ...)
+ *  @param[in] n Number of 3D points
  *  @param[out] a_centroid,b_centroid Centroids (3 values each), can be nullptr
  *  @param[out] rotation 3x3 rotation matrix (9 values), always identity, can be nullptr
  *  @param[out] scale Scale factor, always 1.0, can be nullptr

package/numkong/include/numkong/numkong.h

@@ -1,6 +1,6 @@
 /**
  *  @brief SIMD-accelerated Similarity Measures and Distance Functions.
- *  @file include/numkong.h
+ *  @file include/numkong/numkong.h
  *  @author Ash Vardanian
  *  @date March 14, 2023
  *

package/numkong/include/numkong/numkong.hpp

@@ -1,6 +1,6 @@
 /**
  *  @brief NumKong SDK for C++23 and newer.
- *  @file include/numkong.hpp
+ *  @file include/numkong/numkong.hpp
  *  @author Ash Vardanian
  *  @date January 7, 2026
  *

package/numkong/include/numkong/probability/README.md

@@ -6,7 +6,7 @@ These are used in variational inference, topic modeling, and distribution compar
 Kullback-Leibler divergence from $P$ to $Q$:
 
 $$
-\text{KLD}(P \| Q) = \sum_{i=0}^{n-1} P(i) \log_2 \frac{P(i)}{Q(i)}
+\text{KLD}(P \| Q) = \sum_{i=0}^{n-1} P(i) \ln \frac{P(i)}{Q(i)}
 $$
 
 Jensen-Shannon distance is the square root of the symmetrized KLD through a mixture:
@@ -28,9 +28,8 @@ Reformulating as Python pseudocode:
 ```python
 import numpy as np
 
-def kld(p: np.ndarray, q: np.ndarray) -> float:
-    mask = p > 0
-    return np.sum(p[mask] * np.log2(p[mask] / q[mask]))
+def kld(p: np.ndarray, q: np.ndarray, eps: float = 1e-7) -> float:
+    return np.sum(p * np.log((p + eps) / (q + eps)))
 
 def jsd(p: np.ndarray, q: np.ndarray) -> float:
     m = (p + q) / 2
@@ -45,12 +44,12 @@ __Jensen-Shannon distance__ is commonly used in microbiome community comparison
 
 ## Input & Output Types
 
-| Input Type | Output Type | Description                                    |
-| ---------- | ----------- | ---------------------------------------------- |
-| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output             |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision                 |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output   |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
 
 ## Optimizations
 
@@ -61,24 +60,27 @@ The pipeline on Skylake is:
 
 ```
 exponent = VGETEXPPS(x)
-mantissa = VGETMANTPS(x, normalize_to_[1,2)) - 1
-log2(x) ≈ exponent + polynomial(mantissa)
+mantissa = VGETMANTPS(x, normalize_to_[1,2))
+s        = (mantissa - 1) / (mantissa + 1)
+log2(x)  ≈ exponent + (2/ln2) · s · (1 + s²/3 + s⁴/5 + s⁶/7 + s⁸/9)
 ```
 
 `VGETEXP` extracts the unbiased exponent as a float, while `VGETMANT` normalizes the mantissa to $[1, 2)$.
-A degree-4 minimax polynomial over the normalized mantissa completes the approximation.
+A five-term odd series in $s$ — the $\operatorname{atanh}$ expansion, which converges quickly because $s \in [0, 1/3]$ — completes the approximation.
 These instructions handle subnormals correctly without extra integer bit manipulation.
 
 `nk_kld_f32_neon`, `nk_jsd_f32_neon`, `nk_kld_f16_haswell`, `nk_jsd_f16_haswell` use integer bit extraction instead:
 
 ```
 exponent = (reinterpret_as_int(x) >> 23) - 127
-mantissa = reinterpret_as_float((reinterpret_as_int(x) & 0x7FFFFF) | 0x3F800000) - 1
-log2(x) ≈ exponent + c₁·m + c₂·m² + c₃·m³ + c₄·m⁴ + c₅·m⁵
+mantissa = reinterpret_as_float((reinterpret_as_int(x) & 0x7FFFFF) | 0x3F800000)
+log2(x) ≈ exponent + series_or_polynomial(mantissa)
 ```
 
 This approach reinterprets the float as an integer, shifts out the mantissa bits to obtain the exponent, then masks and recombines to produce a normalized mantissa in $[1, 2)$.
+The Haswell kernels then reuse the same $s$-series as Skylake, while the NEON kernels evaluate a degree-5 polynomial in the mantissa scaled by $m - 1$.
 It works on any ISA with integer-float reinterpretation and avoids the need for specialized exponent/mantissa instructions.
+The SIMD kernels scale the accumulated $\log_2$ sum by $\ln 2$ before returning, while the serial ones call a natural-log helper directly — either way the reported divergence is in nats, not bits.
 
 ### Kahan Compensated Summation for Float64
 

package/numkong/include/numkong/probability/skylake.h

@@ -14,7 +14,7 @@
 
 #include "numkong/types.h"
 #include "numkong/spatial/haswell.h" // `nk_f32_sqrt_haswell`, `nk_f64_sqrt_haswell`
-#include "numkong/spatial/skylake.h" // `nk_f32_sqrt_skylake`, `nk_f64_sqrt_skylake`
+#include "numkong/spatial/skylake.h"
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/reduce/README.md

@@ -31,36 +31,39 @@ def minmax(a: np.ndarray) -> tuple[float, int, float, int]:
 
 ## Input & Output Types
 
+The Output Type column lists the accumulator type of `nk_reduce_moments_*`.
+`nk_reduce_minmax_*` instead reports extrema in the input type — `i4`, `u4`, and `u1` unpack to `i8`, `u8`, and `u8` — alongside two `nk_size_t` indices.
+
 Float reductions:
 
-| Input Type | Output Type | Description                           |
-| ---------- | ----------- | ------------------------------------- |
-| `f64`      | `f64`       | 64-bit double precision               |
-| `f32`      | `f32`       | 32-bit single precision               |
-| `f16`      | `f32`       | 16-bit half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output    |
+| Input Type | Output Type | Description                             |
+| ---------- | ----------- | --------------------------------------- |
+| `f64`      | `f64`       | 64-bit double precision                 |
+| `f32`      | `f64`       | 32-bit single precision, widened output |
+| `bf16`     | `f32`       | 16-bit brain float, widened output      |
+| `f16`      | `f32`       | 16-bit half precision, widened output   |
 
 Mini-float reductions:
 
 | Input Type | Output Type | Description                                  |
 | ---------- | ----------- | -------------------------------------------- |
-| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits    |
 | `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits    |
-| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits |
+| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits    |
 | `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits |
+| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits |
 
 Integer reductions:
 
 | Input Type | Output Type | Description                        |
 | ---------- | ----------- | ---------------------------------- |
-| `i8`       | `i64`       | 8-bit signed, widened to 64-bit    |
-| `u8`       | `u64`       | 8-bit unsigned, widened to 64-bit  |
-| `i16`      | `i64`       | 16-bit signed, widened to 64-bit   |
-| `u16`      | `u64`       | 16-bit unsigned, widened to 64-bit |
-| `i32`      | `i64`       | 32-bit signed, widened to 64-bit   |
-| `u32`      | `u64`       | 32-bit unsigned, widened to 64-bit |
 | `i64`      | `i64`       | 64-bit signed                      |
+| `i32`      | `i64`       | 32-bit signed, widened to 64-bit   |
+| `i16`      | `i64`       | 16-bit signed, widened to 64-bit   |
+| `i8`       | `i64`       | 8-bit signed, widened to 64-bit    |
 | `u64`      | `u64`       | 64-bit unsigned                    |
+| `u32`      | `u64`       | 32-bit unsigned, widened to 64-bit |
+| `u16`      | `u64`       | 16-bit unsigned, widened to 64-bit |
+| `u8`       | `u64`       | 8-bit unsigned, widened to 64-bit  |
 
 Sub-byte reductions:
 
@@ -82,29 +85,32 @@ RVV uses native strided loads (`__riscv_vlse32_v_f32m1`) that accept arbitrary b
 
 ### Kahan-Neumaier Compensated Summation
 
-`nk_reduce_moments_f32_serial`, `nk_reduce_moments_f32_haswell` use Neumaier's variant of Kahan summation — maintaining a running compensation term that captures rounding errors.
+`nk_reduce_moments_f32_serial`, `nk_reduce_moments_f64_serial` use Neumaier's variant of Kahan summation — maintaining a running compensation term that captures rounding errors.
 Standard pairwise summation accumulates $O(\sqrt{n})$ ULP error for n elements; Neumaier compensation bounds error to $O(1)$ ULP regardless of vector length.
 The serial path uses Neumaier's adaptive branch: `if (abs(sum) >= abs(val))` selects the larger summand first, minimizing relative error in the compensation term.
-SIMD backends (`nk_reduce_moments_f32_haswell`) carry 8 independent compensation lanes in a YMM register — computing `round_error = tentative - sum; correction = (sum - (tentative - round_error)) + (val - round_error)` without branches, folding all lanes into a single scalar correction at the end.
+SIMD backends (`nk_reduce_moments_f64_haswell`, `nk_reduce_moments_f64_skylake`) carry 4 or 8 independent compensation lanes in a YMM or ZMM register — computing `round_error = tentative - sum; correction = (sum - (tentative - round_error)) + (val - round_error)` without branches, folding all lanes into a single scalar correction at the end.
+The Float32 SIMD kernels skip compensation: they widen every input to Float64 first, and the wider accumulator already absorbs the rounding error.
 
 ### Fused Moments in a Single Pass
 
-`nk_reduce_moments_f32_haswell`, `nk_reduce_moments_f64_skylake` compute sum and sum-of-squares simultaneously — one load feeds both a `VADDPS` (sum accumulator) and a `VFMADD231PS` (square accumulator).
+`nk_reduce_moments_f32_haswell`, `nk_reduce_moments_f64_skylake` compute sum and sum-of-squares simultaneously — one load feeds both a sum accumulator and a square accumulator.
+On Haswell the Float32 inputs are widened with `VCVTPS2PD`, then folded in with `VADDPD` and `VFMADD231PD`; on Skylake the Float64 path squares with `VMULPD` so the product can go through the compensated add.
 Two accumulators share the same loaded data, halving memory bandwidth compared to separate sum + norm passes.
 The squared-norm $\|a\|^2 = \sum a_i^2$ is a self-dot-product, reused by packing routines (`nk_dots_pack_f32_haswell`) to precompute per-vector norms during layout transformation.
-For Float16/BFloat16/Float8 inputs, all backends widen to Float32 before accumulation — NEON FHM (`nk_reduce_moments_e4m3_neonfhm`) converts e4m3->f16 via lookup, then uses `vfmlalq_low_f16` to fuse the Float16 → Float32 widening with the FMA into the Float32 accumulator.
+For Float16/BFloat16/Float8 inputs, all backends widen to Float32 before accumulation — NEON FHM (`nk_reduce_moments_e4m3_neonfhm`) converts e4m3->f16 by bit manipulation, then uses `vfmlalq_low_f16` and `vfmlalq_high_f16` to fuse the Float16 → Float32 widening with the FMA into the Float32 accumulator.
 
 ### Integer Saturation in Sum-of-Squares
 
 Integer moments accumulate sums in the widest available type: Int8/UInt8/Int16 inputs produce Int64/UInt64 outputs.
 Sums use widening addition chains — NEON uses pairwise widening (`vpaddlq_s16` -> UInt32 -> UInt64 stages); Haswell biases Int8 inputs with 0x80 and uses unsigned SAD (`_mm256_sad_epu8`) for the sum, correcting by subtracting $128 \times \text{count}$ at the end.
-Sum-of-squares can overflow UInt64 when squaring large Int32 values — backends use explicit saturating multiply: checks if `abs(val) < 2^32` (square fits in UInt64), otherwise saturates to I64_MAX.
+Sum-of-squares can overflow UInt64 when squaring Int64/UInt64 inputs — only `nk_reduce_moments_i64_*` and `nk_reduce_moments_u64_*` need an explicit saturating multiply: it checks whether `abs(val) < 2^32` (so the square fits in UInt64) and otherwise saturates to I64_MAX for signed inputs, U64_MAX for unsigned ones.
+Narrower inputs cannot overflow the product itself, so they only need the saturating accumulate.
 Haswell emulates UInt64 saturating add via XOR-based unsigned comparison: flip sign bits to convert unsigned overflow detection into a signed comparison, then OR with the overflow mask to produce all-ones on saturation.
 
 ### Recursive Blocking for Counter Overflow
 
 All SIMD backends use loop iteration counters narrower than `nk_size_t` to save register pressure — UInt8 for Int8 minmax lanes, UInt16 for Float32 moments lanes.
-When `count` exceeds the counter's range x lane count (e.g., Haswell Float32: $256 \times 8 = 2048$ elements for UInt8 counters), the reduction splits recursively: process the left half, process the right half, combine results with saturating arithmetic.
+When `count` exceeds the counter's range x lane count (e.g., Haswell Float32 moments: $65536 \times 8 = 524288$ elements for UInt16 counters), the reduction splits recursively: process the left half, process the right half, combine results with saturating arithmetic.
 Block caps vary by backend and element width: Haswell Int8 minmax uses UInt8 loop counters (cap = $256 \times 32 = 8192$); Skylake Float32 moments uses UInt16 counters (cap = $65536 \times 16 = 1048576$).
 The recursive split is invisible to the caller — the public API accepts arbitrary `count` values; internal dispatch chooses between single-pass and recursive based on the cap.
 
@@ -118,7 +124,7 @@ NEON uses same-width counters (u8x16 for i8x16 minmax), limiting block size to $
 ### NaN-Aware Extrema Tracking
 
 `nk_reduce_minmax_f32_haswell`, `nk_reduce_minmax_f64_skylake` use IEEE ordered-quiet comparisons (`_CMP_LT_OQ`, `_CMP_GT_OQ`) — returning false when either operand is NaN, so NaN inputs never replace the running extremum.
-Tail elements beyond the vector-aligned portion are masked by loading into a NaN-filled register via `_mm256_mask_loadu_ps(nan_vec, mask, ptr)` — NaN tails cannot win any comparison, eliminating out-of-bounds artifacts.
+Tail elements beyond the vector-aligned portion are blended into a quiet-NaN register — `nk_reduce_minmax_f64_haswell` partial-loads the tail and `_mm256_blendv_pd`-s it over a NaN vector, so the padding lanes cannot win any comparison; Skylake instead predicates the tail comparison with the same `__mmask` used for the load, and `nk_reduce_minmax_f32_haswell` finishes its tail in scalar code.
 If all inputs are NaN, the sentinels remain (min = F32_MAX, max = F32_MIN) and indices are set to NK_SIZE_MAX, signaling no valid extremum.
 The final horizontal reduction across lanes uses pairwise `VSHUFPS` + `VMINPS` chains — 3 shuffles for a 256-bit register, $O(\log_2 w)$ for width $w$.
 

package/numkong/include/numkong/reduce/genoa.h

@@ -22,7 +22,8 @@
 #if NK_TARGET_GENOA
 
 #include "numkong/reduce/serial.h"
-#include "numkong/cast/icelake.h" // `nk_e4m3x32_to_bf16x32_icelake_` etc.
+#include "numkong/cast/icelake.h"   // `nk_e4m3x32_to_bf16x32_icelake_` etc.
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_f32x16_skylake_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/reduce/icelake.h

@@ -19,6 +19,7 @@
 #if NK_TARGET_ICELAKE
 
 #include "numkong/reduce/serial.h"
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_i64x8_skylake_`, `nk_reduce_add_u64x8_skylake_`, `nk_stride_mask_b16x32_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/reduce/neon.h

@@ -2218,7 +2218,7 @@ NK_INTERNAL void nk_reduce_moments_i64_neon_contiguous_( //
     nk_i64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
     uint64x2_t sum_low_u64x2 = vdupq_n_u64(0);
     int64x2_t sum_high_i64x2 = vdupq_n_s64(0);
-    // NEON can still load/extract i64 vectors for sumsq via scalar nk_i64_smul_
+    // NEON can still load/extract i64 vectors for sumsq via nk_i64_smul_sq_i64x2_neon_
     uint64x2_t sumsq_u64x2 = vdupq_n_u64(0);
     int sumsq_overflow = 0;
     int64x2_t sign_bit_i64x2 = vdupq_n_s64((nk_i64_t)0x8000000000000000ULL);

package/numkong/include/numkong/reduce/serial.h

@@ -21,15 +21,6 @@
 extern "C" {
 #endif
 
-/*  Keep the serial instantiations below actually scalar, regardless of build type.
- *  See dots/serial.h for rationale. */
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
-#endif
-
 NK_INTERNAL nk_f64_t nk_reduce_sum_f64_serial_(nk_f64_t const *values, nk_f64_t const *compensations, int count) {
     nk_f64_t running_sum = 0, accumulated_error = 0;
     for (int i = 0; i < count; i++) {
@@ -47,6 +38,15 @@ NK_INTERNAL nk_f64_t nk_reduce_sum_f64_serial_(nk_f64_t const *values, nk_f64_t
     return running_sum + accumulated_error;
 }
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_reduce_moments_f32_serial(                       //
     nk_f32_t const *data, nk_size_t count, nk_size_t stride_bytes, //
     nk_f64_t *sum_ptr, nk_f64_t *sumsq_ptr) {