usearch 2.26.1 → 2.26.2

package/numkong/include/numkong/dots/README.md

@@ -1,6 +1,6 @@
 # Batched Dot Products in NumKong
 
-NumKong implements batched GEMM computing C = A × Bᵀ (packed) and C = A × Aᵀ (symmetric). B is pre-packed once and reused across queries. This is the foundation for the spatials, sets, and maxsim modules.
+NumKong implements batched GEMM computing C = A × Bᵀ (packed) and C = A × Aᵀ (symmetric). B is pre-packed once and reused across queries. This is the foundation for the spatials and sets modules.
 
 Packed dot product computes the full cross-product matrix:
 
@@ -28,21 +28,21 @@ def dots_symmetric(a: np.ndarray) -> np.ndarray:
 
 ## Input & Output Types
 
-| Input Type | Output Type | Description                                    |
-| ---------- | ----------- | ---------------------------------------------- |
-| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output             |
-| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits      |
-| `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits      |
-| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits   |
-| `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits   |
-| `i8`       | `i32`       | 8-bit signed integers                          |
-| `u8`       | `u32`       | 8-bit unsigned integers                        |
-| `i4`       | `i32`       | 4-bit signed integers, packed nibble pairs     |
-| `u4`       | `u32`       | 4-bit unsigned integers, packed nibble pairs   |
-| `u1`       | `u32`       | 1-bit binary packed octets, popcount of AND    |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision                 |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output   |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
+| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits        |
+| `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits        |
+| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits     |
+| `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits     |
+| `i8`       | `i32`       | 8-bit signed integers                            |
+| `u8`       | `u32`       | 8-bit unsigned integers                          |
+| `i4`       | `i32`       | 4-bit signed integers, packed nibble pairs       |
+| `u4`       | `u32`       | 4-bit unsigned integers, packed nibble pairs     |
+| `u1`       | `u32`       | 1-bit binary packed octets, popcount of AND      |
 
 ## Optimizations
 
@@ -73,7 +73,7 @@ Tile configuration via `LDTILECFG` sets row counts and column byte-widths per ti
 Morton Z-curve ordering for tile traversal improves cache reuse when both A and B exceed L2.
 This eliminates the explicit M×N×K loop nesting and register file pressure of vector ISAs — the entire dot-product reduction happens inside the tile instruction.
 FP8 inputs on Sapphire AMX go through an on-the-fly E4M3/E5M2 → BF16 pack via the Ice Lake `VPERMI2W` LUT helpers — port-5-bound but the simplest correct route to feed `TDPBF16PS` tiles.
-Granite Rapids adds `TDPFP16PS` (same tile shape, FP16 operands); the E5M2 variant widens inputs with a single `VPUNPCK*BW` against zero into FP16 tiles at pack time and then reuses the native FP16 compute loop — keeps the intermediate at FP16 precision instead of truncating to BF16 like the Sapphire path.
+Granite Rapids adds `TDPFP16PS` (same tile shape, FP16 operands); the E5M2 variant widens inputs with `VPMOVZXBW` and a left shift by 8 into FP16 tiles at pack time and then reuses the native FP16 compute loop — keeps the intermediate at FP16 precision instead of truncating to BF16 like the Sapphire path.
 
 ### SME Outer-Product Streaming
 
@@ -105,17 +105,17 @@ The 4×4 tile accumulates 16 dot products in registers, then stores results 4-wi
 A finalizer function pointer processes 4 results simultaneously, amortizing horizontal reductions and type conversions:
 
 ```
-// 4-wide finalizer signature
-void finalizer(nk_b128_vec_t dots,          // 4 dot products
-               nk_f32_t query_norm,         // precomputed query squared-norm
-               nk_b128_vec_t target_norms,  // 4 target squared-norms
-               nk_b128_vec_t *results)      // 4 output distances
+// 4-wide finalizer signature — per-lane arrays always pass as `nk_bXXX_vec_t const *`
+void finalizer(nk_b128_vec_t const *dots_vec,          // 4 dot products
+               nk_f32_t query_norm,                    // precomputed query squared-norm (scalar)
+               nk_b128_vec_t const *target_norms_vec,  // 4 target squared-norms
+               nk_b128_vec_t *result_vec)              // 4 output distances
 
 // Angular: 4 divisions + 4 subtractions in one call
-results->f32s[i] = 1 - dots.f32s[i] / sqrt(query_norm * target_norms.f32s[i])
+result_vec->f32s[i] = 1 - dots_vec->f32s[i] / sqrt(query_norm * target_norms_vec->f32s[i])
 
 // Euclidean: 4 sqrt(a² + b² - 2ab) in one call
-results->f32s[i] = sqrt(query_norm + target_norms.f32s[i] - 2 * dots.f32s[i])
+result_vec->f32s[i] = sqrt(query_norm + target_norms_vec->f32s[i] - 2 * dots_vec->f32s[i])
 ```
 
 The 4×4 tile emits 4 rows of 4 results each — the finalizer is called 4 times per tile, once per query row.
@@ -359,8 +359,8 @@ Measured with Wasmtime v42 (Cranelift backend).
 | __f16__                            | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_dots_packed_f16_serial`        |      14.8 gso/s, 204 ulp |       14.2 gso/s, 36 ulp |      14.8 gso/s, 326 ulp |
 | `nk_dots_symmetric_f16_serial`     |       24.3 gso/s, 13 ulp |     24.9 gso/s, 24.6 ulp |      26.7 gso/s, 506 ulp |
-| `nk_dots_packed_f16_neonhalf`      |     77.0 gso/s, 16.8 ulp |     79.1 gso/s, 25.5 ulp |      84.2 gso/s, 618 ulp |
-| `nk_dots_symmetric_f16_neonhalf`   |     20.5 gso/s, 12.1 ulp |     20.4 gso/s, 25.0 ulp |      22.5 gso/s, 506 ulp |
+| `nk_dots_packed_f16_neon`          |     77.0 gso/s, 16.8 ulp |     79.1 gso/s, 25.5 ulp |      84.2 gso/s, 618 ulp |
+| `nk_dots_symmetric_f16_neon`       |     20.5 gso/s, 12.1 ulp |     20.4 gso/s, 25.0 ulp |      22.5 gso/s, 506 ulp |
 | `nk_dots_packed_f16_neonfhm`       |      104 gso/s, 16.7 ulp |      110 gso/s, 25.5 ulp |       118 gso/s, 618 ulp |
 | `nk_dots_symmetric_f16_neonfhm`    |     34.5 gso/s, 12.1 ulp |     40.4 gso/s, 25.0 ulp |      41.5 gso/s, 506 ulp |
 | `nk_dots_packed_f16_sme`           |    1,106 gso/s, 14.8 ulp |    1,213 gso/s, 28.2 ulp |    1,190 gso/s, 28.2 ulp |

package/numkong/include/numkong/dots/genoa.h

@@ -13,7 +13,9 @@
 #if NK_TARGET_GENOA
 
 #include "numkong/dot/genoa.h"
-#include "numkong/cast/icelake.h" // `nk_load_e4m3x32_to_bf16x32_icelake_`
+#include "numkong/cast/icelake.h"   // `nk_load_e4m3x32_to_bf16x32_icelake_`
+#include "numkong/dot/skylake.h"    // `nk_dot_through_f32_finalize_skylake_`
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_f32x16_skylake_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dots/icelake.h

@@ -25,6 +25,7 @@
 #if NK_TARGET_ICELAKE
 
 #include "numkong/dot/icelake.h"
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_f32x16_skylake_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dots/powervsx.h

@@ -19,7 +19,6 @@
  *  GEMM kernels use tiled dot products with 4-way parallel accumulation to hide FMA latency.
  *  Type-specific tile sizes: f32 uses depth_simd_dimensions=4, f64 uses depth_simd_dimensions=2,
  *  bf16/f16 use depth_simd_dimensions=8, u1 uses depth_simd_dimensions=128.
- *  Load/store helpers are defined in cast/powervsx.h and included transitively via dot/powervsx.h.
  */
 #ifndef NK_DOTS_POWERVSX_H
 #define NK_DOTS_POWERVSX_H
@@ -27,6 +26,7 @@
 #if NK_TARGET_POWER64_
 #if NK_TARGET_POWERVSX
 
+#include "numkong/cast/powervsx.h" // `nk_load_b128_powervsx_`, `nk_partial_load_b32x4_powervsx_`
 #include "numkong/dot/powervsx.h"
 
 #if defined(__cplusplus)

package/numkong/include/numkong/dots/serial.h

@@ -8,8 +8,8 @@
  *
  *  This file provides two macro families for generating GEMM kernels:
  *
- *  - nk_define_dots_packed_: vectorized inner-products between rows of A and Bᵀ
- *  - nk_define_dots_symmetric_: vectorized inner-products between rows and columns of A
+ *  - nk_define_cross_packed_: vectorized inner-products between rows of A and Bᵀ
+ *  - nk_define_cross_symmetric_: vectorized inner-products between rows and columns of A
  *
  *  Both use the same B packing format (see below), enabling pack-once-use-anywhere.
  *
@@ -229,15 +229,15 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
  *  @brief Generates function to calculate packed B matrix buffer size for GEMM micro-kernels.
  *
  *  Memory layout: B_packed[column_count, depth_padded] with header storing metadata.
- *  Buffer size: sizeof(header) + column_count × depth_padded × sizeof(intermediate_type) + column_count × sizeof(norm)
+ *  Buffer size: sizeof(header) + column_count × depth_padded × sizeof(packed_value_type) + column_count × sizeof(norm)
  *  Depth padding logic: Round up to `depth_simd_dimensions` multiple, then add `depth_simd_dimensions`
  *  if stride is power-of-2.
  *
  *  @param api_name Operation name (hammings, dots)
  *  @param input_type_name Original type's name of B matrix values (i4, f16, bf16, e4m3, e5m2, f32, etc.)
  *  @param isa_suffix Platform Instruct Set Architecture suffix (serial, haswell, icelake, etc.)
- *  @param input_type Original type of B matrix values (i4x2, f16, bf16, e4m3, e5m2, f32, etc.)
- *  @param intermediate_type Internal storage type in packed buffer (often bf16 or f32 for mixed precision)
+ *  @param input_value_type Original type of B matrix values (i4x2, f16, bf16, e4m3, e5m2, f32, etc.)
+ *  @param packed_value_type Internal storage type in packed buffer (often bf16 or f32 for mixed precision)
  *  @param norm_value_type Type of per-column norm values (f32, f64, u32) appended after packed data
  *  @param depth_simd_dimensions SIMD vector width in values for this platform/type combination
  *  @param dimensions_per_value Number of logical dimensions in a single value of input_type_name.
@@ -492,21 +492,20 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
  *  @param api_name Operation family (dots, hammings, jaccards) for codegen namespace
  *  @param input_type_name Type identifier for codegen (f32, bf16, i8, u1, etc.)
  *  @param isa_suffix ISA backend identifier (serial, haswell, neon, sve, icelake, etc.)
- *  @param input_type C type of input matrix values (f32, bf16, i8, u1x8, etc.)
- *  @param intermediate_type Storage type in packed B buffer (often bf16 or f32 for mixed precision)
- *  @param output_type C type of output matrix C values (f32, u32, f64, etc.)
- *  @param vec_type SIMD vector type for depth dimension (e.g., __m256, nk_f32x8_t)
+ *  @param input_value_type C type of input matrix values (f32, bf16, i8, u1x8, etc.)
+ *  @param packed_value_type Storage type in packed B buffer (often bf16 or f32 for mixed precision)
+ *  @param result_value_type C type of output matrix C values (f32, u32, f64, etc.)
+ *  @param vec_type SIMD vector type for depth dimension (e.g., __m256, nk_b256_vec_t)
  *  @param state_type Accumulator state type (often vec_type or wider, e.g., __m256 or __m512)
  *  @param result_vec_type SIMD vector type for reduction results (e.g., __m128 for 4 f32 results)
  *  @param init_accumulator_fn Initialize accumulator: void fn(state_type*)
- *  @param load_a_vec_fn Full A vector load: vec_type fn(input_type const*, nk_size_t offset)
+ *  @param load_a_vec_fn Full A vector load: vec_type fn(input_value_type const*, nk_size_t offset)
  *  @param partial_load_a_vec_fn Partial A load for remainder
- *  @param load_b_vec_fn Full B vector load: vec_type fn(intermediate_type const*, nk_size_t offset)
+ *  @param load_b_vec_fn Full B vector load: vec_type fn(packed_value_type const*, nk_size_t offset)
  *  @param partial_load_b_vec_fn Partial B load for remainder
  *  @param inner_product_fn Inner product accumulate
  *  @param reduce_accumulators_fn Reduce 4 accumulators
  *  @param store_fn Full-width store for results
- *  @param store_fn Full-width store for results
  *  @param partial_store_fn Partial store for results
  *  @param depth_simd_dimensions SIMD vector width in logical dimensions (e.g., 8 for f32 on AVX2, 128 for u1 on serial)
  *  @param dimensions_per_value Packing ratio: dimensions per storage value (1 for f32, 2 for i4x2, 8 for u1x8)
@@ -1190,7 +1189,7 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
                         result_vec_type result_vector;                                                                 \
                         for (nk_size_t r = 0; r < register_row_count; ++r) {                                           \
                             compensated_finalize_fn(&acc[r][0], &acc[r][1], &acc[r][2], &acc[r][3], depth, a_sums[r],  \
-                                                    b_sum_vec, &result_vector);                                        \
+                                                    &b_sum_vec, &result_vector);                                       \
                             nk_##result_value_type##_t *c_row =                                                        \
                                 (nk_##result_value_type##_t *)((char *)c_matrix + (tr + r) * c_stride_in_bytes);       \
                             store_fn(&result_vector, c_row + tc);                                                      \
@@ -1270,9 +1269,9 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
                         result_vec_type rv;                                                                            \
                         nk_##result_value_type##_t *c_row = (nk_##result_value_type##_t *)((char *)c_matrix +          \
                                                                                            ri * c_stride_in_bytes);    \
-                        compensated_finalize_fn(&s0, &s1, &s2, &s3, depth, a_sum_val, b_sum_low, &rv);                 \
+                        compensated_finalize_fn(&s0, &s1, &s2, &s3, depth, a_sum_val, &b_sum_low, &rv);                \
                         store_fn(&rv, c_row + tc);                                                                     \
-                        compensated_finalize_fn(&s4, &s5, &s6, &s7, depth, a_sum_val, b_sum_high, &rv);                \
+                        compensated_finalize_fn(&s4, &s5, &s6, &s7, depth, a_sum_val, &b_sum_high, &rv);               \
                         store_fn(&rv, c_row + tc + 4);                                                                 \
                     }                                                                                                  \
                 }                                                                                                      \
@@ -1445,7 +1444,7 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
                         for (nk_size_t rr = 0; rr < tile_row_count; ++rr) {                                            \
                             result_vec_type rv;                                                                        \
                             compensated_finalize_fn(&acc[rr][0], &acc[rr][1], &acc[rr][2], &acc[rr][3], depth,         \
-                                                    a_sums[rr], b_sum_vec, &rv);                                       \
+                                                    a_sums[rr], &b_sum_vec, &rv);                                      \
                             nk_##result_value_type##_t *c_row =                                                        \
                                 (nk_##result_value_type##_t *)((char *)c_matrix + (tr + rr) * c_stride_in_bytes);      \
                             partial_store_fn(&rv, c_row + tc, tile_col_count);                                         \
@@ -1715,7 +1714,7 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
                     compensated_finalize_fn(                                                                           \
                         &accumulators[row][is_diag ? row : 0], &accumulators[row][(is_diag ? row : 0) + 1],            \
                         &accumulators[row][(is_diag ? row : 0) + 2], &accumulators[row][(is_diag ? row : 0) + 3],      \
-                        depth, row_sums[row], col_sum_vec, &rv);                                                       \
+                        depth, row_sums[row], &col_sum_vec, &rv);                                                      \
                     nk_size_t global_row = i_macro + tile_row_start + row;                                             \
                     nk_size_t global_col_start = i_macro + tile_col_start + (is_diag ? row : 0);                       \
                     nk_size_t store_count = is_diag ? (tile_cols - row) : tile_cols;                                   \
@@ -1875,7 +1874,7 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
                 for (nk_size_t row = 0; row < tile_rows; row++) {                                                      \
                     result_vec_type rv;                                                                                \
                     compensated_finalize_fn(&accumulators[row][0], &accumulators[row][1], &accumulators[row][2],       \
-                                            &accumulators[row][3], depth, row_sums[tile_row_start + row], cs_vec,      \
+                                            &accumulators[row][3], depth, row_sums[tile_row_start + row], &cs_vec,     \
                                             &rv);                                                                      \
                     nk_##result_value_type##_t *dest = result +                                                        \
                                                        (i_macro + tile_row_start + row) * result_stride_values +       \
@@ -2016,13 +2015,13 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
  *  @param api_name Operation family (dots, hammings, jaccards) for codegen namespace
  *  @param input_type_name Type identifier for codegen (f32, bf16, i8, u1, etc.)
  *  @param isa_suffix ISA backend identifier (serial, haswell, neon, sve, icelake, etc.)
- *  @param input_type C type of input matrix values (f32, bf16, i8, u1x8, etc.)
- *  @param output_type C type of output matrix values (f32, u32, f64, etc.)
- *  @param vec_type SIMD vector type for input vectors (e.g., __m256, nk_f32x8_t)
+ *  @param input_value_type C type of input matrix values (f32, bf16, i8, u1x8, etc.)
+ *  @param result_value_type C type of output matrix values (f32, u32, f64, etc.)
+ *  @param vec_type SIMD vector type for input vectors (e.g., __m256, nk_b256_vec_t)
  *  @param state_type Accumulator state type (often vec_type or wider, e.g., __m256 or __m512)
  *  @param result_vec_type SIMD vector type for reduction results (e.g., __m128 for 4 f32 results)
  *  @param init_accumulator_fn Initialize accumulator: void fn(state_type*)
- *  @param load_vec_fn Full vector load: vec_type fn(input_type const*, nk_size_t offset)
+ *  @param load_vec_fn Full vector load: vec_type fn(input_value_type const*, nk_size_t offset)
  *  @param partial_load_vec_fn Partial vector load for remainder
  *  @param inner_product_fn Inner product accumulate
  *  @param reduce_accumulators_fn Reduce 4 accumulators
@@ -2435,10 +2434,12 @@ NK_INTERNAL nk_i32_t nk_dots_reduce_sum_i4_(nk_i4x2_t const *data, nk_size_t cou
  *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
  *  callers in dispatch_*.c, which wastes ~1 MB of binary and — more importantly —
  *  breaks the nk_*_serial-as-scalar-oracle contract that tests and the numerical-
- *  stability docs in this header rely on. */
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
-#elif defined(__GNUC__)
+ *  stability docs in this header rely on.
+ *
+ *  Clang gets no blanket region here: one expansion of `nk_define_cross_packed_` /
+ *  `nk_define_cross_symmetric_` emits `always_inline` `_aligned_` fast paths next to the kernel, and
+ *  clang rejects `noinline` on those. `no-ipa-cp-clone` is the knob that stops the cloning anyway. */
+#if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC push_options
 #pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
 #endif
@@ -2698,9 +2699,7 @@ nk_define_cross_packed_(dots, u1, serial, u1x8, u1x8, u32, nk_b128_vec_t, nk_dot
 #endif
 #endif
 
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
+#if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC pop_options
 #endif
 
@@ -2786,14 +2785,14 @@ NK_PUBLIC void nk_dots_compact_i8_serial(void *c, nk_size_t row_count, nk_size_t
                 vec_type dots_vec, norms_vec, results_vec;                                                            \
                 load_fn(r_row_dots + column_index, &dots_vec);                                                        \
                 load_fn(b_norms + column_index, &norms_vec);                                                          \
-                from_dot_fn(dots_vec, query_norm, norms_vec, &results_vec);                                           \
+                from_dot_fn(&dots_vec, query_norm, &norms_vec, &results_vec);                                         \
                 store_fn(&results_vec, r_row_out + column_index);                                                     \
             }                                                                                                         \
             if (column_index < column_count) {                                                                        \
                 vec_type dots_vec = {0}, norms_vec = {0}, results_vec;                                                \
                 partial_load_fn(r_row_dots + column_index, &dots_vec, column_count - column_index);                   \
                 partial_load_fn(b_norms + column_index, &norms_vec, column_count - column_index);                     \
-                from_dot_fn(dots_vec, query_norm, norms_vec, &results_vec);                                           \
+                from_dot_fn(&dots_vec, query_norm, &norms_vec, &results_vec);                                         \
                 partial_store_fn(&results_vec, r_row_out + column_index, column_count - column_index);                \
             }                                                                                                         \
         }                                                                                                             \
@@ -2849,7 +2848,7 @@ NK_PUBLIC void nk_dots_compact_i8_serial(void *c, nk_size_t row_count, nk_size_t
                     load_fn(&column_norms[j - column_chunk_start], &target_norms_vec);                                 \
                     vec_type dots_vec, results_vec;                                                                    \
                     load_fn(r_dots + j, &dots_vec);                                                                    \
-                    from_dot_fn(dots_vec, sumsq_i, target_norms_vec, &results_vec);                                    \
+                    from_dot_fn(&dots_vec, sumsq_i, &target_norms_vec, &results_vec);                                  \
                     store_fn(&results_vec, r_out + j);                                                                 \
                 }                                                                                                      \
                 /* Remainder */                                                                                        \
@@ -2857,7 +2856,7 @@ NK_PUBLIC void nk_dots_compact_i8_serial(void *c, nk_size_t row_count, nk_size_t
                     vec_type dots_vec = {0}, norms_vec = {0}, results_vec;                                             \
                     partial_load_fn(r_dots + j, &dots_vec, column_chunk_end - j);                                      \
                     partial_load_fn(&column_norms[j - column_chunk_start], &norms_vec, column_chunk_end - j);          \
-                    from_dot_fn(dots_vec, sumsq_i, norms_vec, &results_vec);                                           \
+                    from_dot_fn(&dots_vec, sumsq_i, &norms_vec, &results_vec);                                         \
                     partial_store_fn(&results_vec, r_out + j, column_chunk_end - j);                                   \
                 }                                                                                                      \
             }                                                                                                          \

package/numkong/include/numkong/dots/sierra.h

@@ -16,9 +16,10 @@
 #if NK_TARGET_X8664_
 #if NK_TARGET_SIERRA
 
-#include "numkong/dot/sierra.h"  // Sierra-specific dot product helpers
-#include "numkong/dot/haswell.h" // Haswell partial load functions
-#include "numkong/dots/serial.h" // GEMM macro definitions
+#include "numkong/dot/sierra.h"     // Sierra-specific dot product helpers
+#include "numkong/dot/haswell.h"    // Haswell partial load functions
+#include "numkong/dots/serial.h"    // GEMM macro definitions
+#include "numkong/reduce/haswell.h" // `nk_reduce_add_i32x8_haswell_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/each/README.md

@@ -60,6 +60,8 @@ Real and integer element-wise operations:
 | `bf16`     | `bf16`      | 16-bit brain float                        |
 | `e4m3`     | `e4m3`      | 8-bit Float8: 4 exponent, 3 mantissa bits |
 | `e5m2`     | `e5m2`      | 8-bit Float8: 5 exponent, 2 mantissa bits |
+| `e3m2`     | `e3m2`      | 6-bit Float6: 3 exponent, 2 mantissa bits |
+| `e2m3`     | `e2m3`      | 6-bit Float6: 2 exponent, 3 mantissa bits |
 | `i8`       | `i8`        | 8-bit signed integers, saturating         |
 | `u8`       | `u8`        | 8-bit unsigned integers, saturating       |
 | `i16`      | `i16`       | 16-bit signed integers                    |
@@ -86,7 +88,7 @@ Haswell processes 8 Float16 elements per cycle: `VCVTPH2PS` (widen) -> `VFMADD23
 
 ### Saturating Integer Arithmetic
 
-`nk_each_sum_i8_haswell`, `nk_each_sum_u8_neonhalf` use saturating addition — clamping to type bounds instead of wrapping on overflow.
+`nk_each_sum_i8_haswell`, `nk_each_sum_u8_neon` use saturating addition — clamping to type bounds instead of wrapping on overflow.
 Haswell uses `VPADDSB` / `VPADDUSB` for signed/unsigned 8-bit saturation in a single instruction (32 elements per cycle at YMM width).
 Serial fallback implements saturation via branch-free min/max: `result = min(max(a + b, TYPE_MIN), TYPE_MAX)` with overflow detection through sign-bit comparison.
 
@@ -387,7 +389,7 @@ Workloads that significantly degrade CPU frequencies (Intel AMX, Apple SME) run
 | `nk_each_fma_e3m2_serial`      |        0.506 gb/s, ? ulp |        0.433 gb/s, ? ulp |        0.436 gb/s, ? ulp |
 | __i8__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_each_sum_i8_serial`        |                94.6 gb/s |                55.5 gb/s |                42.7 gb/s |
-| `nk_each_sum_i8_neonhalf`      |                85.5 gb/s |                60.5 gb/s |                39.6 gb/s |
+| `nk_each_sum_i8_neon`          |                85.5 gb/s |                60.5 gb/s |                39.6 gb/s |
 | `nk_each_scale_i8_serial`      |               0.171 gb/s |               0.145 gb/s |               0.148 gb/s |
 | `nk_each_scale_i8_neonhalf`    |                22.6 gb/s |                22.4 gb/s |                20.8 gb/s |
 | `nk_each_blend_i8_serial`      |               0.302 gb/s |               0.254 gb/s |               0.272 gb/s |
@@ -395,7 +397,7 @@ Workloads that significantly degrade CPU frequencies (Intel AMX, Apple SME) run
 | `nk_each_fma_i8_serial`        |               0.436 gb/s |               0.359 gb/s |               0.376 gb/s |
 | __u8__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_each_sum_u8_serial`        |                13.2 gb/s |                12.3 gb/s |                12.3 gb/s |
-| `nk_each_sum_u8_neonhalf`      |                85.3 gb/s |                84.8 gb/s |                67.3 gb/s |
+| `nk_each_sum_u8_neon`          |                85.3 gb/s |                84.8 gb/s |                67.3 gb/s |
 | `nk_each_scale_u8_serial`      |               0.135 gb/s |               0.120 gb/s |               0.119 gb/s |
 | `nk_each_scale_u8_neonhalf`    |                23.0 gb/s |                21.9 gb/s |                21.2 gb/s |
 | `nk_each_blend_u8_serial`      |               0.256 gb/s |               0.218 gb/s |               0.220 gb/s |

package/numkong/include/numkong/each/sapphire.h

@@ -29,6 +29,8 @@
 
 #include "numkong/types.h"
 #include "numkong/cast/sapphire.h" // `nk_f32_to_f16_sapphire`
+#include "numkong/cast/icelake.h"  // `nk_cast_icelake`
+#include "numkong/each/icelake.h"  // `nk_each_sum_i8_icelake`, `nk_each_sum_u8_icelake`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/each/skylake.h

@@ -28,6 +28,7 @@
 
 #include "numkong/types.h"
 #include "numkong/cast/skylake.h" // `nk_e4m3x16_to_f32x16_skylake_`
+#include "numkong/each/haswell.h" // `nk_each_sum_f16_haswell`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/geospatial/README.md

@@ -1,6 +1,6 @@
 # Geospatial Distances in NumKong
 
-NumKong implements geodesic distance functions for points on Earth's surface: Haversine computes great-circle distance on a perfect sphere, while Vincenty solves the inverse geodesic problem on the WGS-84 oblate spheroid.
+NumKong implements geodesic distance functions for points on Earth's surface: Haversine computes great-circle distance on a perfect sphere, while Vincenty solves the inverse geodesic problem on an oblate spheroid, using the IERS-2003 ellipsoid constants by default.
 Both operate on arrays of latitude/longitude pairs in radians and produce distances in meters.
 
 The Haversine formula computes the great-circle distance between two points:
@@ -9,7 +9,7 @@ $$
 \text{haversine}(\phi_1, \lambda_1, \phi_2, \lambda_2) = 2R \arcsin\sqrt{\sin^2\frac{\phi_2 - \phi_1}{2} + \cos\phi_1 \cos\phi_2 \sin^2\frac{\lambda_2 - \lambda_1}{2}}
 $$
 
-where $R$ is Earth's mean radius and $(\phi, \lambda)$ are latitude and longitude in radians.
+where $R$ is the Earth radius from `NK_EARTH_MEDIATORIAL_RADIUS`, 6,335,439 m by default, and $(\phi, \lambda)$ are latitude and longitude in radians.
 
 Vincenty's formula solves the inverse geodesic problem on an oblate spheroid, iteratively refining the reduced latitude difference until convergence:
 
@@ -17,14 +17,14 @@ $$
 \text{vincenty}(\phi_1, \lambda_1, \phi_2, \lambda_2) = b \cdot A \cdot (\sigma - \Delta\sigma)
 $$
 
-where $a$ and $b$ are the equatorial and polar semi-axes of the WGS-84 ellipsoid, $\sigma$ is the angular separation, and $\Delta\sigma$ is the correction term computed through iterative convergence.
+where $a$ and $b$ are the equatorial and polar semi-axes of the configured Earth ellipsoid, $\sigma$ is the angular separation, and $\Delta\sigma$ is the correction term computed through iterative convergence.
 
 Reformulating as Python pseudocode:
 
 ```python
 import numpy as np
 
-def haversine(lat1, lon1, lat2, lon2, R=6371000):
+def haversine(lat1, lon1, lat2, lon2, R=6335439):
     dlat = lat2 - lat1
     dlon = lon2 - lon1
     a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2
@@ -45,8 +45,8 @@ Input coordinates are in radians, output distances are in meters.
 ### Trigonometric Polynomial Approximations
 
 `nk_haversine_f32_haswell`, `nk_haversine_f32_skylake`, `nk_haversine_f32_neon` replace `libm` sin, cos, atan2, and asin with SIMD polynomial approximations achieving approximately 3.5 ULP accuracy.
-Range reduction maps the input angle to $[-\pi/4, \pi/4]$ using Cody-Waite extended precision constants, then an odd-degree minimax polynomial evaluates sin and an even-degree polynomial evaluates cos.
-The `f32` kernels use 5-term polynomials while `f64` kernels use 11-term polynomials for the extra precision required by double-precision inputs.
+Range reduction maps the input angle to $[-\pi/2, \pi/2]$ using Cody-Waite extended precision constants, then an odd-degree minimax polynomial evaluates sin, and cos reuses the same polynomial after shifting the argument by $\pi/2$.
+The `f32` kernels use 5-term polynomials while `f64` kernels use 10-term polynomials for the extra precision required by double-precision inputs.
 This avoids the latency of scalar `libm` calls — each trigonometric evaluation would otherwise serialize through a single execution port, while the polynomial chains pipeline across multiple FMA units.
 
 ### Vincenty Iterative Convergence with Masked Lanes

package/numkong/include/numkong/geospatial/v128relaxed.h

@@ -33,6 +33,7 @@
 
 #include "numkong/types.h"
 #include "numkong/trigonometry/v128relaxed.h"
+#include "numkong/cast/serial.h" // `nk_partial_load_b32x4_serial_`, `nk_partial_load_b64x2_serial_`, `nk_partial_store_b32x4_serial_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/maxsim/README.md

@@ -38,11 +38,11 @@ def maxsim(queries: np.ndarray, documents: np.ndarray) -> float:
 
 ## Input & Output Types
 
-| Input Type | Output Type | Description                        |
-| ---------- | ----------- | ---------------------------------- |
-| `bf16`     | `f32`       | 16-bit brain float, widened output |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision   |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision     |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision                   |
 
 ## Optimizations
 

package/numkong/include/numkong/maxsim/genoa.h

@@ -21,6 +21,10 @@
 #include "numkong/types.h"
 #include "numkong/maxsim/icelake.h" // `nk_maxsim_coarse_argmax_icelake_`
 #include "numkong/dot.h"            // `nk_dot_bf16`
+#include "numkong/cast/icelake.h"   // `nk_e5m2x32_to_bf16x32_icelake_`
+#include "numkong/dot/skylake.h"    // `nk_dot_through_f32_finalize_skylake_`
+#include "numkong/maxsim/serial.h"  // `nk_maxsim_packed_header_setup_`, `nk_maxsim_packed_size_`
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_f32x16_skylake_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/maxsim/icelake.h

@@ -30,6 +30,7 @@
 #include "numkong/dot.h"             // `nk_dot_f32`, `nk_dot_f16`
 #include "numkong/cast/haswell.h"    // `nk_f16_to_f32_haswell`
 #include "numkong/spatial/haswell.h" // `nk_f32_sqrt_haswell`
+#include "numkong/reduce/skylake.h"  // `nk_reduce_add_f32x16_skylake_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/maxsim/serial.h

@@ -71,15 +71,6 @@ NK_STATIC_ASSERT(sizeof(nk_maxsim_vector_metadata_t) == 12, nk_maxsim_vector_met
  */
 typedef void (*nk_maxsim_to_f32_t)(void const *source, nk_f32_t *destination);
 
-/*  Keep the serial instantiations below actually scalar, regardless of build type.
- *  See dots/serial.h for rationale. */
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
-#endif
-
 /** @brief Identity conversion for f32 sources — just a typed memcpy. */
 NK_INTERNAL void nk_f32_to_f32_(void const *source, nk_f32_t *destination) { *destination = *(nk_f32_t const *)source; }
 
@@ -234,6 +225,15 @@ NK_INTERNAL nk_size_t nk_maxsim_packed_size_( //
     return header_size + i8_region_size + metadata_region_size + originals_region_size;
 }
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC nk_size_t nk_maxsim_packed_size_bf16_serial(nk_size_t vector_count, nk_size_t depth) {
     return nk_maxsim_packed_size_(vector_count, depth, sizeof(nk_bf16_t), 1);
 }
@@ -323,6 +323,12 @@ NK_PUBLIC void nk_maxsim_pack_f16_serial( //
     }
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 /**
  *  @brief Dtype-agnostic coarse i8 argmax kernel for the serial backend.
  *  Produces per-query best document indices using signed i8×i8 dot products.
@@ -389,6 +395,13 @@ NK_INTERNAL void nk_maxsim_coarse_argmax_serial_( //
     }
 }
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_maxsim_packed_bf16_serial( //
     void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
     nk_size_t depth, nk_f32_t *result) {

package/numkong/include/numkong/maxsim/sme.h

@@ -46,8 +46,9 @@
 #if NK_TARGET_ARM64_
 #if NK_TARGET_SME
 
-#include "numkong/dots/sme.h"   // `nk_dots_sme_packed_header_t`
-#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
+#include "numkong/dots/sme.h"    // `nk_dots_sme_packed_header_t`
+#include "numkong/reduce/sve.h"  // `nk_svaddv_f64_`
+#include "numkong/scalar/neon.h" // `nk_f64_rsqrt_neon`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/mesh/README.md

@@ -51,10 +51,13 @@ def rmsd(a: np.ndarray, b: np.ndarray) -> float:
 | Input Type | Output Type | Description                                    |
 | ---------- | ----------- | ---------------------------------------------- |
 | `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened RMSD |
 | `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
 | `bf16`     | `f32`       | 16-bit brain float, widened output             |
 
+The output type above is the RMSD result.
+The centroid, rotation, and scale outputs use the transform type instead: `f64` for `f64` inputs, `f32` for `f32`, `f16`, and `bf16` inputs.
+
 ## Optimizations
 
 ### McAdams Branching-Free 3×3 SVD
@@ -199,9 +202,9 @@ Measured with Wasmtime v43 (Cranelift backend), WASI-SDK 24, `-msimd128 -mrelaxe
 | `nk_kabsch_bf16_neonbfdot`  |        180 mp/s, 0.7 ulp |        448 mp/s, 0.9 ulp |        726 mp/s, 1.3 ulp |
 | `nk_umeyama_bf16_neonbfdot` |        176 mp/s, 0.2 ulp |        433 mp/s, 0.4 ulp |        705 mp/s, 0.8 ulp |
 | __f16__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_rmsd_f16_neonhalf`      |      2,998 mp/s, 0.4 ulp |      3,215 mp/s, 1.7 ulp |      3,216 mp/s, 4.6 ulp |
-| `nk_kabsch_f16_neonhalf`    |        178 mp/s, 0.9 ulp |        443 mp/s, 1.3 ulp |        711 mp/s, 2.4 ulp |
-| `nk_umeyama_f16_neonhalf`   |        175 mp/s, 0.4 ulp |        408 mp/s, 0.8 ulp |        620 mp/s, 1.5 ulp |
+| `nk_rmsd_f16_neon`          |      2,998 mp/s, 0.4 ulp |      3,215 mp/s, 1.7 ulp |      3,216 mp/s, 4.6 ulp |
+| `nk_kabsch_f16_neon`        |        178 mp/s, 0.9 ulp |        443 mp/s, 1.3 ulp |        711 mp/s, 2.4 ulp |
+| `nk_umeyama_f16_neon`       |        175 mp/s, 0.4 ulp |        408 mp/s, 0.8 ulp |        620 mp/s, 1.5 ulp |
 
 #### WASM
 

package/numkong/include/numkong/mesh/genoa.h

@@ -410,7 +410,7 @@ NK_PUBLIC void nk_umeyama_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_
         optimal_rotation[3] = 0.0f, optimal_rotation[4] = 1.0f, optimal_rotation[5] = 0.0f;
         optimal_rotation[6] = 0.0f, optimal_rotation[7] = 0.0f, optimal_rotation[8] = 1.0f;
         trace_rotation_covariance = cross_covariance[0] + cross_covariance[4] + cross_covariance[8];
-        c = centered_norm_squared_a > 0.0f ? trace_rotation_covariance / centered_norm_squared_a : 0.0f;
+        c = trace_rotation_covariance / centered_norm_squared_a;
     }
     else {
         nk_f32_t svd_left[9], svd_diagonal[9], svd_right[9];
@@ -422,7 +422,7 @@ NK_PUBLIC void nk_umeyama_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_
         nk_f32_t d3 = det < 0.0f ? -1.0f : 1.0f;
         nk_f32_t trace_ds = nk_sum_three_products_f32_(svd_diagonal[0], 1.0f, svd_diagonal[4], 1.0f, svd_diagonal[8],
                                                        d3);
-        c = centered_norm_squared_a > 0.0f ? trace_ds / centered_norm_squared_a : 0.0f;
+        c = trace_ds / centered_norm_squared_a;
 
         if (det < 0.0f) {
             svd_right[2] = -svd_right[2], svd_right[5] = -svd_right[5], svd_right[8] = -svd_right[8];