usearch 2.26.1 → 2.26.3

package/numkong/include/numkong/cast.h

@@ -11,9 +11,9 @@
  *  - `f16` & `bf16` ↔ `f32` - often used for half-precision dot-products on older CPUs,
  *
  *  Unlike most operation classes in NumKong, these are dependent on two input types: "from" & "to".
- *  It contains scalar helpers named like `nk_f16_to_f32_serial_` as well as buffer-to-buffer
- *  `memcpy`-like vectorized operations, such as `nk_cast_f16_to_f32` with `nk_cast_f16_to_f32_serial`,
- *  `nk_cast_f16_to_f32_neon`, `nk_cast_f16_to_f32_skylake`, and other platform-specific variants.
+ *  It contains scalar helpers named like `nk_f16_to_f32_serial` as well as buffer-to-buffer
+ *  `memcpy`-like vectorized operations, such as `nk_cast` with `nk_cast_serial`,
+ *  `nk_cast_neon`, `nk_cast_skylake`, and other platform-specific variants.
  *
  *  It also includes "partial load" and "partial store" type-punned helper functions for handling
  *  IO between memory and registers, that are extensively reused in reductions, elementwise ops, and

package/numkong/include/numkong/curved/README.md

@@ -42,19 +42,19 @@ def bilinear_complex(a: np.ndarray, b: np.ndarray, C: np.ndarray) -> complex:
 
 Real bilinear and Mahalanobis:
 
-| Input Type | Output Type | Description                                    |
-| ---------- | ----------- | ---------------------------------------------- |
-| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output             |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision                 |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output   |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
 
 Complex bilinear:
 
 | Input Type | Output Type | Description                                |
 | ---------- | ----------- | ------------------------------------------ |
 | `f64c`     | `f64c`      | 64-bit complex pairs                       |
-| `f32c`     | `f32c`      | 32-bit complex pairs                       |
+| `f32c`     | `f64c`      | 32-bit complex pairs, widened output       |
 | `f16c`     | `f32c`      | 16-bit complex pairs, widened output       |
 | `bf16c`    | `f32c`      | 16-bit brain complex pairs, widened output |
 
@@ -179,7 +179,7 @@ Measured with Wasmtime v42 (Cranelift backend).
 | `nk_bilinear_bf16c_neonbfdot`   |     5.05 gso/s, 17.0 ulp |     4.20 gso/s, 17.0 ulp |     4.04 gso/s, 17.0 ulp |
 | __f16c__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_bilinear_f16c_serial`       |     2.81 gso/s, 51.8 ulp |     2.54 gso/s, 51.8 ulp |     2.48 gso/s, 51.8 ulp |
-| `nk_bilinear_f16c_neonhalf`     |     5.00 gso/s, 17.3 ulp |     4.16 gso/s, 17.3 ulp |     4.00 gso/s, 16.4 ulp |
+| `nk_bilinear_f16c_neon`         |     5.00 gso/s, 17.3 ulp |     4.16 gso/s, 17.3 ulp |     4.00 gso/s, 16.4 ulp |
 | __f64__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_bilinear_f64_serial`        |     0.717 gso/s, 0.4 ulp |     0.711 gso/s, 0.4 ulp |     0.721 gso/s, 0.4 ulp |
 | `nk_mahalanobis_f64_serial`     |     0.664 gso/s, 0.5 ulp |     0.667 gso/s, 0.5 ulp |     0.672 gso/s, 0.5 ulp |
@@ -196,8 +196,8 @@ Measured with Wasmtime v42 (Cranelift backend).
 | __f16__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_bilinear_f16_serial`        |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
 | `nk_mahalanobis_f16_serial`     |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
-| `nk_bilinear_f16_neonhalf`      |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
-| `nk_mahalanobis_f16_neonhalf`   |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_bilinear_f16_neon`          |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_mahalanobis_f16_neon`       |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
 
 #### WASM
 

package/numkong/include/numkong/curved/neon.h

@@ -29,6 +29,7 @@
 
 #include "numkong/types.h"
 #include "numkong/spatial/neon.h" // nk_f64_sqrt_neon
+#include "numkong/dot/serial.h"   // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/curved/neonbfdot.h

@@ -30,6 +30,7 @@
 #include "numkong/types.h"        // `nk_bf16_t`
 #include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
 #include "numkong/cast/serial.h"  // `nk_bf16_to_f32_serial`
+#include "numkong/dot/serial.h"   // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/curved/smef64.h

@@ -56,6 +56,7 @@
 #include "numkong/spatial/neon.h"  // `nk_f64_sqrt_neon`
 #include "numkong/dots/sme.h"      // `nk_sme_zero_za64_tile_0_`
 #include "numkong/curved/serial.h" // `nk_bilinear_f64_serial`
+#include "numkong/dot/serial.h"    // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/README.md

@@ -46,28 +46,28 @@ def vdot_complex(a: List[number], b: List[number]) -> Tuple[number, number]:
 
 Real and integer dot products:
 
-| Input Type | Output Type | Description                                    |
-| ---------- | ----------- | ---------------------------------------------- |
-| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output             |
-| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits      |
-| `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits      |
-| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits   |
-| `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits   |
-| `i8`       | `i32`       | 8-bit signed integers                          |
-| `u8`       | `u32`       | 8-bit unsigned integers                        |
-| `i4`       | `i32`       | 4-bit signed integers, packed nibble pairs     |
-| `u4`       | `u32`       | 4-bit unsigned integers, packed nibble pairs   |
-| `u1`       | `u32`       | 1-bit binary packed octets, popcount of AND    |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision                 |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output   |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
+| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits        |
+| `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits        |
+| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits     |
+| `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits     |
+| `i8`       | `i32`       | 8-bit signed integers                            |
+| `u8`       | `u32`       | 8-bit unsigned integers                          |
+| `i4`       | `i32`       | 4-bit signed integers, packed nibble pairs       |
+| `u4`       | `u32`       | 4-bit unsigned integers, packed nibble pairs     |
+| `u1`       | `u32`       | 1-bit binary packed octets, popcount of AND      |
 
 Complex dot products (both `dot` and `vdot`):
 
 | Input Type | Output Type | Description                                |
 | ---------- | ----------- | ------------------------------------------ |
 | `f64c`     | `f64c`      | 64-bit complex pairs                       |
-| `f32c`     | `f32c`      | 32-bit complex pairs                       |
+| `f32c`     | `f64c`      | 32-bit complex pairs, widened output       |
 | `f16c`     | `f32c`      | 16-bit complex pairs, widened output       |
 | `bf16c`    | `f32c`      | 16-bit brain complex pairs, widened output |
 
@@ -330,8 +330,8 @@ Measured with Wasmtime v42 (Cranelift backend).
 | __f16c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_dot_f16c_serial`      |      15.8 gb/s, 20.8 ulp |      13.0 gb/s, 64.1 ulp |      12.3 gb/s, 73.1 ulp |
 | `nk_vdot_f16c_serial`     |      15.8 gb/s, 24.8 ulp |      13.0 gb/s, 31.9 ulp |       12.3 gb/s, 137 ulp |
-| `nk_dot_f16c_neonhalf`    |       26.1 gb/s, 3.0 ulp |       18.4 gb/s, 6.5 ulp |      16.8 gb/s, 20.5 ulp |
-| `nk_vdot_f16c_neonhalf`   |      26.1 gb/s, 34.9 ulp |      18.5 gb/s, 40.7 ulp |      17.0 gb/s, 73.1 ulp |
+| `nk_dot_f16c_neon`        |       26.1 gb/s, 3.0 ulp |       18.4 gb/s, 6.5 ulp |      16.8 gb/s, 20.5 ulp |
+| `nk_vdot_f16c_neon`       |      26.1 gb/s, 34.9 ulp |      18.5 gb/s, 40.7 ulp |      17.0 gb/s, 73.1 ulp |
 | `nk_dot_f16c_neonfhm`     |       25.3 gb/s, 3.0 ulp |       17.1 gb/s, 6.5 ulp |      15.9 gb/s, 20.5 ulp |
 | `nk_vdot_f16c_neonfhm`    |      25.0 gb/s, 31.4 ulp |      17.0 gb/s, 38.6 ulp |      15.8 gb/s, 67.6 ulp |
 | __f64__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |

package/numkong/include/numkong/dot/alder.h

@@ -192,8 +192,8 @@ NK_INTERNAL void nk_dot_i8x32_finalize_alder(
     nk_dot_i8x32_state_alder_t const *state_c, nk_dot_i8x32_state_alder_t const *state_d, //
     nk_size_t total_dimensions,                                                           //
     nk_i32_t a_sum, /* A row sum (unused for i8) */                                       //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                     //
-    nk_b128_vec_t *results) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                          //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -217,8 +217,8 @@ NK_INTERNAL void nk_dot_i8x32_finalize_alder(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased − 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    results->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
 }
 
 NK_PUBLIC void nk_dot_u8_alder(nk_u8_t const *a_scalars, nk_u8_t const *b_scalars, nk_size_t count_scalars,
@@ -312,8 +312,8 @@ NK_INTERNAL void nk_dot_u8x32_finalize_alder(
     nk_dot_u8x32_state_alder_t const *state_c, nk_dot_u8x32_state_alder_t const *state_d, //
     nk_size_t total_dimensions,                                                           //
     nk_i32_t a_sum, /* A row sum (unused for u8) */                                       //
-    nk_b128_vec_t b_sums, /* 4 × u32 B column sums */                                     //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × u32 B column sums */                          //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -337,8 +337,8 @@ NK_INTERNAL void nk_dot_u8x32_finalize_alder(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased + 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    result->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
 }
 
 /**

package/numkong/include/numkong/dot/haswell.h

@@ -895,7 +895,7 @@ nk_dot_e3m2_haswell_cycle:
 
 /**
  *  @brief Internal helper state for dot-products of low-precision types, where 32-bit accumulation is enough.
- *  @sa nk_dot_f16x8_state_haswell_t, nk_dot_bf16x8_state_haswell_t
+ *  @sa nk_dot_f16x8_state_haswell_t, nk_dot_bf16x16_state_haswell_t
  *  @sa nk_dot_e4m3x16_state_haswell_t, nk_dot_e5m2x16_state_haswell_t
  */
 typedef struct nk_dot_through_f32_state_haswell_t_ {
@@ -904,7 +904,7 @@ typedef struct nk_dot_through_f32_state_haswell_t_ {
 
 /**
  *  @brief Initializes 32-bit accumulators for low-precision dot-products.
- *  @sa nk_dot_f16x8_init_haswell, nk_dot_bf16x8_init_haswell
+ *  @sa nk_dot_f16x8_init_haswell, nk_dot_bf16x16_init_haswell
  *  @sa nk_dot_e4m3x16_init_haswell, nk_dot_e5m2x16_init_haswell
  */
 NK_INTERNAL void nk_dot_through_f32_init_haswell_(nk_dot_through_f32_state_haswell_t_ *state) {
@@ -913,7 +913,7 @@ NK_INTERNAL void nk_dot_through_f32_init_haswell_(nk_dot_through_f32_state_haswe
 
 /**
  *  @brief Fuses 32-bit multiplication and accumulation for low-precision dot-products.
- *  @sa nk_dot_f16x8_update_haswell, nk_dot_bf16x8_update_haswell
+ *  @sa nk_dot_f16x8_update_haswell, nk_dot_bf16x16_update_haswell
  *  @sa nk_dot_e4m3x16_update_haswell, nk_dot_e5m2x16_update_haswell
  */
 NK_INTERNAL void nk_dot_through_f32_update_haswell_(nk_dot_through_f32_state_haswell_t_ *state, nk_b256_vec_t a,
@@ -991,7 +991,7 @@ NK_INTERNAL void nk_dot_e4m3x32_update_haswell_(nk_dot_through_f32_state_haswell
 
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_f16x8_finalize_haswell, nk_dot_bf16x8_finalize_haswell
+ *  @sa nk_dot_f16x8_finalize_haswell, nk_dot_bf16x16_finalize_haswell
  *  @sa nk_dot_e4m3x16_finalize_haswell, nk_dot_e5m2x16_finalize_haswell
  *
  *  The goal of this kernel is simple - compute 4x horizontal reductions, each involving 8x floats.
@@ -1637,8 +1637,8 @@ NK_INTERNAL void nk_dot_i4x32_finalize_haswell(
     nk_dot_i4x32_state_haswell_t const *state_c, nk_dot_i4x32_state_haswell_t const *state_d, //
     nk_size_t total_dimensions,                                                               //
     nk_i32_t a_sum, /* A row sum (signed sum of i4 values) */                                 //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                         //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                              //
+    nk_b128_vec_t *result_vec) {
 
     // Compensated 4-way reduction with external correction sums.
     // Formula: result = biased_product − 8×(Σa + Σb) − 64×depth_padded
@@ -1666,10 +1666,10 @@ NK_INTERNAL void nk_dot_i4x32_finalize_haswell(
 
     // Apply compensation: result = biased − 8×(Σa + Σb) − 64×depth_padded
     __m128i a_sum_broadcast_i32x4 = _mm_set1_epi32(a_sum);
-    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums.xmm);
+    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums_vec->xmm);
     __m128i correction_i32x4 = _mm_slli_epi32(ab_sums_i32x4, 3); // × 8
     __m128i offset_i32x4 = _mm_set1_epi32((nk_i32_t)(-64LL * (nk_i64_t)depth_nibbles));
-    result->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
+    result_vec->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
 }
 
 /**

package/numkong/include/numkong/dot/icelake.h

@@ -246,8 +246,8 @@ NK_INTERNAL void nk_dot_i8x64_finalize_icelake(
     nk_dot_i8x64_state_icelake_t const *state_c, nk_dot_i8x64_state_icelake_t const *state_d, //
     nk_size_t total_dimensions,                                                               //
     nk_i32_t a_sum, /* A row sum (unused for i8) */                                           //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                         //
-    nk_b128_vec_t *results) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                              //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -277,8 +277,8 @@ NK_INTERNAL void nk_dot_i8x64_finalize_icelake(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased − 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    results->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
 }
 
 typedef struct nk_dot_u8x64_state_icelake_t {
@@ -313,8 +313,8 @@ NK_INTERNAL void nk_dot_u8x64_finalize_icelake(
     nk_dot_u8x64_state_icelake_t const *state_c, nk_dot_u8x64_state_icelake_t const *state_d, //
     nk_size_t total_dimensions,                                                               //
     nk_i32_t a_sum, /* A row sum (unused for u8) */                                           //
-    nk_b128_vec_t b_sums, /* 4 × u32 B column sums */                                         //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × u32 B column sums */                              //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -344,8 +344,8 @@ NK_INTERNAL void nk_dot_u8x64_finalize_icelake(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased + 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    result->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
 }
 
 /**
@@ -568,8 +568,8 @@ NK_INTERNAL void nk_dot_i4x128_finalize_icelake(
     nk_dot_i4x128_state_icelake_t const *state_c, nk_dot_i4x128_state_icelake_t const *state_d, //
     nk_size_t total_dimensions,                                                                 //
     nk_i32_t a_sum, /* A row sum (signed sum of i4 values) */                                   //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                           //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                                //
+    nk_b128_vec_t *result_vec) {
 
     // Compensated 4-way reduction with external correction sums.
     // Formula: result = biased_product − 8×(Σa + Σb) − 64×depth_padded
@@ -606,10 +606,10 @@ NK_INTERNAL void nk_dot_i4x128_finalize_icelake(
 
     // Apply compensation: result = biased − 8×(Σa + Σb) − 64×depth_padded
     __m128i a_sum_broadcast_i32x4 = _mm_set1_epi32(a_sum);
-    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums.xmm);
+    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums_vec->xmm);
     __m128i correction_i32x4 = _mm_slli_epi32(ab_sums_i32x4, 3); // × 8
     __m128i offset_i32x4 = _mm_set1_epi32((nk_i32_t)(-64LL * (nk_i64_t)depth_nibbles));
-    result->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
+    result_vec->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
 }
 
 typedef struct nk_dot_u4x128_state_icelake_t {

package/numkong/include/numkong/dot/loongsonasx.h

@@ -19,7 +19,7 @@
  *  @section dot_loongsonasx_stateful Stateful Streaming Logic
  *
  *  - nk_dot_f64x4 state with Dot2 stable dot-products,
- *  - nk_dot_f32x4 state with double-precision numerics,
+ *  - nk_dot_f32x8 state with double-precision numerics,
  *  - nk_dot_through_i32 state for 8-bit signed and unsigned integer inputs.
  */
 #ifndef NK_DOT_LOONGSONASX_H
@@ -348,7 +348,7 @@ NK_INTERNAL void nk_dot_f32x8_finalize_loongsonasx(
 
 /**
  *  @brief Internal helper state for dot-products of integer types, where 32-bit accumulation is enough.
- *  @sa nk_dot_i8x16_state_loongsonasx_t, nk_dot_u8x16_state_loongsonasx_t
+ *  @sa nk_dot_i8x32_state_loongsonasx_t, nk_dot_u8x32_state_loongsonasx_t
  */
 typedef struct nk_dot_through_i32_state_loongsonasx_t_ {
     __m256i sum_i32x8;
@@ -356,7 +356,7 @@ typedef struct nk_dot_through_i32_state_loongsonasx_t_ {
 
 /**
  *  @brief Initializes 32-bit accumulators for integer dot-products.
- *  @sa nk_dot_i8x16_update_loongsonasx, nk_dot_u8x16_update_loongsonasx
+ *  @sa nk_dot_i8x32_update_loongsonasx, nk_dot_u8x32_update_loongsonasx
  */
 NK_INTERNAL void nk_dot_through_i32_init_loongsonasx_(nk_dot_through_i32_state_loongsonasx_t_ *state) {
     state->sum_i32x8 = __lasx_xvreplgr2vr_w(0);
@@ -364,7 +364,7 @@ NK_INTERNAL void nk_dot_through_i32_init_loongsonasx_(nk_dot_through_i32_state_l
 
 /**
  *  @brief Finalizes 4x integer dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_i8x16_update_loongsonasx, nk_dot_u8x16_update_loongsonasx
+ *  @sa nk_dot_i8x32_update_loongsonasx, nk_dot_u8x32_update_loongsonasx
  */
 NK_INTERNAL void nk_dot_through_i32_finalize_loongsonasx_(                                                          //
     nk_dot_through_i32_state_loongsonasx_t_ const *state_a, nk_dot_through_i32_state_loongsonasx_t_ const *state_b, //
@@ -474,7 +474,7 @@ NK_INTERNAL void nk_dot_through_f32_init_loongsonasx_(nk_dot_through_f32_state_l
 
 /**
  *  @brief Fuses 32-bit multiplication and accumulation for pre-converted f32 vectors.
- *  @sa nk_dot_bf16x8_update_loongsonasx
+ *  @sa nk_dot_bf16x16_update_loongsonasx
  */
 NK_INTERNAL void nk_dot_through_f32_update_loongsonasx_(nk_dot_through_f32_state_loongsonasx_t_ *state, nk_b256_vec_t a,
                                                         nk_b256_vec_t b, nk_size_t depth_offset,
@@ -486,7 +486,7 @@ NK_INTERNAL void nk_dot_through_f32_update_loongsonasx_(nk_dot_through_f32_state
 
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_bf16x8_finalize_loongsonasx
+ *  @sa nk_dot_bf16x16_finalize_loongsonasx
  *
  *  Computes 4x horizontal reductions, each involving 8x floats, using LASX interleave instructions.
  */

package/numkong/include/numkong/dot/neon.h

@@ -89,7 +89,8 @@
 #if NK_TARGET_ARM64_
 #if NK_TARGET_NEON
 
-#include "numkong/cast/neon.h" // `nk_e4m3x8_to_f16x8_neon_`
+#include "numkong/cast/neon.h"  // `nk_e4m3x8_to_f16x8_neon_`
+#include "numkong/dot/serial.h" // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/neonbfdot.h

@@ -63,6 +63,7 @@
 #include "numkong/types.h"
 #include "numkong/cast/serial.h" // `nk_partial_load_b8x8_serial_`
 #include "numkong/cast/neon.h"   // `nk_e4m3x8_to_bf16x8_neon_`
+#include "numkong/dot/serial.h"  // `nk_dot_bf16c_serial`, `nk_vdot_bf16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/neonfhm.h

@@ -65,6 +65,7 @@
 #include "numkong/types.h"
 #include "numkong/cast/serial.h" // `nk_partial_load_b8x8_serial_`
 #include "numkong/cast/neon.h"   // `nk_e4m3x8_to_f16x8_neon_`
+#include "numkong/dot/serial.h"  // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/neonsdot.h

@@ -100,6 +100,7 @@
 #if NK_TARGET_NEONSDOT
 
 #include "numkong/types.h"
+#include "numkong/cast/serial.h" // `nk_partial_load_b8x16_serial_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/powervsx.h

@@ -599,7 +599,7 @@ NK_INTERNAL void nk_dot_i8x16_finalize_powervsx(
     nk_dot_i8x16_state_powervsx_t const *state_a, nk_dot_i8x16_state_powervsx_t const *state_b, //
     nk_dot_i8x16_state_powervsx_t const *state_c, nk_dot_i8x16_state_powervsx_t const *state_d, //
     nk_size_t total_dimensions,                                                                 //
-    nk_i32_t a_sum, nk_b128_vec_t b_sums, nk_b128_vec_t *result) {
+    nk_i32_t a_sum, nk_b128_vec_t const *b_sums_vec, nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -624,8 +624,8 @@ NK_INTERNAL void nk_dot_i8x16_finalize_powervsx(
     // Correction: VMSUMMBM(b, a⊕0x80) = Σ(b_i·(a_i+128)) = a·b + 128·Σb
     // So a·b = biased − 128·Σb. B column sums are precomputed during packing.
     nk_vu32x4_t shift_u32x4 = vec_splats((nk_u32_t)7);
-    nk_vi32x4_t correction_i32x4 = (nk_vi32x4_t)vec_sl((nk_vu32x4_t)b_sums.vi32x4, shift_u32x4);
-    result->vi32x4 = vec_sub(biased_i32x4, correction_i32x4);
+    nk_vi32x4_t correction_i32x4 = (nk_vi32x4_t)vec_sl((nk_vu32x4_t)b_sums_vec->vi32x4, shift_u32x4);
+    result_vec->vi32x4 = vec_sub(biased_i32x4, correction_i32x4);
 }
 
 /** @brief Running state for i8 column sum precomputation on Power VSX. */

package/numkong/include/numkong/dot/serial.h

@@ -273,6 +273,12 @@ typedef struct nk_dot_f64x2_state_serial_t {
     nk_f64_t compensations[2];
 } nk_dot_f64x2_state_serial_t;
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 NK_INTERNAL void nk_dot_f64x2_init_serial(nk_dot_f64x2_state_serial_t *state) {
     state->sums[0] = 0, state->sums[1] = 0;
     state->compensations[0] = 0, state->compensations[1] = 0;
@@ -808,6 +814,13 @@ NK_INTERNAL void nk_dot_i4x16_finalize_serial(nk_dot_i4x16_state_serial_t const
 
 #pragma region Binary
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_dot_u1_serial(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size_t n_bits, nk_u32_t *result) {
     nk_u32_t dot = 0;
     nk_size_t bytes = nk_size_divide_round_up_(n_bits, NK_BITS_PER_BYTE);
@@ -819,6 +832,12 @@ typedef struct nk_dot_u1x128_state_serial_t {
     nk_u32_t dot_count;
 } nk_dot_u1x128_state_serial_t;
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 NK_INTERNAL void nk_dot_u1x128_init_serial(nk_dot_u1x128_state_serial_t *state) { state->dot_count = 0; }
 
 NK_INTERNAL void nk_dot_u1x128_update_serial(nk_dot_u1x128_state_serial_t *state, nk_b128_vec_t a, nk_b128_vec_t b,
@@ -876,12 +895,6 @@ NK_INTERNAL nk_i32_t nk_sum_i4x32_finalize_serial(nk_sum_i4x32_state_serial_t co
 
 #pragma endregion Stateful Element Sum Helpers
 
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/numkong/include/numkong/dot/skylake.h

@@ -145,8 +145,8 @@ NK_INTERNAL void nk_dot_through_f32_init_skylake_(nk_dot_through_f32_state_skyla
 
 /**
  *  @brief Fuses 32-bit multiplication and accumulation for low-precision dot-products.
- *  @sa nk_dot_f16x16_udpate_skylake, nk_dot_bf16x16_udpate_skylake
- *  @sa nk_dot_e4m3x16_udpate_skylake, nk_dot_e5m2x16_udpate_skylake
+ *  @sa nk_dot_f16x16_update_skylake, nk_dot_bf16x16_update_skylake
+ *  @sa nk_dot_e4m3x16_update_skylake, nk_dot_e5m2x16_update_skylake
  */
 NK_INTERNAL void nk_dot_through_f32_update_skylake_(nk_dot_through_f32_state_skylake_t_ *state, nk_b512_vec_t a,
                                                     nk_b512_vec_t b, nk_size_t depth_offset,
@@ -188,8 +188,8 @@ NK_INTERNAL void nk_dot_e5m2x64_update_skylake_(nk_dot_through_f32_state_skylake
 
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_f16x16_udpate_skylake, nk_dot_bf16x16_udpate_skylake
- *  @sa nk_dot_e4m3x16_udpate_skylake, nk_dot_e5m2x16_udpate_skylake
+ *  @sa nk_dot_f16x16_update_skylake, nk_dot_bf16x16_update_skylake
+ *  @sa nk_dot_e4m3x16_update_skylake, nk_dot_e5m2x16_update_skylake
  *
  *  The goal of this kernel is simple - compute 4x horizontal reductions, each involing 16x floats.
  *  The lack of vectorized horizontal instruction implies many consecutive shuffles producing a tree-like

package/numkong/include/numkong/dot/svehalf.h

@@ -1,6 +1,6 @@
 /**
  *  @brief SIMD-accelerated Dot Products for SVE FP16.
- *  @file include/numkong/dot/sve.h
+ *  @file include/numkong/dot/svehalf.h
  *  @author Ash Vardanian
  *  @date December 27, 2025
  *

package/numkong/include/numkong/dot/v128relaxed.h

@@ -691,21 +691,22 @@ NK_INTERNAL void nk_dot_u8x16_update_v128relaxed(nk_dot_u8x16_state_v128relaxed_
 NK_INTERNAL void nk_dot_u8x16_finalize_v128relaxed(                                                   //
     nk_dot_u8x16_state_v128relaxed_t const *state_a, nk_dot_u8x16_state_v128relaxed_t const *state_b, //
     nk_dot_u8x16_state_v128relaxed_t const *state_c, nk_dot_u8x16_state_v128relaxed_t const *state_d, //
-    nk_size_t total_dimensions, nk_u32_t a_sum, nk_b128_vec_t b_sums, nk_b128_vec_t *result) {
+    nk_size_t total_dimensions, nk_u32_t a_sum, nk_b128_vec_t const *b_sums_vec, nk_b128_vec_t *result_vec) {
+    nk_unused_(total_dimensions);
     nk_unused_(a_sum);
     // Σ a·b = reduce(lo) + 128·reduce(hi) + 128·Σb
-    result->u32s[0] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_a->product_low_i32x4) +
-                                 128 * nk_reduce_add_i32x4_v128relaxed_(state_a->product_high_i32x4) +
-                                 128 * (nk_i32_t)b_sums.u32s[0]);
-    result->u32s[1] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_b->product_low_i32x4) +
-                                 128 * nk_reduce_add_i32x4_v128relaxed_(state_b->product_high_i32x4) +
-                                 128 * (nk_i32_t)b_sums.u32s[1]);
-    result->u32s[2] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_c->product_low_i32x4) +
-                                 128 * nk_reduce_add_i32x4_v128relaxed_(state_c->product_high_i32x4) +
-                                 128 * (nk_i32_t)b_sums.u32s[2]);
-    result->u32s[3] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_d->product_low_i32x4) +
-                                 128 * nk_reduce_add_i32x4_v128relaxed_(state_d->product_high_i32x4) +
-                                 128 * (nk_i32_t)b_sums.u32s[3]);
+    result_vec->u32s[0] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_a->product_low_i32x4) +
+                                     128 * nk_reduce_add_i32x4_v128relaxed_(state_a->product_high_i32x4) +
+                                     128 * (nk_i32_t)b_sums_vec->u32s[0]);
+    result_vec->u32s[1] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_b->product_low_i32x4) +
+                                     128 * nk_reduce_add_i32x4_v128relaxed_(state_b->product_high_i32x4) +
+                                     128 * (nk_i32_t)b_sums_vec->u32s[1]);
+    result_vec->u32s[2] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_c->product_low_i32x4) +
+                                     128 * nk_reduce_add_i32x4_v128relaxed_(state_c->product_high_i32x4) +
+                                     128 * (nk_i32_t)b_sums_vec->u32s[2]);
+    result_vec->u32s[3] = (nk_u32_t)(nk_reduce_add_i32x4_v128relaxed_(state_d->product_low_i32x4) +
+                                     128 * nk_reduce_add_i32x4_v128relaxed_(state_d->product_high_i32x4) +
+                                     128 * (nk_i32_t)b_sums_vec->u32s[3]);
 }
 
 typedef struct nk_sum_u8x16_state_v128relaxed_t {
@@ -1099,18 +1100,18 @@ NK_INTERNAL void nk_dot_i4x32_finalize_v128relaxed(
     nk_dot_i4x32_state_v128relaxed_t const *state_c, nk_dot_i4x32_state_v128relaxed_t const *state_d, //
     nk_size_t total_dimensions,                                                                       //
     nk_i32_t a_sum, /* Row sum of A (signed sum of i4 values) */                                      //
-    nk_b128_vec_t b_sums, /* 4 × i32 column sums of B */                                              //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 column sums of B */                                   //
+    nk_b128_vec_t *result_vec) {
     // Match x86 compensated i4 finalizers: result = biased_dot - 8*(a_sum + b_sum) - 64*depth_padded
     nk_i64_t depth_padded = (nk_i64_t)nk_size_round_up_to_multiple_(total_dimensions, 32);
-    result->i32s[0] = nk_reduce_add_i32x4_v128relaxed_(state_a->biased_product_sum_i32x4) -
-                      8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums.i32s[0]) - 64 * depth_padded;
-    result->i32s[1] = nk_reduce_add_i32x4_v128relaxed_(state_b->biased_product_sum_i32x4) -
-                      8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums.i32s[1]) - 64 * depth_padded;
-    result->i32s[2] = nk_reduce_add_i32x4_v128relaxed_(state_c->biased_product_sum_i32x4) -
-                      8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums.i32s[2]) - 64 * depth_padded;
-    result->i32s[3] = nk_reduce_add_i32x4_v128relaxed_(state_d->biased_product_sum_i32x4) -
-                      8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums.i32s[3]) - 64 * depth_padded;
+    result_vec->i32s[0] = nk_reduce_add_i32x4_v128relaxed_(state_a->biased_product_sum_i32x4) -
+                          8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums_vec->i32s[0]) - 64 * depth_padded;
+    result_vec->i32s[1] = nk_reduce_add_i32x4_v128relaxed_(state_b->biased_product_sum_i32x4) -
+                          8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums_vec->i32s[1]) - 64 * depth_padded;
+    result_vec->i32s[2] = nk_reduce_add_i32x4_v128relaxed_(state_c->biased_product_sum_i32x4) -
+                          8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums_vec->i32s[2]) - 64 * depth_padded;
+    result_vec->i32s[3] = nk_reduce_add_i32x4_v128relaxed_(state_d->biased_product_sum_i32x4) -
+                          8 * ((nk_i64_t)a_sum + (nk_i64_t)b_sums_vec->i32s[3]) - 64 * depth_padded;
 }
 
 typedef struct nk_sum_i4x32_state_v128relaxed_t {