usearch 2.26.0 → 2.26.2

package/numkong/include/numkong/cast/serial.h

@@ -13,17 +13,6 @@
 extern "C" {
 #endif
 
-/*  Keep the serial conversions below actually scalar, regardless of build type.
- *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
- *  callers in dispatch_*.c, which wastes binary and breaks the nk_*_serial-as-scalar-oracle
- *  contract. See dots/serial.h. */
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
-#endif
-
 #pragma region Type Punned Loads and Stores
 
 /** @brief Type-agnostic 32-bit full load (scalar). */
@@ -563,6 +552,17 @@ NK_INTERNAL void nk_strided_load_b8x16_serial_(void const *src, nk_size_t stride
 
 #pragma endregion Type Punned Loads and Stores
 
+/*  Keep the serial conversions below actually scalar, regardless of build type.
+ *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
+ *  callers in dispatch_*.c, which wastes binary and breaks the nk_*_serial-as-scalar-oracle
+ *  contract. See dots/serial.h. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 /**
  *  @brief Expands an `f16` (IEEE-754 16-bit) to a `float`.
  *
@@ -622,6 +622,12 @@ NK_PUBLIC void nk_f16_to_f32_serial(nk_f16_t const *src, nk_f32_t *dest) {
 #endif
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 /** @brief Load 4 × f16 from memory and upcast them to 4 × f32. */
 NK_INTERNAL void nk_load_f16x4_to_f32x4_serial_(void const *src, nk_b128_vec_t *dst) {
     nk_f16_t const *scalars = (nk_f16_t const *)src;
@@ -644,6 +650,13 @@ NK_INTERNAL void nk_partial_load_f16x4_to_f32x4_serial_(nk_f16_t const *src, nk_
     }
 }
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 /**
  *  @brief Compresses a `float` to an `f16` (IEEE-754 16-bit).
  *
@@ -946,6 +959,12 @@ NK_PUBLIC void nk_f32_to_e4m3_serial(nk_f32_t const *src, nk_e4m3_t *dest) {
     *dest = (nk_e4m3_t)(sign | (exp_field << 3) | mant_field);
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 /**
  *  @brief Convert FP8 E4M3 to IEEE 754 half-precision float.
  *
@@ -1039,6 +1058,13 @@ NK_INTERNAL void nk_e5m2_to_f32_manual_(nk_e5m2_t const *src, nk_f32_t *dest) {
     *dest = conv.f;
 }
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_e5m2_to_f32_serial(nk_e5m2_t const *src, nk_f32_t *dest) {
     static nk_u32_t const lut[128] = {
         0x00000000, 0x37800000, 0x38000000, 0x38400000, // exp=0  sub
@@ -1181,6 +1207,12 @@ NK_PUBLIC void nk_f32_to_e5m2_serial(nk_f32_t const *src, nk_e5m2_t *dest) {
     *dest = (nk_e5m2_t)(sign | (exp_field << 2) | mant_field);
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 /**
  *  @brief Convert FP8 E5M2 to IEEE 754 half-precision float.
  *
@@ -1315,6 +1347,13 @@ NK_INTERNAL void nk_e2m3_to_f32_manual_(nk_e2m3_t const *src, nk_f32_t *dest) {
     *dest = conv.f;
 }
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_e2m3_to_f32_serial(nk_e2m3_t const *src, nk_f32_t *dest) {
     static nk_u32_t const lut[32] = {
         0x00000000, 0x3E000000, 0x3E800000, 0x3EC00000, 0x3F000000, 0x3F200000, 0x3F400000, 0x3F600000, // exp=0 sub
@@ -1415,6 +1454,12 @@ NK_PUBLIC void nk_f32_to_e2m3_serial(nk_f32_t const *src, nk_e2m3_t *dest) {
     *dest = (nk_e2m3_t)(sign | (exp_field << 3) | mant_field);
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 /**
  *  @brief Convert FP6 E3M2FN to IEEE 754 single-precision float.
  *
@@ -1452,6 +1497,13 @@ NK_INTERNAL void nk_e3m2_to_f32_manual_(nk_e3m2_t const *src, nk_f32_t *dest) {
     *dest = conv.f;
 }
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_e3m2_to_f32_serial(nk_e3m2_t const *src, nk_f32_t *dest) {
     static nk_u32_t const lut[32] = {
         0x00000000, 0x3D800000, 0x3E000000, 0x3E400000, // exp=0 sub
@@ -1557,6 +1609,12 @@ NK_PUBLIC void nk_f32_to_e3m2_serial(nk_f32_t const *src, nk_e3m2_t *dest) {
     *dest = (nk_e3m2_t)(sign | (exp_field << 2) | mant_field);
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 NK_INTERNAL void nk_f16_to_f64_serial(nk_f16_t const *x, nk_f64_t *y) {
     nk_f32_t f32;
     nk_f16_to_f32_serial(x, &f32);
@@ -1731,6 +1789,13 @@ NK_INTERNAL void nk_u64_to_bf16_serial(nk_u64_t const *x, nk_bf16_t *y) {
     nk_f32_to_bf16_serial(&f32, y);
 }
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 /** @brief Convert a pair of i4 (4-bit signed integer, -8 to 7) nibbles into signed integers. */
 NK_PUBLIC void nk_i4x2_to_i8x2_serial(nk_i4x2_t const *src, nk_i8_t *dest) {
     nk_u8_t byte = *(nk_u8_t const *)src;
@@ -1776,6 +1841,12 @@ typedef union NK_MAY_ALIAS_ nk_scalar_buffer_t {
     nk_u8_t u8;
 } nk_scalar_buffer_t;
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 /** @brief Reads a typed scalar from @p buf and writes the widened f64c into @p result.
  *  Real types set `.imag = 0`. Safe when @p result aliases @p buf (in-place conversion).
  *  @return 1 on success, 0 for unsupported types (sub-byte, unknown). */
@@ -2247,6 +2318,13 @@ NK_INTERNAL int nk_scalar_buffer_from_f64(nk_f64_t const *value, nk_scalar_buffe
 
 #pragma region Public API
 
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_PUBLIC void nk_cast_serial(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type) {
     if (from_type == to_type) {
         nk_size_t size_bits = nk_dtype_bits(from_type);

package/numkong/include/numkong/cast.h

@@ -11,9 +11,9 @@
  *  - `f16` & `bf16` ↔ `f32` - often used for half-precision dot-products on older CPUs,
  *
  *  Unlike most operation classes in NumKong, these are dependent on two input types: "from" & "to".
- *  It contains scalar helpers named like `nk_f16_to_f32_serial_` as well as buffer-to-buffer
- *  `memcpy`-like vectorized operations, such as `nk_cast_f16_to_f32` with `nk_cast_f16_to_f32_serial`,
- *  `nk_cast_f16_to_f32_neon`, `nk_cast_f16_to_f32_skylake`, and other platform-specific variants.
+ *  It contains scalar helpers named like `nk_f16_to_f32_serial` as well as buffer-to-buffer
+ *  `memcpy`-like vectorized operations, such as `nk_cast` with `nk_cast_serial`,
+ *  `nk_cast_neon`, `nk_cast_skylake`, and other platform-specific variants.
  *
  *  It also includes "partial load" and "partial store" type-punned helper functions for handling
  *  IO between memory and registers, that are extensively reused in reductions, elementwise ops, and

package/numkong/include/numkong/curved/README.md

@@ -42,19 +42,19 @@ def bilinear_complex(a: np.ndarray, b: np.ndarray, C: np.ndarray) -> complex:
 
 Real bilinear and Mahalanobis:
 
-| Input Type | Output Type | Description                                    |
-| ---------- | ----------- | ---------------------------------------------- |
-| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output             |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision                 |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output   |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
 
 Complex bilinear:
 
 | Input Type | Output Type | Description                                |
 | ---------- | ----------- | ------------------------------------------ |
 | `f64c`     | `f64c`      | 64-bit complex pairs                       |
-| `f32c`     | `f32c`      | 32-bit complex pairs                       |
+| `f32c`     | `f64c`      | 32-bit complex pairs, widened output       |
 | `f16c`     | `f32c`      | 16-bit complex pairs, widened output       |
 | `bf16c`    | `f32c`      | 16-bit brain complex pairs, widened output |
 
@@ -179,7 +179,7 @@ Measured with Wasmtime v42 (Cranelift backend).
 | `nk_bilinear_bf16c_neonbfdot`   |     5.05 gso/s, 17.0 ulp |     4.20 gso/s, 17.0 ulp |     4.04 gso/s, 17.0 ulp |
 | __f16c__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_bilinear_f16c_serial`       |     2.81 gso/s, 51.8 ulp |     2.54 gso/s, 51.8 ulp |     2.48 gso/s, 51.8 ulp |
-| `nk_bilinear_f16c_neonhalf`     |     5.00 gso/s, 17.3 ulp |     4.16 gso/s, 17.3 ulp |     4.00 gso/s, 16.4 ulp |
+| `nk_bilinear_f16c_neon`         |     5.00 gso/s, 17.3 ulp |     4.16 gso/s, 17.3 ulp |     4.00 gso/s, 16.4 ulp |
 | __f64__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_bilinear_f64_serial`        |     0.717 gso/s, 0.4 ulp |     0.711 gso/s, 0.4 ulp |     0.721 gso/s, 0.4 ulp |
 | `nk_mahalanobis_f64_serial`     |     0.664 gso/s, 0.5 ulp |     0.667 gso/s, 0.5 ulp |     0.672 gso/s, 0.5 ulp |
@@ -196,8 +196,8 @@ Measured with Wasmtime v42 (Cranelift backend).
 | __f16__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_bilinear_f16_serial`        |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
 | `nk_mahalanobis_f16_serial`     |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
-| `nk_bilinear_f16_neonhalf`      |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
-| `nk_mahalanobis_f16_neonhalf`   |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_bilinear_f16_neon`          |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_mahalanobis_f16_neon`       |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
 
 #### WASM
 

package/numkong/include/numkong/curved/neon.h

@@ -29,6 +29,7 @@
 
 #include "numkong/types.h"
 #include "numkong/spatial/neon.h" // nk_f64_sqrt_neon
+#include "numkong/dot/serial.h"   // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/curved/neonbfdot.h

@@ -30,6 +30,7 @@
 #include "numkong/types.h"        // `nk_bf16_t`
 #include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
 #include "numkong/cast/serial.h"  // `nk_bf16_to_f32_serial`
+#include "numkong/dot/serial.h"   // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/curved/smef64.h

@@ -56,6 +56,7 @@
 #include "numkong/spatial/neon.h"  // `nk_f64_sqrt_neon`
 #include "numkong/dots/sme.h"      // `nk_sme_zero_za64_tile_0_`
 #include "numkong/curved/serial.h" // `nk_bilinear_f64_serial`
+#include "numkong/dot/serial.h"    // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/README.md

@@ -46,28 +46,28 @@ def vdot_complex(a: List[number], b: List[number]) -> Tuple[number, number]:
 
 Real and integer dot products:
 
-| Input Type | Output Type | Description                                    |
-| ---------- | ----------- | ---------------------------------------------- |
-| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
-| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
-| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
-| `bf16`     | `f32`       | 16-bit brain float, widened output             |
-| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits      |
-| `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits      |
-| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits   |
-| `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits   |
-| `i8`       | `i32`       | 8-bit signed integers                          |
-| `u8`       | `u32`       | 8-bit unsigned integers                        |
-| `i4`       | `i32`       | 4-bit signed integers, packed nibble pairs     |
-| `u4`       | `u32`       | 4-bit unsigned integers, packed nibble pairs   |
-| `u1`       | `u32`       | 1-bit binary packed octets, popcount of AND    |
+| Input Type | Output Type | Description                                      |
+| ---------- | ----------- | ------------------------------------------------ |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision                 |
+| `f32`      | `f64`       | 32-bit IEEE 754 single precision, widened output |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output   |
+| `bf16`     | `f32`       | 16-bit brain float, widened output               |
+| `e4m3`     | `f32`       | 8-bit Float8: 4 exponent, 3 mantissa bits        |
+| `e5m2`     | `f32`       | 8-bit Float8: 5 exponent, 2 mantissa bits        |
+| `e2m3`     | `f32`       | 8-bit MX format: 2 exponent, 3 mantissa bits     |
+| `e3m2`     | `f32`       | 8-bit MX format: 3 exponent, 2 mantissa bits     |
+| `i8`       | `i32`       | 8-bit signed integers                            |
+| `u8`       | `u32`       | 8-bit unsigned integers                          |
+| `i4`       | `i32`       | 4-bit signed integers, packed nibble pairs       |
+| `u4`       | `u32`       | 4-bit unsigned integers, packed nibble pairs     |
+| `u1`       | `u32`       | 1-bit binary packed octets, popcount of AND      |
 
 Complex dot products (both `dot` and `vdot`):
 
 | Input Type | Output Type | Description                                |
 | ---------- | ----------- | ------------------------------------------ |
 | `f64c`     | `f64c`      | 64-bit complex pairs                       |
-| `f32c`     | `f32c`      | 32-bit complex pairs                       |
+| `f32c`     | `f64c`      | 32-bit complex pairs, widened output       |
 | `f16c`     | `f32c`      | 16-bit complex pairs, widened output       |
 | `bf16c`    | `f32c`      | 16-bit brain complex pairs, widened output |
 
@@ -330,8 +330,8 @@ Measured with Wasmtime v42 (Cranelift backend).
 | __f16c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
 | `nk_dot_f16c_serial`      |      15.8 gb/s, 20.8 ulp |      13.0 gb/s, 64.1 ulp |      12.3 gb/s, 73.1 ulp |
 | `nk_vdot_f16c_serial`     |      15.8 gb/s, 24.8 ulp |      13.0 gb/s, 31.9 ulp |       12.3 gb/s, 137 ulp |
-| `nk_dot_f16c_neonhalf`    |       26.1 gb/s, 3.0 ulp |       18.4 gb/s, 6.5 ulp |      16.8 gb/s, 20.5 ulp |
-| `nk_vdot_f16c_neonhalf`   |      26.1 gb/s, 34.9 ulp |      18.5 gb/s, 40.7 ulp |      17.0 gb/s, 73.1 ulp |
+| `nk_dot_f16c_neon`        |       26.1 gb/s, 3.0 ulp |       18.4 gb/s, 6.5 ulp |      16.8 gb/s, 20.5 ulp |
+| `nk_vdot_f16c_neon`       |      26.1 gb/s, 34.9 ulp |      18.5 gb/s, 40.7 ulp |      17.0 gb/s, 73.1 ulp |
 | `nk_dot_f16c_neonfhm`     |       25.3 gb/s, 3.0 ulp |       17.1 gb/s, 6.5 ulp |      15.9 gb/s, 20.5 ulp |
 | `nk_vdot_f16c_neonfhm`    |      25.0 gb/s, 31.4 ulp |      17.0 gb/s, 38.6 ulp |      15.8 gb/s, 67.6 ulp |
 | __f64__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |

package/numkong/include/numkong/dot/alder.h

@@ -192,8 +192,8 @@ NK_INTERNAL void nk_dot_i8x32_finalize_alder(
     nk_dot_i8x32_state_alder_t const *state_c, nk_dot_i8x32_state_alder_t const *state_d, //
     nk_size_t total_dimensions,                                                           //
     nk_i32_t a_sum, /* A row sum (unused for i8) */                                       //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                     //
-    nk_b128_vec_t *results) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                          //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -217,8 +217,8 @@ NK_INTERNAL void nk_dot_i8x32_finalize_alder(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased − 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    results->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
 }
 
 NK_PUBLIC void nk_dot_u8_alder(nk_u8_t const *a_scalars, nk_u8_t const *b_scalars, nk_size_t count_scalars,
@@ -312,8 +312,8 @@ NK_INTERNAL void nk_dot_u8x32_finalize_alder(
     nk_dot_u8x32_state_alder_t const *state_c, nk_dot_u8x32_state_alder_t const *state_d, //
     nk_size_t total_dimensions,                                                           //
     nk_i32_t a_sum, /* A row sum (unused for u8) */                                       //
-    nk_b128_vec_t b_sums, /* 4 × u32 B column sums */                                     //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × u32 B column sums */                          //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -337,8 +337,8 @@ NK_INTERNAL void nk_dot_u8x32_finalize_alder(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased + 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    result->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
 }
 
 /**

package/numkong/include/numkong/dot/haswell.h

@@ -895,7 +895,7 @@ nk_dot_e3m2_haswell_cycle:
 
 /**
  *  @brief Internal helper state for dot-products of low-precision types, where 32-bit accumulation is enough.
- *  @sa nk_dot_f16x8_state_haswell_t, nk_dot_bf16x8_state_haswell_t
+ *  @sa nk_dot_f16x8_state_haswell_t, nk_dot_bf16x16_state_haswell_t
  *  @sa nk_dot_e4m3x16_state_haswell_t, nk_dot_e5m2x16_state_haswell_t
  */
 typedef struct nk_dot_through_f32_state_haswell_t_ {
@@ -904,7 +904,7 @@ typedef struct nk_dot_through_f32_state_haswell_t_ {
 
 /**
  *  @brief Initializes 32-bit accumulators for low-precision dot-products.
- *  @sa nk_dot_f16x8_init_haswell, nk_dot_bf16x8_init_haswell
+ *  @sa nk_dot_f16x8_init_haswell, nk_dot_bf16x16_init_haswell
  *  @sa nk_dot_e4m3x16_init_haswell, nk_dot_e5m2x16_init_haswell
  */
 NK_INTERNAL void nk_dot_through_f32_init_haswell_(nk_dot_through_f32_state_haswell_t_ *state) {
@@ -913,7 +913,7 @@ NK_INTERNAL void nk_dot_through_f32_init_haswell_(nk_dot_through_f32_state_haswe
 
 /**
  *  @brief Fuses 32-bit multiplication and accumulation for low-precision dot-products.
- *  @sa nk_dot_f16x8_update_haswell, nk_dot_bf16x8_update_haswell
+ *  @sa nk_dot_f16x8_update_haswell, nk_dot_bf16x16_update_haswell
  *  @sa nk_dot_e4m3x16_update_haswell, nk_dot_e5m2x16_update_haswell
  */
 NK_INTERNAL void nk_dot_through_f32_update_haswell_(nk_dot_through_f32_state_haswell_t_ *state, nk_b256_vec_t a,
@@ -991,7 +991,7 @@ NK_INTERNAL void nk_dot_e4m3x32_update_haswell_(nk_dot_through_f32_state_haswell
 
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_f16x8_finalize_haswell, nk_dot_bf16x8_finalize_haswell
+ *  @sa nk_dot_f16x8_finalize_haswell, nk_dot_bf16x16_finalize_haswell
  *  @sa nk_dot_e4m3x16_finalize_haswell, nk_dot_e5m2x16_finalize_haswell
  *
  *  The goal of this kernel is simple - compute 4x horizontal reductions, each involving 8x floats.
@@ -1637,8 +1637,8 @@ NK_INTERNAL void nk_dot_i4x32_finalize_haswell(
     nk_dot_i4x32_state_haswell_t const *state_c, nk_dot_i4x32_state_haswell_t const *state_d, //
     nk_size_t total_dimensions,                                                               //
     nk_i32_t a_sum, /* A row sum (signed sum of i4 values) */                                 //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                         //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                              //
+    nk_b128_vec_t *result_vec) {
 
     // Compensated 4-way reduction with external correction sums.
     // Formula: result = biased_product − 8×(Σa + Σb) − 64×depth_padded
@@ -1666,10 +1666,10 @@ NK_INTERNAL void nk_dot_i4x32_finalize_haswell(
 
     // Apply compensation: result = biased − 8×(Σa + Σb) − 64×depth_padded
     __m128i a_sum_broadcast_i32x4 = _mm_set1_epi32(a_sum);
-    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums.xmm);
+    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums_vec->xmm);
     __m128i correction_i32x4 = _mm_slli_epi32(ab_sums_i32x4, 3); // × 8
     __m128i offset_i32x4 = _mm_set1_epi32((nk_i32_t)(-64LL * (nk_i64_t)depth_nibbles));
-    result->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
+    result_vec->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
 }
 
 /**

package/numkong/include/numkong/dot/icelake.h

@@ -246,8 +246,8 @@ NK_INTERNAL void nk_dot_i8x64_finalize_icelake(
     nk_dot_i8x64_state_icelake_t const *state_c, nk_dot_i8x64_state_icelake_t const *state_d, //
     nk_size_t total_dimensions,                                                               //
     nk_i32_t a_sum, /* A row sum (unused for i8) */                                           //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                         //
-    nk_b128_vec_t *results) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                              //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -277,8 +277,8 @@ NK_INTERNAL void nk_dot_i8x64_finalize_icelake(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased − 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    results->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_sub_epi32(biased_i32x4, correction_i32x4);
 }
 
 typedef struct nk_dot_u8x64_state_icelake_t {
@@ -313,8 +313,8 @@ NK_INTERNAL void nk_dot_u8x64_finalize_icelake(
     nk_dot_u8x64_state_icelake_t const *state_c, nk_dot_u8x64_state_icelake_t const *state_d, //
     nk_size_t total_dimensions,                                                               //
     nk_i32_t a_sum, /* A row sum (unused for u8) */                                           //
-    nk_b128_vec_t b_sums, /* 4 × u32 B column sums */                                         //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × u32 B column sums */                              //
+    nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -344,8 +344,8 @@ NK_INTERNAL void nk_dot_u8x64_finalize_icelake(
         _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased + 128 × Σb
-    __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
-    result->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
+    __m128i correction_i32x4 = _mm_slli_epi32(b_sums_vec->xmm, 7); // × 128
+    result_vec->xmm = _mm_add_epi32(biased_i32x4, correction_i32x4);
 }
 
 /**
@@ -568,8 +568,8 @@ NK_INTERNAL void nk_dot_i4x128_finalize_icelake(
     nk_dot_i4x128_state_icelake_t const *state_c, nk_dot_i4x128_state_icelake_t const *state_d, //
     nk_size_t total_dimensions,                                                                 //
     nk_i32_t a_sum, /* A row sum (signed sum of i4 values) */                                   //
-    nk_b128_vec_t b_sums, /* 4 × i32 B column sums */                                           //
-    nk_b128_vec_t *result) {
+    nk_b128_vec_t const *b_sums_vec, /* 4 × i32 B column sums */                                //
+    nk_b128_vec_t *result_vec) {
 
     // Compensated 4-way reduction with external correction sums.
     // Formula: result = biased_product − 8×(Σa + Σb) − 64×depth_padded
@@ -606,10 +606,10 @@ NK_INTERNAL void nk_dot_i4x128_finalize_icelake(
 
     // Apply compensation: result = biased − 8×(Σa + Σb) − 64×depth_padded
     __m128i a_sum_broadcast_i32x4 = _mm_set1_epi32(a_sum);
-    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums.xmm);
+    __m128i ab_sums_i32x4 = _mm_add_epi32(a_sum_broadcast_i32x4, b_sums_vec->xmm);
     __m128i correction_i32x4 = _mm_slli_epi32(ab_sums_i32x4, 3); // × 8
     __m128i offset_i32x4 = _mm_set1_epi32((nk_i32_t)(-64LL * (nk_i64_t)depth_nibbles));
-    result->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
+    result_vec->xmm = _mm_add_epi32(_mm_sub_epi32(biased_i32x4, correction_i32x4), offset_i32x4);
 }
 
 typedef struct nk_dot_u4x128_state_icelake_t {

package/numkong/include/numkong/dot/loongsonasx.h

@@ -19,7 +19,7 @@
  *  @section dot_loongsonasx_stateful Stateful Streaming Logic
  *
  *  - nk_dot_f64x4 state with Dot2 stable dot-products,
- *  - nk_dot_f32x4 state with double-precision numerics,
+ *  - nk_dot_f32x8 state with double-precision numerics,
  *  - nk_dot_through_i32 state for 8-bit signed and unsigned integer inputs.
  */
 #ifndef NK_DOT_LOONGSONASX_H
@@ -348,7 +348,7 @@ NK_INTERNAL void nk_dot_f32x8_finalize_loongsonasx(
 
 /**
  *  @brief Internal helper state for dot-products of integer types, where 32-bit accumulation is enough.
- *  @sa nk_dot_i8x16_state_loongsonasx_t, nk_dot_u8x16_state_loongsonasx_t
+ *  @sa nk_dot_i8x32_state_loongsonasx_t, nk_dot_u8x32_state_loongsonasx_t
  */
 typedef struct nk_dot_through_i32_state_loongsonasx_t_ {
     __m256i sum_i32x8;
@@ -356,7 +356,7 @@ typedef struct nk_dot_through_i32_state_loongsonasx_t_ {
 
 /**
  *  @brief Initializes 32-bit accumulators for integer dot-products.
- *  @sa nk_dot_i8x16_update_loongsonasx, nk_dot_u8x16_update_loongsonasx
+ *  @sa nk_dot_i8x32_update_loongsonasx, nk_dot_u8x32_update_loongsonasx
  */
 NK_INTERNAL void nk_dot_through_i32_init_loongsonasx_(nk_dot_through_i32_state_loongsonasx_t_ *state) {
     state->sum_i32x8 = __lasx_xvreplgr2vr_w(0);
@@ -364,7 +364,7 @@ NK_INTERNAL void nk_dot_through_i32_init_loongsonasx_(nk_dot_through_i32_state_l
 
 /**
  *  @brief Finalizes 4x integer dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_i8x16_update_loongsonasx, nk_dot_u8x16_update_loongsonasx
+ *  @sa nk_dot_i8x32_update_loongsonasx, nk_dot_u8x32_update_loongsonasx
  */
 NK_INTERNAL void nk_dot_through_i32_finalize_loongsonasx_(                                                          //
     nk_dot_through_i32_state_loongsonasx_t_ const *state_a, nk_dot_through_i32_state_loongsonasx_t_ const *state_b, //
@@ -474,7 +474,7 @@ NK_INTERNAL void nk_dot_through_f32_init_loongsonasx_(nk_dot_through_f32_state_l
 
 /**
  *  @brief Fuses 32-bit multiplication and accumulation for pre-converted f32 vectors.
- *  @sa nk_dot_bf16x8_update_loongsonasx
+ *  @sa nk_dot_bf16x16_update_loongsonasx
  */
 NK_INTERNAL void nk_dot_through_f32_update_loongsonasx_(nk_dot_through_f32_state_loongsonasx_t_ *state, nk_b256_vec_t a,
                                                         nk_b256_vec_t b, nk_size_t depth_offset,
@@ -486,7 +486,7 @@ NK_INTERNAL void nk_dot_through_f32_update_loongsonasx_(nk_dot_through_f32_state
 
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
- *  @sa nk_dot_bf16x8_finalize_loongsonasx
+ *  @sa nk_dot_bf16x16_finalize_loongsonasx
  *
  *  Computes 4x horizontal reductions, each involving 8x floats, using LASX interleave instructions.
  */

package/numkong/include/numkong/dot/neon.h

@@ -89,7 +89,8 @@
 #if NK_TARGET_ARM64_
 #if NK_TARGET_NEON
 
-#include "numkong/cast/neon.h" // `nk_e4m3x8_to_f16x8_neon_`
+#include "numkong/cast/neon.h"  // `nk_e4m3x8_to_f16x8_neon_`
+#include "numkong/dot/serial.h" // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/neonbfdot.h

@@ -63,6 +63,7 @@
 #include "numkong/types.h"
 #include "numkong/cast/serial.h" // `nk_partial_load_b8x8_serial_`
 #include "numkong/cast/neon.h"   // `nk_e4m3x8_to_bf16x8_neon_`
+#include "numkong/dot/serial.h"  // `nk_dot_bf16c_serial`, `nk_vdot_bf16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/neonfhm.h

@@ -65,6 +65,7 @@
 #include "numkong/types.h"
 #include "numkong/cast/serial.h" // `nk_partial_load_b8x8_serial_`
 #include "numkong/cast/neon.h"   // `nk_e4m3x8_to_f16x8_neon_`
+#include "numkong/dot/serial.h"  // `nk_dot_f16c_serial`, `nk_vdot_f16c_serial`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/neonsdot.h

@@ -100,6 +100,7 @@
 #if NK_TARGET_NEONSDOT
 
 #include "numkong/types.h"
+#include "numkong/cast/serial.h" // `nk_partial_load_b8x16_serial_`
 
 #if defined(__cplusplus)
 extern "C" {

package/numkong/include/numkong/dot/powervsx.h

@@ -599,7 +599,7 @@ NK_INTERNAL void nk_dot_i8x16_finalize_powervsx(
     nk_dot_i8x16_state_powervsx_t const *state_a, nk_dot_i8x16_state_powervsx_t const *state_b, //
     nk_dot_i8x16_state_powervsx_t const *state_c, nk_dot_i8x16_state_powervsx_t const *state_d, //
     nk_size_t total_dimensions,                                                                 //
-    nk_i32_t a_sum, nk_b128_vec_t b_sums, nk_b128_vec_t *result) {
+    nk_i32_t a_sum, nk_b128_vec_t const *b_sums_vec, nk_b128_vec_t *result_vec) {
     nk_unused_(total_dimensions);
     nk_unused_(a_sum);
 
@@ -624,8 +624,8 @@ NK_INTERNAL void nk_dot_i8x16_finalize_powervsx(
     // Correction: VMSUMMBM(b, a⊕0x80) = Σ(b_i·(a_i+128)) = a·b + 128·Σb
     // So a·b = biased − 128·Σb. B column sums are precomputed during packing.
     nk_vu32x4_t shift_u32x4 = vec_splats((nk_u32_t)7);
-    nk_vi32x4_t correction_i32x4 = (nk_vi32x4_t)vec_sl((nk_vu32x4_t)b_sums.vi32x4, shift_u32x4);
-    result->vi32x4 = vec_sub(biased_i32x4, correction_i32x4);
+    nk_vi32x4_t correction_i32x4 = (nk_vi32x4_t)vec_sl((nk_vu32x4_t)b_sums_vec->vi32x4, shift_u32x4);
+    result_vec->vi32x4 = vec_sub(biased_i32x4, correction_i32x4);
 }
 
 /** @brief Running state for i8 column sum precomputation on Power VSX. */