usearch 2.21.4 → 2.25.1

package/numkong/include/numkong/each/sapphire.h

@@ -0,0 +1,343 @@
+/**
+ *  @brief SIMD-accelerated Elementwise Arithmetic for Sapphire Rapids.
+ *  @file include/numkong/each/sapphire.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/each.h
+ *
+ *  @section sapphire_elementwise_instructions Relevant Instructions
+ *
+ *      Intrinsic                 Instruction                Sapphire   Genoa
+ *      _mm512_add_ph             VADDPH (ZMM, ZMM, ZMM)     4cy @ p05  3cy @ p01
+ *      _mm512_mul_ph             VMULPH (ZMM, ZMM, ZMM)     4cy @ p05  3cy @ p01
+ *      _mm512_fmadd_ph           VFMADD (ZMM, ZMM, ZMM)     4cy @ p05  4cy @ p01
+ *      _mm512_cvtepi16_ph        VCVTW2PH (ZMM, ZMM)        4cy @ p05  4cy @ p01
+ *      _mm512_cvtph_epi16        VCVTPH2W (ZMM, ZMM)        4cy @ p05  4cy @ p01
+ *      _mm512_cvtepi8_epi16      VPMOVSXBW (ZMM, YMM)       3cy @ p5   3cy @ p12
+ *      _mm512_cvtsepi16_epi8     VPMOVSWB (YMM, ZMM)        4cy @ p5   4cy @ p12
+ *      _mm512_packus_epi16       VPACKUSWB (ZMM, ZMM, ZMM)  1cy @ p5   1cy @ p12
+ *      _mm256_add_ph             VADDPH (YMM, YMM, YMM)     4cy @ p05  3cy @ p01
+ *      _mm512_maskz_loadu_epi16  VMOVDQU16 (ZMM {K}, M512)  7cy @ p23  7cy @ p23
+ *      _mm512_mask_storeu_epi16  VMOVDQU16 (M512 {K}, ZMM)  4cy @ p4   4cy @ p4
+ */
+#ifndef NK_EACH_SAPPHIRE_H
+#define NK_EACH_SAPPHIRE_H
+
+#if NK_TARGET_X8664_
+#if NK_TARGET_SAPPHIRE
+
+#include "numkong/types.h"
+#include "numkong/cast/sapphire.h" // `nk_f32_to_f16_sapphire`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512fp16,f16c,fma,bmi,bmi2"))), \
+                             apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512fp16", "f16c", "fma", "bmi", "bmi2")
+#endif
+
+NK_PUBLIC void nk_each_sum_f16_sapphire(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f16_t *result) {
+    __mmask32 mask = 0xFFFFFFFF;
+    __m512h a_f16_vec, b_f16_vec;
+    __m512h sum_f16_vec;
+nk_each_sum_f16_sapphire_cycle:
+    if (n < 32) {
+        mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
+        b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
+        n = 0;
+    }
+    else {
+        a_f16_vec = _mm512_castsi512_ph(_mm512_loadu_epi16(a));
+        b_f16_vec = _mm512_castsi512_ph(_mm512_loadu_epi16(b));
+        a += 32, b += 32, n -= 32;
+    }
+    sum_f16_vec = _mm512_add_ph(a_f16_vec, b_f16_vec);
+    _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
+    result += 32;
+    if (n) goto nk_each_sum_f16_sapphire_cycle;
+}
+
+NK_PUBLIC void nk_each_scale_u8_sapphire(nk_u8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
+                                         nk_u8_t *result) {
+    short alpha_short, beta_short;
+    nk_f32_to_f16_sapphire(alpha, (nk_f16_t *)&alpha_short);
+    nk_f32_to_f16_sapphire(beta, (nk_f16_t *)&beta_short);
+    __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
+    __m512h alpha_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(alpha_short));
+    __m512h beta_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(beta_short));
+    __m512i a_u8x64, result_u8x64;
+    __m512h a_low_f16x32, a_high_f16x32;
+    __m512h result_low_f16x32, result_high_f16x32;
+    __m512i result_low_i16x32, result_high_i16x32;
+nk_each_scale_u8_sapphire_cycle:
+    if (n < 64) {
+        mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
+        a_u8x64 = _mm512_maskz_loadu_epi8(mask, a);
+        n = 0;
+    }
+    else {
+        a_u8x64 = _mm512_loadu_epi8(a);
+        a += 64, n -= 64;
+    }
+    // Upcast:
+    a_low_f16x32 = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8x64, _mm512_setzero_si512()));
+    a_high_f16x32 = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8x64, _mm512_setzero_si512()));
+    // Scale:
+    result_low_f16x32 = _mm512_fmadd_ph(a_low_f16x32, alpha_f16x32, beta_f16x32);
+    result_high_f16x32 = _mm512_fmadd_ph(a_high_f16x32, alpha_f16x32, beta_f16x32);
+    // Downcast:
+    result_low_i16x32 = _mm512_cvtph_epi16(result_low_f16x32);
+    result_high_i16x32 = _mm512_cvtph_epi16(result_high_f16x32);
+    result_u8x64 = _mm512_packus_epi16(result_low_i16x32, result_high_i16x32);
+    _mm512_mask_storeu_epi8(result, mask, result_u8x64);
+    result += 64;
+    if (n) goto nk_each_scale_u8_sapphire_cycle;
+}
+
+NK_PUBLIC void nk_each_blend_u8_sapphire(            //
+    nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, //
+    nk_f32_t const *alpha, nk_f32_t const *beta, nk_u8_t *result) {
+
+    nk_f32_t alpha_val = *alpha;
+    nk_f32_t beta_val = *beta;
+
+    // There are several special cases we may want to implement:
+    // 1. Simple addition, when both weights are equal to 1.0.
+    if (alpha_val == 1 && beta_val == 1) {
+        // In this case we can avoid expensive multiplications.
+        nk_each_sum_u8_icelake(a, b, n, result);
+        return;
+    }
+    // 2. Just scaling, when one of the weights is equal to zero.
+    else if (alpha_val == 0 || beta_val == 0) {
+        // In this case we can avoid half of the load instructions.
+        nk_f32_t zero = 0;
+        if (beta_val == 0) { nk_each_scale_u8_sapphire(a, n, alpha, &zero, result); }
+        else { nk_each_scale_u8_sapphire(b, n, beta, &zero, result); }
+        return;
+    }
+
+    // The general case.
+    short alpha_short, beta_short;
+    nk_f32_to_f16_sapphire(&alpha_val, (nk_f16_t *)&alpha_short);
+    nk_f32_to_f16_sapphire(&beta_val, (nk_f16_t *)&beta_short);
+    __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
+    __m512h alpha_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(alpha_short));
+    __m512h beta_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(beta_short));
+    __m512i a_u8x64, b_u8x64, result_u8x64;
+    __m512h a_low_f16x32, a_high_f16x32, b_low_f16x32, b_high_f16x32;
+    __m512h a_scaled_low_f16x32, a_scaled_high_f16x32, result_low_f16x32, result_high_f16x32;
+    __m512i result_low_i16x32, result_high_i16x32;
+nk_each_blend_u8_sapphire_cycle:
+    if (n < 64) {
+        mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
+        a_u8x64 = _mm512_maskz_loadu_epi8(mask, a);
+        b_u8x64 = _mm512_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_u8x64 = _mm512_loadu_epi8(a);
+        b_u8x64 = _mm512_loadu_epi8(b);
+        a += 64, b += 64, n -= 64;
+    }
+    // Upcast:
+    a_low_f16x32 = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8x64, _mm512_setzero_si512()));
+    a_high_f16x32 = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8x64, _mm512_setzero_si512()));
+    b_low_f16x32 = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8x64, _mm512_setzero_si512()));
+    b_high_f16x32 = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8x64, _mm512_setzero_si512()));
+    // Scale:
+    a_scaled_low_f16x32 = _mm512_mul_ph(a_low_f16x32, alpha_f16x32);
+    a_scaled_high_f16x32 = _mm512_mul_ph(a_high_f16x32, alpha_f16x32);
+    // Add:
+    result_low_f16x32 = _mm512_fmadd_ph(b_low_f16x32, beta_f16x32, a_scaled_low_f16x32);
+    result_high_f16x32 = _mm512_fmadd_ph(b_high_f16x32, beta_f16x32, a_scaled_high_f16x32);
+    // Downcast:
+    result_low_i16x32 = _mm512_cvtph_epi16(result_low_f16x32);
+    result_high_i16x32 = _mm512_cvtph_epi16(result_high_f16x32);
+    result_u8x64 = _mm512_packus_epi16(result_low_i16x32, result_high_i16x32);
+    _mm512_mask_storeu_epi8(result, mask, result_u8x64);
+    result += 64;
+    if (n) goto nk_each_blend_u8_sapphire_cycle;
+}
+
+NK_PUBLIC void nk_each_scale_i8_sapphire(nk_i8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
+                                         nk_i8_t *result) {
+    short alpha_short, beta_short;
+    nk_f32_to_f16_sapphire(alpha, (nk_f16_t *)&alpha_short);
+    nk_f32_to_f16_sapphire(beta, (nk_f16_t *)&beta_short);
+    __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
+    __m512h alpha_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(alpha_short));
+    __m512h beta_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(beta_short));
+    __m256i a_low_i8x32, a_high_i8x32;
+    __m512i result_i8x64;
+    __m512h a_low_f16x32, a_high_f16x32;
+    __m512h result_low_f16x32, result_high_f16x32;
+    __m512i result_low_i16x32, result_high_i16x32;
+nk_each_scale_i8_sapphire_cycle:
+    if (n < 64) {
+        // Tail: use masked 512-bit load and extract (runs once)
+        mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
+        __m512i a_i8x64 = _mm512_maskz_loadu_epi8(mask, a);
+        a_low_i8x32 = _mm512_castsi512_si256(a_i8x64);
+        a_high_i8x32 = _mm512_extracti64x4_epi64(a_i8x64, 1);
+        n = 0;
+    }
+    else {
+        // Hot path: 2×256-bit loads to avoid VEXTRACTI64X4 (Port 5)
+        a_low_i8x32 = _mm256_loadu_epi8(a);
+        a_high_i8x32 = _mm256_loadu_epi8(a + 32);
+        a += 64, n -= 64;
+    }
+    // Upcast from 256-bit halves:
+    a_low_f16x32 = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(a_low_i8x32));
+    a_high_f16x32 = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(a_high_i8x32));
+    // Scale:
+    result_low_f16x32 = _mm512_fmadd_ph(a_low_f16x32, alpha_f16x32, beta_f16x32);
+    result_high_f16x32 = _mm512_fmadd_ph(a_high_f16x32, alpha_f16x32, beta_f16x32);
+    // Downcast:
+    result_low_i16x32 = _mm512_cvtph_epi16(result_low_f16x32);
+    result_high_i16x32 = _mm512_cvtph_epi16(result_high_f16x32);
+    result_i8x64 = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(result_low_i16x32)),
+                                      _mm512_cvtsepi16_epi8(result_high_i16x32), 1);
+    _mm512_mask_storeu_epi8(result, mask, result_i8x64);
+    result += 64;
+    if (n) goto nk_each_scale_i8_sapphire_cycle;
+}
+
+NK_PUBLIC void nk_each_blend_i8_sapphire(            //
+    nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, //
+    nk_f32_t const *alpha, nk_f32_t const *beta, nk_i8_t *result) {
+
+    nk_f32_t alpha_val = *alpha;
+    nk_f32_t beta_val = *beta;
+
+    // There are several special cases we may want to implement:
+    // 1. Simple addition, when both weights are equal to 1.0.
+    if (alpha_val == 1 && beta_val == 1) {
+        // In this case we can avoid expensive multiplications.
+        nk_each_sum_i8_icelake(a, b, n, result);
+        return;
+    }
+    // 2. Just scaling, when one of the weights is equal to zero.
+    else if (alpha_val == 0 || beta_val == 0) {
+        // In this case we can avoid half of the load instructions.
+        nk_f32_t zero = 0;
+        if (beta_val == 0) { nk_each_scale_i8_sapphire(a, n, alpha, &zero, result); }
+        else { nk_each_scale_i8_sapphire(b, n, beta, &zero, result); }
+        return;
+    }
+
+    // The general case.
+    short alpha_short, beta_short;
+    nk_f32_to_f16_sapphire(&alpha_val, (nk_f16_t *)&alpha_short);
+    nk_f32_to_f16_sapphire(&beta_val, (nk_f16_t *)&beta_short);
+    __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
+    __m512h alpha_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(alpha_short));
+    __m512h beta_f16x32 = _mm512_castsi512_ph(_mm512_set1_epi16(beta_short));
+    __m256i a_low_i8x32, a_high_i8x32, b_low_i8x32, b_high_i8x32;
+    __m512i result_i8x64;
+    __m512h a_low_f16x32, a_high_f16x32, b_low_f16x32, b_high_f16x32;
+    __m512h a_scaled_low_f16x32, a_scaled_high_f16x32, result_low_f16x32, result_high_f16x32;
+    __m512i result_low_i16x32, result_high_i16x32;
+nk_each_blend_i8_sapphire_cycle:
+    if (n < 64) {
+        // Tail: use masked 512-bit loads and extract (runs once)
+        mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
+        __m512i a_i8x64 = _mm512_maskz_loadu_epi8(mask, a);
+        __m512i b_i8x64 = _mm512_maskz_loadu_epi8(mask, b);
+        a_low_i8x32 = _mm512_castsi512_si256(a_i8x64);
+        a_high_i8x32 = _mm512_extracti64x4_epi64(a_i8x64, 1);
+        b_low_i8x32 = _mm512_castsi512_si256(b_i8x64);
+        b_high_i8x32 = _mm512_extracti64x4_epi64(b_i8x64, 1);
+        n = 0;
+    }
+    else {
+        // Hot path: 2×256-bit loads per vector to avoid VEXTRACTI64X4 (Port 5)
+        a_low_i8x32 = _mm256_loadu_epi8(a);
+        a_high_i8x32 = _mm256_loadu_epi8(a + 32);
+        b_low_i8x32 = _mm256_loadu_epi8(b);
+        b_high_i8x32 = _mm256_loadu_epi8(b + 32);
+        a += 64, b += 64, n -= 64;
+    }
+    // Upcast from 256-bit halves:
+    a_low_f16x32 = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(a_low_i8x32));
+    a_high_f16x32 = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(a_high_i8x32));
+    b_low_f16x32 = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(b_low_i8x32));
+    b_high_f16x32 = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(b_high_i8x32));
+    // Scale:
+    a_scaled_low_f16x32 = _mm512_mul_ph(a_low_f16x32, alpha_f16x32);
+    a_scaled_high_f16x32 = _mm512_mul_ph(a_high_f16x32, alpha_f16x32);
+    // Add:
+    result_low_f16x32 = _mm512_fmadd_ph(b_low_f16x32, beta_f16x32, a_scaled_low_f16x32);
+    result_high_f16x32 = _mm512_fmadd_ph(b_high_f16x32, beta_f16x32, a_scaled_high_f16x32);
+    // Downcast:
+    result_low_i16x32 = _mm512_cvtph_epi16(result_low_f16x32);
+    result_high_i16x32 = _mm512_cvtph_epi16(result_high_f16x32);
+    result_i8x64 = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(result_low_i16x32)),
+                                      _mm512_cvtsepi16_epi8(result_high_i16x32), 1);
+    _mm512_mask_storeu_epi8(result, mask, result_i8x64);
+    result += 64;
+    if (n) goto nk_each_blend_i8_sapphire_cycle;
+}
+
+NK_PUBLIC void nk_each_sum_e4m3_sapphire(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_e4m3_t *result) {
+    __m256i a_e4m3x32, b_e4m3x32;
+    __m256h a_low_f16x16, a_high_f16x16, b_low_f16x16, b_high_f16x16;
+    __m256h sum_low_f16x16, sum_high_f16x16;
+    __m128i result_low_e4m3x16, result_high_e4m3x16;
+    __mmask32 mask = 0xFFFFFFFF;
+nk_each_sum_e4m3_sapphire_cycle:
+    if (n < 32) {
+        mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, (unsigned int)n);
+        a_e4m3x32 = _mm256_maskz_loadu_epi8(mask, a);
+        b_e4m3x32 = _mm256_maskz_loadu_epi8(mask, b);
+        n = 0;
+    }
+    else {
+        a_e4m3x32 = _mm256_loadu_si256((__m256i const *)a);
+        b_e4m3x32 = _mm256_loadu_si256((__m256i const *)b);
+        a += 32, b += 32, n -= 32;
+    }
+
+    // Convert e4m3x16 → f16x16 (two halves)
+    a_low_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(_mm256_castsi256_si128(a_e4m3x32));
+    a_high_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(_mm256_extracti128_si256(a_e4m3x32, 1));
+    b_low_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(_mm256_castsi256_si128(b_e4m3x32));
+    b_high_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(_mm256_extracti128_si256(b_e4m3x32, 1));
+
+    // Add in F16 - e4m3 sum is safe (max 896 < 65504)
+    sum_low_f16x16 = _mm256_add_ph(a_low_f16x16, b_low_f16x16);
+    sum_high_f16x16 = _mm256_add_ph(a_high_f16x16, b_high_f16x16);
+
+    // Convert f16x16 → e4m3x16
+    result_low_e4m3x16 = nk_f16x16_to_e4m3x16_sapphire_(sum_low_f16x16);
+    result_high_e4m3x16 = nk_f16x16_to_e4m3x16_sapphire_(sum_high_f16x16);
+
+    // Pack and store
+    __m256i result_e4m3x32 = _mm256_inserti128_si256(_mm256_castsi128_si256(result_low_e4m3x16), result_high_e4m3x16,
+                                                     1);
+    _mm256_mask_storeu_epi8(result, mask, result_e4m3x32);
+    result += 32;
+    if (n) goto nk_each_sum_e4m3_sapphire_cycle;
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SAPPHIRE
+#endif // NK_TARGET_X8664_
+#endif // NK_EACH_SAPPHIRE_H

package/numkong/include/numkong/each/serial.h

@@ -0,0 +1,282 @@
+/**
+ *  @brief SWAR-accelerated Elementwise Arithmetic for SIMD-free CPUs.
+ *  @file include/numkong/each/serial.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/each.h
+ */
+#ifndef NK_EACH_SERIAL_H
+#define NK_EACH_SERIAL_H
+
+#include "numkong/types.h"
+#include "numkong/cast/serial.h" // `nk_f16_to_f32_serial`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define nk_define_each_scale_(input_type, accumulator_type, load_and_convert, convert_and_store) \
+    NK_PUBLIC void nk_each_scale_##input_type##_serial(                                          \
+        nk_##input_type##_t const *a, nk_size_t n, nk_##accumulator_type##_t const *alpha,       \
+        nk_##accumulator_type##_t const *beta, nk_##input_type##_t *result) {                    \
+        nk_##accumulator_type##_t alpha_val = *alpha;                                            \
+        nk_##accumulator_type##_t beta_val = *beta;                                              \
+        nk_##accumulator_type##_t ai, sum;                                                       \
+        for (nk_size_t i = 0; i != n; ++i) {                                                     \
+            load_and_convert(a + i, &ai);                                                        \
+            sum = (nk_##accumulator_type##_t)(alpha_val * ai + beta_val);                        \
+            convert_and_store(&sum, result + i);                                                 \
+        }                                                                                        \
+    }
+#define nk_define_each_sum_(input_type, accumulator_type, load_and_convert, convert_and_store)                   \
+    NK_PUBLIC void nk_each_sum_##input_type##_serial(nk_##input_type##_t const *a, nk_##input_type##_t const *b, \
+                                                     nk_size_t n, nk_##input_type##_t *result) {                 \
+        nk_##accumulator_type##_t ai, bi, sum;                                                                   \
+        for (nk_size_t i = 0; i != n; ++i) {                                                                     \
+            load_and_convert(a + i, &ai);                                                                        \
+            load_and_convert(b + i, &bi);                                                                        \
+            sum = ai + bi;                                                                                       \
+            convert_and_store(&sum, result + i);                                                                 \
+        }                                                                                                        \
+    }
+
+#define nk_define_each_blend_(input_type, accumulator_type, load_and_convert, convert_and_store)                      \
+    NK_PUBLIC void nk_each_blend_##input_type##_serial(                                                               \
+        nk_##input_type##_t const *a, nk_##input_type##_t const *b, nk_size_t n,                                      \
+        nk_##accumulator_type##_t const *alpha, nk_##accumulator_type##_t const *beta, nk_##input_type##_t *result) { \
+        nk_##accumulator_type##_t alpha_val = *alpha;                                                                 \
+        nk_##accumulator_type##_t beta_val = *beta;                                                                   \
+        nk_##accumulator_type##_t ai, bi, ai_scaled, bi_scaled, sum;                                                  \
+        for (nk_size_t i = 0; i != n; ++i) {                                                                          \
+            load_and_convert(a + i, &ai);                                                                             \
+            load_and_convert(b + i, &bi);                                                                             \
+            ai_scaled = ai * alpha_val;                                                                               \
+            bi_scaled = bi * beta_val;                                                                                \
+            sum = ai_scaled + bi_scaled;                                                                              \
+            convert_and_store(&sum, result + i);                                                                      \
+        }                                                                                                             \
+    }
+
+#define nk_define_each_fma_(input_type, accumulator_type, load_and_convert, convert_and_store)                        \
+    NK_PUBLIC void nk_each_fma_##input_type##_serial(                                                                 \
+        nk_##input_type##_t const *a, nk_##input_type##_t const *b, nk_##input_type##_t const *c, nk_size_t n,        \
+        nk_##accumulator_type##_t const *alpha, nk_##accumulator_type##_t const *beta, nk_##input_type##_t *result) { \
+        nk_##accumulator_type##_t alpha_val = *alpha;                                                                 \
+        nk_##accumulator_type##_t beta_val = *beta;                                                                   \
+        nk_##accumulator_type##_t ai, bi, ci, abi_scaled, ci_scaled, sum;                                             \
+        for (nk_size_t i = 0; i != n; ++i) {                                                                          \
+            load_and_convert(a + i, &ai);                                                                             \
+            load_and_convert(b + i, &bi);                                                                             \
+            load_and_convert(c + i, &ci);                                                                             \
+            abi_scaled = ai * bi * alpha_val;                                                                         \
+            ci_scaled = ci * beta_val;                                                                                \
+            sum = abi_scaled + ci_scaled;                                                                             \
+            convert_and_store(&sum, result + i);                                                                      \
+        }                                                                                                             \
+    }
+
+/* Optimize serial fallbacks for size — see dots/serial.h for rationale. */
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("s", on)
+#elif defined(__clang__)
+#pragma clang attribute push(__attribute__((minsize)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("Os")
+#endif
+#endif
+
+nk_define_each_sum_(f64, f64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_sum_f64_serial
+nk_define_each_sum_(f32, f32, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_sum_f32_serial
+nk_define_each_sum_(f16, f32, nk_f16_to_f32_serial, nk_f32_to_f16_serial)    // nk_each_sum_f16_serial
+nk_define_each_sum_(bf16, f32, nk_bf16_to_f32_serial, nk_f32_to_bf16_serial) // nk_each_sum_bf16_serial
+nk_define_each_sum_(e4m3, f32, nk_e4m3_to_f32_serial, nk_f32_to_e4m3_serial) // nk_each_sum_e4m3_serial
+nk_define_each_sum_(e5m2, f32, nk_e5m2_to_f32_serial, nk_f32_to_e5m2_serial) // nk_each_sum_e5m2_serial
+nk_define_each_sum_(e2m3, f32, nk_e2m3_to_f32_serial, nk_f32_to_e2m3_serial) // nk_each_sum_e2m3_serial
+nk_define_each_sum_(e3m2, f32, nk_e3m2_to_f32_serial, nk_f32_to_e3m2_serial) // nk_each_sum_e3m2_serial
+nk_define_each_sum_(i8, i64, nk_assign_from_to_, nk_i64_to_i8_serial)        // nk_each_sum_i8_serial
+nk_define_each_sum_(u8, i64, nk_assign_from_to_, nk_i64_to_u8_serial)        // nk_each_sum_u8_serial
+nk_define_each_sum_(i16, i64, nk_assign_from_to_, nk_i64_to_i16_serial)      // nk_each_sum_i16_serial
+nk_define_each_sum_(u16, i64, nk_assign_from_to_, nk_i64_to_u16_serial)      // nk_each_sum_u16_serial
+nk_define_each_sum_(i32, i64, nk_assign_from_to_, nk_i64_to_i32_serial)      // nk_each_sum_i32_serial
+nk_define_each_sum_(u32, i64, nk_assign_from_to_, nk_i64_to_u32_serial)      // nk_each_sum_u32_serial
+nk_define_each_sum_(i64, i64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_sum_i64_serial
+nk_define_each_sum_(u64, u64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_sum_u64_serial
+
+nk_define_each_scale_(f64, f64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_scale_f64_serial
+nk_define_each_scale_(f32, f32, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_scale_f32_serial
+nk_define_each_scale_(f16, f32, nk_f16_to_f32_serial, nk_f32_to_f16_serial)    // nk_each_scale_f16_serial
+nk_define_each_scale_(bf16, f32, nk_bf16_to_f32_serial, nk_f32_to_bf16_serial) // nk_each_scale_bf16_serial
+nk_define_each_scale_(e4m3, f32, nk_e4m3_to_f32_serial, nk_f32_to_e4m3_serial) // nk_each_scale_e4m3_serial
+nk_define_each_scale_(e5m2, f32, nk_e5m2_to_f32_serial, nk_f32_to_e5m2_serial) // nk_each_scale_e5m2_serial
+nk_define_each_scale_(e2m3, f32, nk_e2m3_to_f32_serial, nk_f32_to_e2m3_serial) // nk_each_scale_e2m3_serial
+nk_define_each_scale_(e3m2, f32, nk_e3m2_to_f32_serial, nk_f32_to_e3m2_serial) // nk_each_scale_e3m2_serial
+nk_define_each_scale_(i8, f32, nk_assign_from_to_, nk_f32_to_i8_serial)        // nk_each_scale_i8_serial
+nk_define_each_scale_(u8, f32, nk_assign_from_to_, nk_f32_to_u8_serial)        // nk_each_scale_u8_serial
+nk_define_each_scale_(i16, f32, nk_assign_from_to_, nk_f32_to_i16_serial)      // nk_each_scale_i16_serial
+nk_define_each_scale_(u16, f32, nk_assign_from_to_, nk_f32_to_u16_serial)      // nk_each_scale_u16_serial
+nk_define_each_scale_(i32, f64, nk_assign_from_to_, nk_f64_to_i32_serial)      // nk_each_scale_i32_serial
+nk_define_each_scale_(u32, f64, nk_assign_from_to_, nk_f64_to_u32_serial)      // nk_each_scale_u32_serial
+nk_define_each_scale_(i64, f64, nk_f64_from_i64_, nk_f64_to_i64_serial)        // nk_each_scale_i64_serial
+nk_define_each_scale_(u64, f64, nk_f64_from_u64_, nk_f64_to_u64_serial)        // nk_each_scale_u64_serial
+
+nk_define_each_blend_(f64, f64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_blend_f64_serial
+nk_define_each_blend_(f32, f32, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_blend_f32_serial
+nk_define_each_blend_(f16, f32, nk_f16_to_f32_serial, nk_f32_to_f16_serial)    // nk_each_blend_f16_serial
+nk_define_each_blend_(bf16, f32, nk_bf16_to_f32_serial, nk_f32_to_bf16_serial) // nk_each_blend_bf16_serial
+nk_define_each_blend_(e4m3, f32, nk_e4m3_to_f32_serial, nk_f32_to_e4m3_serial) // nk_each_blend_e4m3_serial
+nk_define_each_blend_(e5m2, f32, nk_e5m2_to_f32_serial, nk_f32_to_e5m2_serial) // nk_each_blend_e5m2_serial
+nk_define_each_blend_(e2m3, f32, nk_e2m3_to_f32_serial, nk_f32_to_e2m3_serial) // nk_each_blend_e2m3_serial
+nk_define_each_blend_(e3m2, f32, nk_e3m2_to_f32_serial, nk_f32_to_e3m2_serial) // nk_each_blend_e3m2_serial
+nk_define_each_blend_(i8, f32, nk_assign_from_to_, nk_f32_to_i8_serial)        // nk_each_blend_i8_serial
+nk_define_each_blend_(u8, f32, nk_assign_from_to_, nk_f32_to_u8_serial)        // nk_each_blend_u8_serial
+nk_define_each_blend_(i16, f32, nk_assign_from_to_, nk_f32_to_i16_serial)      // nk_each_blend_i16_serial
+nk_define_each_blend_(u16, f32, nk_assign_from_to_, nk_f32_to_u16_serial)      // nk_each_blend_u16_serial
+nk_define_each_blend_(i32, f64, nk_assign_from_to_, nk_f64_to_i32_serial)      // nk_each_blend_i32_serial
+nk_define_each_blend_(u32, f64, nk_assign_from_to_, nk_f64_to_u32_serial)      // nk_each_blend_u32_serial
+nk_define_each_blend_(i64, f64, nk_f64_from_i64_, nk_f64_to_i64_serial)        // nk_each_blend_i64_serial
+nk_define_each_blend_(u64, f64, nk_f64_from_u64_, nk_f64_to_u64_serial)        // nk_each_blend_u64_serial
+
+nk_define_each_fma_(f64, f64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_fma_f64_serial
+nk_define_each_fma_(f32, f32, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_fma_f32_serial
+nk_define_each_fma_(f16, f32, nk_f16_to_f32_serial, nk_f32_to_f16_serial)    // nk_each_fma_f16_serial
+nk_define_each_fma_(bf16, f32, nk_bf16_to_f32_serial, nk_f32_to_bf16_serial) // nk_each_fma_bf16_serial
+nk_define_each_fma_(e4m3, f32, nk_e4m3_to_f32_serial, nk_f32_to_e4m3_serial) // nk_each_fma_e4m3_serial
+nk_define_each_fma_(e5m2, f32, nk_e5m2_to_f32_serial, nk_f32_to_e5m2_serial) // nk_each_fma_e5m2_serial
+nk_define_each_fma_(e2m3, f32, nk_e2m3_to_f32_serial, nk_f32_to_e2m3_serial) // nk_each_fma_e2m3_serial
+nk_define_each_fma_(e3m2, f32, nk_e3m2_to_f32_serial, nk_f32_to_e3m2_serial) // nk_each_fma_e3m2_serial
+nk_define_each_fma_(i8, f32, nk_assign_from_to_, nk_f32_to_i8_serial)        // nk_each_fma_i8_serial
+nk_define_each_fma_(u8, f32, nk_assign_from_to_, nk_f32_to_u8_serial)        // nk_each_fma_u8_serial
+nk_define_each_fma_(i16, f32, nk_assign_from_to_, nk_f32_to_i16_serial)      // nk_each_fma_i16_serial
+nk_define_each_fma_(u16, f32, nk_assign_from_to_, nk_f32_to_u16_serial)      // nk_each_fma_u16_serial
+nk_define_each_fma_(i32, f64, nk_assign_from_to_, nk_f64_to_i32_serial)      // nk_each_fma_i32_serial
+nk_define_each_fma_(u32, f64, nk_assign_from_to_, nk_f64_to_u32_serial)      // nk_each_fma_u32_serial
+nk_define_each_fma_(i64, f64, nk_f64_from_i64_, nk_f64_to_i64_serial)        // nk_each_fma_i64_serial
+nk_define_each_fma_(u64, f64, nk_f64_from_u64_, nk_f64_to_u64_serial)        // nk_each_fma_u64_serial
+
+#undef nk_define_each_scale_
+#undef nk_define_each_sum_
+#undef nk_define_each_blend_
+#undef nk_define_each_fma_
+
+NK_DYNAMIC void nk_each_sum_f32(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *result);
+
+NK_PUBLIC void nk_each_sum_f32c_serial(nk_f32c_t const *a, nk_f32c_t const *b, nk_size_t n, nk_f32c_t *result) {
+    nk_each_sum_f32((nk_f32_t const *)a, (nk_f32_t const *)b, 2 * n, (nk_f32_t *)result);
+}
+
+NK_PUBLIC void nk_each_sum_f64c_serial(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n, nk_f64c_t *result) {
+    nk_each_sum_f64((nk_f64_t const *)a, (nk_f64_t const *)b, 2 * n, (nk_f64_t *)result);
+}
+
+NK_DYNAMIC void nk_each_sum_f64(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result);
+
+NK_PUBLIC void nk_each_scale_f32c_serial(nk_f32c_t const *a, nk_size_t n, nk_f32c_t const *alpha, nk_f32c_t const *beta,
+                                         nk_f32c_t *result) {
+    nk_f32_t alpha_real = alpha->real, alpha_imag = alpha->imag;
+    nk_f32_t beta_real = beta->real, beta_imag = beta->imag;
+    for (nk_size_t i = 0; i != n; ++i) {
+        nk_f32_t a_real = a[i].real, a_imag = a[i].imag;
+        result[i].real = alpha_real * a_real - alpha_imag * a_imag + beta_real;
+        result[i].imag = alpha_real * a_imag + alpha_imag * a_real + beta_imag;
+    }
+}
+
+NK_PUBLIC void nk_each_scale_f64c_serial(nk_f64c_t const *a, nk_size_t n, nk_f64c_t const *alpha, nk_f64c_t const *beta,
+                                         nk_f64c_t *result) {
+    nk_f64_t alpha_real = alpha->real, alpha_imag = alpha->imag;
+    nk_f64_t beta_real = beta->real, beta_imag = beta->imag;
+    for (nk_size_t i = 0; i != n; ++i) {
+        nk_f64_t a_real = a[i].real, a_imag = a[i].imag;
+        result[i].real = alpha_real * a_real - alpha_imag * a_imag + beta_real;
+        result[i].imag = alpha_real * a_imag + alpha_imag * a_real + beta_imag;
+    }
+}
+
+NK_PUBLIC void nk_each_blend_f32c_serial(nk_f32c_t const *a, nk_f32c_t const *b, nk_size_t n, nk_f32c_t const *alpha,
+                                         nk_f32c_t const *beta, nk_f32c_t *result) {
+    nk_f32_t alpha_real = alpha->real, alpha_imag = alpha->imag;
+    nk_f32_t beta_real = beta->real, beta_imag = beta->imag;
+    for (nk_size_t i = 0; i != n; ++i) {
+        nk_f32_t a_real = a[i].real, a_imag = a[i].imag;
+        nk_f32_t b_real = b[i].real, b_imag = b[i].imag;
+        nk_f32_t alpha_a_real = alpha_real * a_real - alpha_imag * a_imag;
+        nk_f32_t alpha_a_imag = alpha_real * a_imag + alpha_imag * a_real;
+        nk_f32_t beta_b_real = beta_real * b_real - beta_imag * b_imag;
+        nk_f32_t beta_b_imag = beta_real * b_imag + beta_imag * b_real;
+        result[i].real = alpha_a_real + beta_b_real;
+        result[i].imag = alpha_a_imag + beta_b_imag;
+    }
+}
+
+NK_PUBLIC void nk_each_blend_f64c_serial(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n, nk_f64c_t const *alpha,
+                                         nk_f64c_t const *beta, nk_f64c_t *result) {
+    nk_f64_t alpha_real = alpha->real, alpha_imag = alpha->imag;
+    nk_f64_t beta_real = beta->real, beta_imag = beta->imag;
+    for (nk_size_t i = 0; i != n; ++i) {
+        nk_f64_t a_real = a[i].real, a_imag = a[i].imag;
+        nk_f64_t b_real = b[i].real, b_imag = b[i].imag;
+        nk_f64_t alpha_a_real = alpha_real * a_real - alpha_imag * a_imag;
+        nk_f64_t alpha_a_imag = alpha_real * a_imag + alpha_imag * a_real;
+        nk_f64_t beta_b_real = beta_real * b_real - beta_imag * b_imag;
+        nk_f64_t beta_b_imag = beta_real * b_imag + beta_imag * b_real;
+        result[i].real = alpha_a_real + beta_b_real;
+        result[i].imag = alpha_a_imag + beta_b_imag;
+    }
+}
+
+NK_PUBLIC void nk_each_fma_f32c_serial(nk_f32c_t const *a, nk_f32c_t const *b, nk_f32c_t const *c, nk_size_t n,
+                                       nk_f32c_t const *alpha, nk_f32c_t const *beta, nk_f32c_t *result) {
+    nk_f32_t alpha_real = alpha->real, alpha_imag = alpha->imag;
+    nk_f32_t beta_real = beta->real, beta_imag = beta->imag;
+    for (nk_size_t i = 0; i != n; ++i) {
+        nk_f32_t a_real = a[i].real, a_imag = a[i].imag;
+        nk_f32_t b_real = b[i].real, b_imag = b[i].imag;
+        nk_f32_t c_real = c[i].real, c_imag = c[i].imag;
+        nk_f32_t product_real = a_real * b_real - a_imag * b_imag;
+        nk_f32_t product_imag = a_real * b_imag + a_imag * b_real;
+        nk_f32_t alpha_product_real = alpha_real * product_real - alpha_imag * product_imag;
+        nk_f32_t alpha_product_imag = alpha_real * product_imag + alpha_imag * product_real;
+        nk_f32_t beta_c_real = beta_real * c_real - beta_imag * c_imag;
+        nk_f32_t beta_c_imag = beta_real * c_imag + beta_imag * c_real;
+        result[i].real = alpha_product_real + beta_c_real;
+        result[i].imag = alpha_product_imag + beta_c_imag;
+    }
+}
+
+NK_PUBLIC void nk_each_fma_f64c_serial(nk_f64c_t const *a, nk_f64c_t const *b, nk_f64c_t const *c, nk_size_t n,
+                                       nk_f64c_t const *alpha, nk_f64c_t const *beta, nk_f64c_t *result) {
+    nk_f64_t alpha_real = alpha->real, alpha_imag = alpha->imag;
+    nk_f64_t beta_real = beta->real, beta_imag = beta->imag;
+    for (nk_size_t i = 0; i != n; ++i) {
+        nk_f64_t a_real = a[i].real, a_imag = a[i].imag;
+        nk_f64_t b_real = b[i].real, b_imag = b[i].imag;
+        nk_f64_t c_real = c[i].real, c_imag = c[i].imag;
+        nk_f64_t product_real = a_real * b_real - a_imag * b_imag;
+        nk_f64_t product_imag = a_real * b_imag + a_imag * b_real;
+        nk_f64_t alpha_product_real = alpha_real * product_real - alpha_imag * product_imag;
+        nk_f64_t alpha_product_imag = alpha_real * product_imag + alpha_imag * product_real;
+        nk_f64_t beta_c_real = beta_real * c_real - beta_imag * c_imag;
+        nk_f64_t beta_c_imag = beta_real * c_imag + beta_imag * c_real;
+        result[i].real = alpha_product_real + beta_c_real;
+        result[i].imag = alpha_product_imag + beta_c_imag;
+    }
+}
+
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("", on)
+#elif defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_EACH_SERIAL_H