usearch 2.21.3 → 2.25.0

package/numkong/bench/bench_each.cpp

@@ -0,0 +1,503 @@
+/**
+ *  @brief Elementwise operations benchmarks (fma, blend, sum, scale).
+ *  @file bench/bench_each.cpp
+ *  @author Ash Vardanian
+ *  @date March 14, 2023
+ */
+
+#include <cstring> // std::memset
+
+#include "numkong/capabilities.h" // nk_kernel_kind_t
+#include "numkong/each.h"
+
+#include "bench.hpp"
+
+#if NK_COMPARE_TO_BLAS || NK_COMPARE_TO_MKL || NK_COMPARE_TO_ACCELERATE
+
+void sum_f32_with_blas(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *result) {
+    int const ni = static_cast<int>(n);
+    cblas_scopy(ni, a, 1, result, 1);
+    cblas_saxpy(ni, 1.0f, b, 1, result, 1);
+}
+
+void sum_f64_with_blas(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result) {
+    int const ni = static_cast<int>(n);
+    cblas_dcopy(ni, a, 1, result, 1);
+    cblas_daxpy(ni, 1.0, b, 1, result, 1);
+}
+
+void blend_f32_with_blas(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
+                         nk_f32_t *result) {
+    int const ni = static_cast<int>(n);
+    std::memset(result, 0, n * sizeof(nk_f32_t));
+    if (*alpha != 0) cblas_saxpy(ni, *alpha, a, 1, result, 1);
+    if (*beta != 0) cblas_saxpy(ni, *beta, b, 1, result, 1);
+}
+
+void blend_f64_with_blas(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
+                         nk_f64_t *result) {
+    int const ni = static_cast<int>(n);
+    std::memset(result, 0, n * sizeof(nk_f64_t));
+    if (*alpha != 0) cblas_daxpy(ni, *alpha, a, 1, result, 1);
+    if (*beta != 0) cblas_daxpy(ni, *beta, b, 1, result, 1);
+}
+
+#endif // NK_COMPARE_TO_BLAS || NK_COMPARE_TO_MKL || NK_COMPARE_TO_ACCELERATE
+
+/**
+ *  @brief Measures the performance of elementwise operations (sum, blend, fma, scale) using Google Benchmark.
+ *  @param state The benchmark state object provided by Google Benchmark.
+ *  @param kernel The kernel function to benchmark.
+ *  @param dimensions The number of dimensions in the vectors.
+ */
+template <nk_dtype_t input_dtype_, nk_kernel_kind_t kernel_kind_, nk_dtype_t alpha_dtype_, typename kernel_type_ = void>
+void measure_each(bm::State &state, kernel_type_ kernel, std::size_t dimensions) {
+
+    using input_t = typename nk::type_for<input_dtype_>::type;
+    using alpha_t = typename nk::type_for<alpha_dtype_>::type;
+    using input_vector_t = nk::vector<input_t>;
+
+    // Scaling parameters for FMA/blend/scale kernels
+    alpha_t alpha = alpha_t(0.2f);
+    alpha_t beta = alpha_t(0.3f);
+
+    // Preallocate vectors for different kernel types:
+    // - sum: input_a, input_c -> output
+    // - blend: input_a, input_c + alpha, beta -> output
+    // - fma: input_a, input_b, input_c + alpha, beta -> output
+    // - scale: input_a + alpha, beta -> output
+    std::size_t bytes_per_set = bench_dtype_bytes(input_dtype_, 4 * dimensions);
+    std::size_t const vectors_count = bench_input_count(bytes_per_set);
+    std::vector<input_vector_t> input_a(vectors_count), input_b(vectors_count);
+    std::vector<input_vector_t> input_c(vectors_count), output(vectors_count);
+    auto generator = make_random_engine();
+    for (std::size_t index = 0; index != vectors_count; ++index) {
+        input_a[index] = make_vector<input_t>(dimensions);
+        input_b[index] = make_vector<input_t>(dimensions);
+        input_c[index] = make_vector<input_t>(dimensions);
+        output[index] = make_vector<input_t>(dimensions);
+        nk::fill_uniform(generator, input_a[index].values_data(), dimensions);
+        std::fill(input_b[index].values_data(), input_b[index].values_data() + dimensions,
+                  input_t(2)); // Small constant
+        nk::fill_uniform(generator, input_c[index].values_data(), dimensions);
+    }
+
+    // Benchmark loop with kernel dispatch
+    std::size_t iterations = 0;
+    for (auto _ : state) {
+        std::size_t const index = iterations & (vectors_count - 1);
+        if constexpr (kernel_kind_ == nk_kernel_each_blend_k) {
+            kernel(input_a[index].raw_values_data(), input_c[index].raw_values_data(), dimensions, &alpha.raw_,
+                   &beta.raw_, output[index].raw_values_data());
+        }
+        else if constexpr (kernel_kind_ == nk_kernel_each_fma_k) {
+            kernel(input_a[index].raw_values_data(), input_b[index].raw_values_data(), input_c[index].raw_values_data(),
+                   dimensions, &alpha.raw_, &beta.raw_, output[index].raw_values_data());
+        }
+        else if constexpr (kernel_kind_ == nk_kernel_each_sum_k) {
+            kernel(input_a[index].raw_values_data(), input_c[index].raw_values_data(), dimensions,
+                   output[index].raw_values_data());
+        }
+        else if constexpr (kernel_kind_ == nk_kernel_each_scale_k) {
+            kernel(input_a[index].raw_values_data(), dimensions, &alpha.raw_, &beta.raw_,
+                   output[index].raw_values_data());
+        }
+        bm::ClobberMemory();
+        iterations++;
+    }
+
+    std::size_t bytes_per_call = input_a[0].size_bytes();
+    if constexpr (kernel_kind_ == nk_kernel_each_blend_k) bytes_per_call *= 2;
+    else if constexpr (kernel_kind_ == nk_kernel_each_fma_k) bytes_per_call *= 3;
+    else if constexpr (kernel_kind_ == nk_kernel_each_sum_k) bytes_per_call *= 2;
+    else if constexpr (kernel_kind_ == nk_kernel_each_scale_k) bytes_per_call *= 1;
+
+    state.counters["bytes"] = bm::Counter(1.0 * iterations * bytes_per_call, bm::Counter::kIsRate);
+    state.counters["calls"] = bm::Counter(iterations, bm::Counter::kIsRate);
+}
+
+template <nk_dtype_t input_dtype_, nk_kernel_kind_t kernel_kind_ = nk_kernel_unknown_k,
+          nk_dtype_t alpha_dtype_ = nk_dtype_unknown_k, typename kernel_type_ = void>
+void run_each(std::string name, kernel_type_ *kernel) {
+    std::string bench_name = name + "<" + std::to_string(bench_config.dense_dimensions) + "d>";
+    bm::RegisterBenchmark(bench_name.c_str(), measure_each<input_dtype_, kernel_kind_, alpha_dtype_, kernel_type_ *>,
+                          kernel, bench_config.dense_dimensions);
+}
+
+void bench_each() {
+    constexpr nk_dtype_t i8_k = nk_i8_k;
+    constexpr nk_dtype_t u8_k = nk_u8_k;
+    constexpr nk_dtype_t i16_k = nk_i16_k;
+    constexpr nk_dtype_t u16_k = nk_u16_k;
+    constexpr nk_dtype_t i32_k = nk_i32_k;
+    constexpr nk_dtype_t u32_k = nk_u32_k;
+    constexpr nk_dtype_t i64_k = nk_i64_k;
+    constexpr nk_dtype_t u64_k = nk_u64_k;
+    constexpr nk_dtype_t f64_k = nk_f64_k;
+    constexpr nk_dtype_t f32_k = nk_f32_k;
+    constexpr nk_dtype_t f16_k = nk_f16_k;
+    constexpr nk_dtype_t bf16_k = nk_bf16_k;
+    constexpr nk_dtype_t e2m3_k = nk_e2m3_k;
+    constexpr nk_dtype_t e3m2_k = nk_e3m2_k;
+    constexpr nk_dtype_t e4m3_k = nk_e4m3_k;
+    constexpr nk_dtype_t e5m2_k = nk_e5m2_k;
+    constexpr nk_dtype_t f32c_k = nk_f32c_k;
+    constexpr nk_dtype_t f64c_k = nk_f64c_k;
+
+    constexpr nk_kernel_kind_t fma_k = nk_kernel_each_fma_k;
+    constexpr nk_kernel_kind_t blend_k = nk_kernel_each_blend_k;
+    constexpr nk_kernel_kind_t sum_k = nk_kernel_each_sum_k;
+    constexpr nk_kernel_kind_t scale_k = nk_kernel_each_scale_k;
+
+#if NK_COMPARE_TO_BLAS || NK_COMPARE_TO_MKL || NK_COMPARE_TO_ACCELERATE
+    run_each<f32_k, sum_k, f32_k>("sum_f32_with_blas", sum_f32_with_blas);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_with_blas", blend_f32_with_blas);
+    run_each<f64_k, sum_k, f64_k>("sum_f64_with_blas", sum_f64_with_blas);
+    run_each<f64_k, blend_k, f64_k>("each_blend_f64_with_blas", blend_f64_with_blas);
+#endif
+
+#if NK_TARGET_NEON
+    // f64
+    run_each<f64_k, sum_k, f64_k>("each_sum_f64_neon", nk_each_sum_f64_neon);
+    run_each<f64_k, scale_k, f64_k>("each_scale_f64_neon", nk_each_scale_f64_neon);
+    run_each<f64_k, blend_k, f64_k>("each_blend_f64_neon", nk_each_blend_f64_neon);
+    run_each<f64_k, fma_k, f64_k>("each_fma_f64_neon", nk_each_fma_f64_neon);
+    // f32
+    run_each<f32_k, sum_k, f32_k>("each_sum_f32_neon", nk_each_sum_f32_neon);
+    run_each<f32_k, scale_k, f32_k>("each_scale_f32_neon", nk_each_scale_f32_neon);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_neon", nk_each_blend_f32_neon);
+    run_each<f32_k, fma_k, f32_k>("each_fma_f32_neon", nk_each_fma_f32_neon);
+    // e4m3
+    run_each<e4m3_k, sum_k, f32_k>("each_sum_e4m3_neon", nk_each_sum_e4m3_neon);
+    run_each<e4m3_k, scale_k, f32_k>("each_scale_e4m3_neon", nk_each_scale_e4m3_neon);
+    run_each<e4m3_k, blend_k, f32_k>("each_blend_e4m3_neon", nk_each_blend_e4m3_neon);
+    run_each<e4m3_k, fma_k, f32_k>("each_fma_e4m3_neon", nk_each_fma_e4m3_neon);
+    // e5m2
+    run_each<e5m2_k, sum_k, f32_k>("each_sum_e5m2_neon", nk_each_sum_e5m2_neon);
+    run_each<e5m2_k, scale_k, f32_k>("each_scale_e5m2_neon", nk_each_scale_e5m2_neon);
+    run_each<e5m2_k, blend_k, f32_k>("each_blend_e5m2_neon", nk_each_blend_e5m2_neon);
+    run_each<e5m2_k, fma_k, f32_k>("each_fma_e5m2_neon", nk_each_fma_e5m2_neon);
+    // u8, i8
+    run_each<u8_k, sum_k, f32_k>("each_sum_u8_neon", nk_each_sum_u8_neon);
+    run_each<i8_k, sum_k, f32_k>("each_sum_i8_neon", nk_each_sum_i8_neon);
+    // i16, u16
+    run_each<i16_k, sum_k, f32_k>("each_sum_i16_neon", nk_each_sum_i16_neon);
+    run_each<i16_k, scale_k, f32_k>("each_scale_i16_neon", nk_each_scale_i16_neon);
+    run_each<i16_k, fma_k, f32_k>("each_fma_i16_neon", nk_each_fma_i16_neon);
+    run_each<u16_k, sum_k, f32_k>("each_sum_u16_neon", nk_each_sum_u16_neon);
+    run_each<u16_k, scale_k, f32_k>("each_scale_u16_neon", nk_each_scale_u16_neon);
+    run_each<u16_k, fma_k, f32_k>("each_fma_u16_neon", nk_each_fma_u16_neon);
+    // i32, u32
+    run_each<i32_k, sum_k, f64_k>("each_sum_i32_neon", nk_each_sum_i32_neon);
+    run_each<i32_k, scale_k, f64_k>("each_scale_i32_neon", nk_each_scale_i32_neon);
+    run_each<i32_k, fma_k, f64_k>("each_fma_i32_neon", nk_each_fma_i32_neon);
+    run_each<u32_k, sum_k, f64_k>("each_sum_u32_neon", nk_each_sum_u32_neon);
+    run_each<u32_k, scale_k, f64_k>("each_scale_u32_neon", nk_each_scale_u32_neon);
+    run_each<u32_k, fma_k, f64_k>("each_fma_u32_neon", nk_each_fma_u32_neon);
+    // i64, u64
+    run_each<i64_k, sum_k, f64_k>("each_sum_i64_neon", nk_each_sum_i64_neon);
+    run_each<i64_k, scale_k, f64_k>("each_scale_i64_neon", nk_each_scale_i64_neon);
+    run_each<i64_k, fma_k, f64_k>("each_fma_i64_neon", nk_each_fma_i64_neon);
+    run_each<u64_k, sum_k, f64_k>("each_sum_u64_neon", nk_each_sum_u64_neon);
+    run_each<u64_k, scale_k, f64_k>("each_scale_u64_neon", nk_each_scale_u64_neon);
+    run_each<u64_k, fma_k, f64_k>("each_fma_u64_neon", nk_each_fma_u64_neon);
+    // complex
+    run_each<f32c_k, scale_k, f32c_k>("each_scale_f32c_neon", nk_each_scale_f32c_neon);
+    run_each<f32c_k, blend_k, f32c_k>("each_blend_f32c_neon", nk_each_blend_f32c_neon);
+    run_each<f32c_k, fma_k, f32c_k>("each_fma_f32c_neon", nk_each_fma_f32c_neon);
+    run_each<f64c_k, scale_k, f64c_k>("each_scale_f64c_neon", nk_each_scale_f64c_neon);
+    run_each<f64c_k, blend_k, f64c_k>("each_blend_f64c_neon", nk_each_blend_f64c_neon);
+    run_each<f64c_k, fma_k, f64c_k>("each_fma_f64c_neon", nk_each_fma_f64c_neon);
+#endif
+
+#if NK_TARGET_NEONHALF
+    // f16
+    run_each<f16_k, sum_k, f32_k>("each_sum_f16_neonhalf", nk_each_sum_f16_neonhalf);
+    run_each<f16_k, scale_k, f32_k>("each_scale_f16_neonhalf", nk_each_scale_f16_neonhalf);
+    run_each<f16_k, blend_k, f32_k>("each_blend_f16_neonhalf", nk_each_blend_f16_neonhalf);
+    run_each<f16_k, fma_k, f32_k>("each_fma_f16_neonhalf", nk_each_fma_f16_neonhalf);
+    // u8
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_neonhalf", nk_each_scale_u8_neonhalf);
+    run_each<u8_k, blend_k, f32_k>("each_blend_u8_neonhalf", nk_each_blend_u8_neonhalf);
+    // i8
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_neonhalf", nk_each_scale_i8_neonhalf);
+    run_each<i8_k, blend_k, f32_k>("each_blend_i8_neonhalf", nk_each_blend_i8_neonhalf);
+#endif
+
+#if NK_TARGET_NEONBFDOT
+    run_each<bf16_k, sum_k, f32_k>("each_sum_bf16_neonbfdot", nk_each_sum_bf16_neonbfdot);
+    run_each<bf16_k, scale_k, f32_k>("each_scale_bf16_neonbfdot", nk_each_scale_bf16_neonbfdot);
+    run_each<bf16_k, blend_k, f32_k>("each_blend_bf16_neonbfdot", nk_each_blend_bf16_neonbfdot);
+    run_each<bf16_k, fma_k, f32_k>("each_fma_bf16_neonbfdot", nk_each_fma_bf16_neonbfdot);
+#endif
+
+#if NK_TARGET_HASWELL
+    // f64
+    run_each<f64_k, sum_k, f64_k>("each_sum_f64_haswell", nk_each_sum_f64_haswell);
+    run_each<f64_k, scale_k, f64_k>("each_scale_f64_haswell", nk_each_scale_f64_haswell);
+    run_each<f64_k, blend_k, f64_k>("each_blend_f64_haswell", nk_each_blend_f64_haswell);
+    run_each<f64_k, fma_k, f64_k>("each_fma_f64_haswell", nk_each_fma_f64_haswell);
+    // f32
+    run_each<f32_k, sum_k, f32_k>("each_sum_f32_haswell", nk_each_sum_f32_haswell);
+    run_each<f32_k, scale_k, f32_k>("each_scale_f32_haswell", nk_each_scale_f32_haswell);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_haswell", nk_each_blend_f32_haswell);
+    run_each<f32_k, fma_k, f32_k>("each_fma_f32_haswell", nk_each_fma_f32_haswell);
+    // f16
+    run_each<f16_k, sum_k, f32_k>("each_sum_f16_haswell", nk_each_sum_f16_haswell);
+    run_each<f16_k, scale_k, f32_k>("each_scale_f16_haswell", nk_each_scale_f16_haswell);
+    run_each<f16_k, blend_k, f32_k>("each_blend_f16_haswell", nk_each_blend_f16_haswell);
+    run_each<f16_k, fma_k, f32_k>("each_fma_f16_haswell", nk_each_fma_f16_haswell);
+    // bf16
+    run_each<bf16_k, sum_k, f32_k>("each_sum_bf16_haswell", nk_each_sum_bf16_haswell);
+    run_each<bf16_k, scale_k, f32_k>("each_scale_bf16_haswell", nk_each_scale_bf16_haswell);
+    run_each<bf16_k, blend_k, f32_k>("each_blend_bf16_haswell", nk_each_blend_bf16_haswell);
+    run_each<bf16_k, fma_k, f32_k>("each_fma_bf16_haswell", nk_each_fma_bf16_haswell);
+    // e4m3
+    run_each<e4m3_k, sum_k, f32_k>("each_sum_e4m3_haswell", nk_each_sum_e4m3_haswell);
+    run_each<e4m3_k, scale_k, f32_k>("each_scale_e4m3_haswell", nk_each_scale_e4m3_haswell);
+    run_each<e4m3_k, blend_k, f32_k>("each_blend_e4m3_haswell", nk_each_blend_e4m3_haswell);
+    run_each<e4m3_k, fma_k, f32_k>("each_fma_e4m3_haswell", nk_each_fma_e4m3_haswell);
+    // e5m2
+    run_each<e5m2_k, sum_k, f32_k>("each_sum_e5m2_haswell", nk_each_sum_e5m2_haswell);
+    run_each<e5m2_k, scale_k, f32_k>("each_scale_e5m2_haswell", nk_each_scale_e5m2_haswell);
+    run_each<e5m2_k, blend_k, f32_k>("each_blend_e5m2_haswell", nk_each_blend_e5m2_haswell);
+    run_each<e5m2_k, fma_k, f32_k>("each_fma_e5m2_haswell", nk_each_fma_e5m2_haswell);
+    // i8, u8
+    run_each<i8_k, sum_k, f32_k>("each_sum_i8_haswell", nk_each_sum_i8_haswell);
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_haswell", nk_each_scale_i8_haswell);
+    run_each<i8_k, blend_k, f32_k>("each_blend_i8_haswell", nk_each_blend_i8_haswell);
+    run_each<i8_k, fma_k, f32_k>("each_fma_i8_haswell", nk_each_fma_i8_haswell);
+    run_each<u8_k, sum_k, f32_k>("each_sum_u8_haswell", nk_each_sum_u8_haswell);
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_haswell", nk_each_scale_u8_haswell);
+    run_each<u8_k, blend_k, f32_k>("each_blend_u8_haswell", nk_each_blend_u8_haswell);
+    run_each<u8_k, fma_k, f32_k>("each_fma_u8_haswell", nk_each_fma_u8_haswell);
+    // i16, u16
+    run_each<i16_k, sum_k, f32_k>("each_sum_i16_haswell", nk_each_sum_i16_haswell);
+    run_each<i16_k, scale_k, f32_k>("each_scale_i16_haswell", nk_each_scale_i16_haswell);
+    run_each<i16_k, fma_k, f32_k>("each_fma_i16_haswell", nk_each_fma_i16_haswell);
+    run_each<u16_k, sum_k, f32_k>("each_sum_u16_haswell", nk_each_sum_u16_haswell);
+    run_each<u16_k, scale_k, f32_k>("each_scale_u16_haswell", nk_each_scale_u16_haswell);
+    run_each<u16_k, fma_k, f32_k>("each_fma_u16_haswell", nk_each_fma_u16_haswell);
+    // i32, u32
+    run_each<i32_k, sum_k, f64_k>("each_sum_i32_haswell", nk_each_sum_i32_haswell);
+    run_each<i32_k, scale_k, f64_k>("each_scale_i32_haswell", nk_each_scale_i32_haswell);
+    run_each<i32_k, fma_k, f64_k>("each_fma_i32_haswell", nk_each_fma_i32_haswell);
+    run_each<u32_k, sum_k, f64_k>("each_sum_u32_haswell", nk_each_sum_u32_haswell);
+    run_each<u32_k, scale_k, f64_k>("each_scale_u32_haswell", nk_each_scale_u32_haswell);
+    run_each<u32_k, fma_k, f64_k>("each_fma_u32_haswell", nk_each_fma_u32_haswell);
+    // complex
+    run_each<f32c_k, scale_k, f32c_k>("each_scale_f32c_haswell", nk_each_scale_f32c_haswell);
+    run_each<f32c_k, blend_k, f32c_k>("each_blend_f32c_haswell", nk_each_blend_f32c_haswell);
+    run_each<f32c_k, fma_k, f32c_k>("each_fma_f32c_haswell", nk_each_fma_f32c_haswell);
+    run_each<f64c_k, scale_k, f64c_k>("each_scale_f64c_haswell", nk_each_scale_f64c_haswell);
+    run_each<f64c_k, blend_k, f64c_k>("each_blend_f64c_haswell", nk_each_blend_f64c_haswell);
+    run_each<f64c_k, fma_k, f64c_k>("each_fma_f64c_haswell", nk_each_fma_f64c_haswell);
+#endif
+
+#if NK_TARGET_SKYLAKE
+    // f64
+    run_each<f64_k, sum_k, f64_k>("each_sum_f64_skylake", nk_each_sum_f64_skylake);
+    run_each<f64_k, scale_k, f64_k>("each_scale_f64_skylake", nk_each_scale_f64_skylake);
+    run_each<f64_k, blend_k, f64_k>("each_blend_f64_skylake", nk_each_blend_f64_skylake);
+    run_each<f64_k, fma_k, f64_k>("each_fma_f64_skylake", nk_each_fma_f64_skylake);
+    // f32
+    run_each<f32_k, sum_k, f32_k>("each_sum_f32_skylake", nk_each_sum_f32_skylake);
+    run_each<f32_k, scale_k, f32_k>("each_scale_f32_skylake", nk_each_scale_f32_skylake);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_skylake", nk_each_blend_f32_skylake);
+    run_each<f32_k, fma_k, f32_k>("each_fma_f32_skylake", nk_each_fma_f32_skylake);
+    // f16
+    run_each<f16_k, scale_k, f32_k>("each_scale_f16_skylake", nk_each_scale_f16_skylake);
+    run_each<f16_k, blend_k, f32_k>("each_blend_f16_skylake", nk_each_blend_f16_skylake);
+    run_each<f16_k, fma_k, f32_k>("each_fma_f16_skylake", nk_each_fma_f16_skylake);
+    // bf16
+    run_each<bf16_k, sum_k, f32_k>("each_sum_bf16_skylake", nk_each_sum_bf16_skylake);
+    run_each<bf16_k, scale_k, f32_k>("each_scale_bf16_skylake", nk_each_scale_bf16_skylake);
+    run_each<bf16_k, blend_k, f32_k>("each_blend_bf16_skylake", nk_each_blend_bf16_skylake);
+    run_each<bf16_k, fma_k, f32_k>("each_fma_bf16_skylake", nk_each_fma_bf16_skylake);
+    // e4m3
+    run_each<e4m3_k, sum_k, f32_k>("each_sum_e4m3_skylake", nk_each_sum_e4m3_skylake);
+    run_each<e4m3_k, scale_k, f32_k>("each_scale_e4m3_skylake", nk_each_scale_e4m3_skylake);
+    run_each<e4m3_k, blend_k, f32_k>("each_blend_e4m3_skylake", nk_each_blend_e4m3_skylake);
+    run_each<e4m3_k, fma_k, f32_k>("each_fma_e4m3_skylake", nk_each_fma_e4m3_skylake);
+    // e5m2
+    run_each<e5m2_k, sum_k, f32_k>("each_sum_e5m2_skylake", nk_each_sum_e5m2_skylake);
+    run_each<e5m2_k, scale_k, f32_k>("each_scale_e5m2_skylake", nk_each_scale_e5m2_skylake);
+    run_each<e5m2_k, blend_k, f32_k>("each_blend_e5m2_skylake", nk_each_blend_e5m2_skylake);
+    run_each<e5m2_k, fma_k, f32_k>("each_fma_e5m2_skylake", nk_each_fma_e5m2_skylake);
+    // i8, u8
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_skylake", nk_each_scale_i8_skylake);
+    run_each<i8_k, fma_k, f32_k>("each_fma_i8_skylake", nk_each_fma_i8_skylake);
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_skylake", nk_each_scale_u8_skylake);
+    run_each<u8_k, fma_k, f32_k>("each_fma_u8_skylake", nk_each_fma_u8_skylake);
+    // i16, u16
+    run_each<i16_k, scale_k, f32_k>("each_scale_i16_skylake", nk_each_scale_i16_skylake);
+    run_each<i16_k, fma_k, f32_k>("each_fma_i16_skylake", nk_each_fma_i16_skylake);
+    run_each<u16_k, scale_k, f32_k>("each_scale_u16_skylake", nk_each_scale_u16_skylake);
+    run_each<u16_k, fma_k, f32_k>("each_fma_u16_skylake", nk_each_fma_u16_skylake);
+    // i32, u32
+    run_each<i32_k, scale_k, f64_k>("each_scale_i32_skylake", nk_each_scale_i32_skylake);
+    run_each<i32_k, fma_k, f64_k>("each_fma_i32_skylake", nk_each_fma_i32_skylake);
+    run_each<u32_k, scale_k, f64_k>("each_scale_u32_skylake", nk_each_scale_u32_skylake);
+    run_each<u32_k, fma_k, f64_k>("each_fma_u32_skylake", nk_each_fma_u32_skylake);
+    // i64, u64
+    run_each<i64_k, scale_k, f64_k>("each_scale_i64_skylake", nk_each_scale_i64_skylake);
+    run_each<i64_k, fma_k, f64_k>("each_fma_i64_skylake", nk_each_fma_i64_skylake);
+    run_each<u64_k, scale_k, f64_k>("each_scale_u64_skylake", nk_each_scale_u64_skylake);
+    run_each<u64_k, fma_k, f64_k>("each_fma_u64_skylake", nk_each_fma_u64_skylake);
+    // complex
+    run_each<f32c_k, scale_k, f32c_k>("each_scale_f32c_skylake", nk_each_scale_f32c_skylake);
+    run_each<f32c_k, blend_k, f32c_k>("each_blend_f32c_skylake", nk_each_blend_f32c_skylake);
+    run_each<f32c_k, fma_k, f32c_k>("each_fma_f32c_skylake", nk_each_fma_f32c_skylake);
+    run_each<f64c_k, scale_k, f64c_k>("each_scale_f64c_skylake", nk_each_scale_f64c_skylake);
+    run_each<f64c_k, blend_k, f64c_k>("each_blend_f64c_skylake", nk_each_blend_f64c_skylake);
+    run_each<f64c_k, fma_k, f64c_k>("each_fma_f64c_skylake", nk_each_fma_f64c_skylake);
+#endif
+
+#if NK_TARGET_ICELAKE
+    run_each<i8_k, sum_k, f32_k>("each_sum_i8_icelake", nk_each_sum_i8_icelake);
+    run_each<u8_k, sum_k, f32_k>("each_sum_u8_icelake", nk_each_sum_u8_icelake);
+    run_each<i16_k, sum_k, f32_k>("each_sum_i16_icelake", nk_each_sum_i16_icelake);
+    run_each<u16_k, sum_k, f32_k>("each_sum_u16_icelake", nk_each_sum_u16_icelake);
+    run_each<i32_k, sum_k, f32_k>("each_sum_i32_icelake", nk_each_sum_i32_icelake);
+    run_each<u32_k, sum_k, f32_k>("each_sum_u32_icelake", nk_each_sum_u32_icelake);
+    run_each<i64_k, sum_k, f64_k>("each_sum_i64_icelake", nk_each_sum_i64_icelake);
+    run_each<u64_k, sum_k, f64_k>("each_sum_u64_icelake", nk_each_sum_u64_icelake);
+#endif
+
+#if NK_TARGET_SAPPHIRE
+    // i8, u8
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_sapphire", nk_each_scale_i8_sapphire);
+    run_each<i8_k, blend_k, f32_k>("each_blend_i8_sapphire", nk_each_blend_i8_sapphire);
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_sapphire", nk_each_scale_u8_sapphire);
+    run_each<u8_k, blend_k, f32_k>("each_blend_u8_sapphire", nk_each_blend_u8_sapphire);
+    // f16, e4m3
+    run_each<f16_k, sum_k, f32_k>("each_sum_f16_sapphire", nk_each_sum_f16_sapphire);
+    run_each<e4m3_k, sum_k, f32_k>("each_sum_e4m3_sapphire", nk_each_sum_e4m3_sapphire);
+#endif
+
+#if NK_TARGET_RVV
+    run_each<f64_k, scale_k, f64_k>("each_scale_f64_rvv", nk_each_scale_f64_rvv);
+    run_each<f64_k, fma_k, f64_k>("each_fma_f64_rvv", nk_each_fma_f64_rvv);
+    run_each<f64_k, blend_k, f64_k>("each_blend_f64_rvv", nk_each_blend_f64_rvv);
+    run_each<f32_k, scale_k, f32_k>("each_scale_f32_rvv", nk_each_scale_f32_rvv);
+    run_each<f32_k, fma_k, f32_k>("each_fma_f32_rvv", nk_each_fma_f32_rvv);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_rvv", nk_each_blend_f32_rvv);
+    run_each<f16_k, scale_k, f32_k>("each_scale_f16_rvv", nk_each_scale_f16_rvv);
+    run_each<f16_k, fma_k, f32_k>("each_fma_f16_rvv", nk_each_fma_f16_rvv);
+    run_each<f16_k, blend_k, f32_k>("each_blend_f16_rvv", nk_each_blend_f16_rvv);
+    run_each<bf16_k, scale_k, f32_k>("each_scale_bf16_rvv", nk_each_scale_bf16_rvv);
+    run_each<bf16_k, fma_k, f32_k>("each_fma_bf16_rvv", nk_each_fma_bf16_rvv);
+    run_each<bf16_k, blend_k, f32_k>("each_blend_bf16_rvv", nk_each_blend_bf16_rvv);
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_rvv", nk_each_scale_i8_rvv);
+    run_each<i8_k, fma_k, f32_k>("each_fma_i8_rvv", nk_each_fma_i8_rvv);
+    run_each<i8_k, blend_k, f32_k>("each_blend_i8_rvv", nk_each_blend_i8_rvv);
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_rvv", nk_each_scale_u8_rvv);
+    run_each<u8_k, fma_k, f32_k>("each_fma_u8_rvv", nk_each_fma_u8_rvv);
+    run_each<u8_k, blend_k, f32_k>("each_blend_u8_rvv", nk_each_blend_u8_rvv);
+#endif
+
+#if NK_TARGET_V128RELAXED
+    run_each<f32_k, sum_k, f32_k>("each_sum_f32_v128relaxed", nk_each_sum_f32_v128relaxed);
+    run_each<f32_k, scale_k, f32_k>("each_scale_f32_v128relaxed", nk_each_scale_f32_v128relaxed);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_v128relaxed", nk_each_blend_f32_v128relaxed);
+    run_each<f32_k, fma_k, f32_k>("each_fma_f32_v128relaxed", nk_each_fma_f32_v128relaxed);
+    run_each<f16_k, sum_k, f32_k>("each_sum_f16_v128relaxed", nk_each_sum_f16_v128relaxed);
+    run_each<f16_k, scale_k, f32_k>("each_scale_f16_v128relaxed", nk_each_scale_f16_v128relaxed);
+    run_each<f16_k, blend_k, f32_k>("each_blend_f16_v128relaxed", nk_each_blend_f16_v128relaxed);
+    run_each<f16_k, fma_k, f32_k>("each_fma_f16_v128relaxed", nk_each_fma_f16_v128relaxed);
+    run_each<bf16_k, sum_k, f32_k>("each_sum_bf16_v128relaxed", nk_each_sum_bf16_v128relaxed);
+    run_each<bf16_k, scale_k, f32_k>("each_scale_bf16_v128relaxed", nk_each_scale_bf16_v128relaxed);
+    run_each<bf16_k, blend_k, f32_k>("each_blend_bf16_v128relaxed", nk_each_blend_bf16_v128relaxed);
+    run_each<bf16_k, fma_k, f32_k>("each_fma_bf16_v128relaxed", nk_each_fma_bf16_v128relaxed);
+    run_each<i8_k, sum_k, f32_k>("each_sum_i8_v128relaxed", nk_each_sum_i8_v128relaxed);
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_v128relaxed", nk_each_scale_i8_v128relaxed);
+    run_each<i8_k, blend_k, f32_k>("each_blend_i8_v128relaxed", nk_each_blend_i8_v128relaxed);
+    run_each<i8_k, fma_k, f32_k>("each_fma_i8_v128relaxed", nk_each_fma_i8_v128relaxed);
+    run_each<u8_k, sum_k, f32_k>("each_sum_u8_v128relaxed", nk_each_sum_u8_v128relaxed);
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_v128relaxed", nk_each_scale_u8_v128relaxed);
+    run_each<u8_k, blend_k, f32_k>("each_blend_u8_v128relaxed", nk_each_blend_u8_v128relaxed);
+    run_each<u8_k, fma_k, f32_k>("each_fma_u8_v128relaxed", nk_each_fma_u8_v128relaxed);
+#endif
+
+    // Serial fallbacks — f64
+    run_each<f64_k, sum_k, f64_k>("each_sum_f64_serial", nk_each_sum_f64_serial);
+    run_each<f64_k, scale_k, f64_k>("each_scale_f64_serial", nk_each_scale_f64_serial);
+    run_each<f64_k, blend_k, f64_k>("each_blend_f64_serial", nk_each_blend_f64_serial);
+    run_each<f64_k, fma_k, f64_k>("each_fma_f64_serial", nk_each_fma_f64_serial);
+    // Serial fallbacks — f32
+    run_each<f32_k, sum_k, f32_k>("each_sum_f32_serial", nk_each_sum_f32_serial);
+    run_each<f32_k, scale_k, f32_k>("each_scale_f32_serial", nk_each_scale_f32_serial);
+    run_each<f32_k, blend_k, f32_k>("each_blend_f32_serial", nk_each_blend_f32_serial);
+    run_each<f32_k, fma_k, f32_k>("each_fma_f32_serial", nk_each_fma_f32_serial);
+    // Serial fallbacks — f16, bf16
+    run_each<f16_k, sum_k, f32_k>("each_sum_f16_serial", nk_each_sum_f16_serial);
+    run_each<f16_k, scale_k, f32_k>("each_scale_f16_serial", nk_each_scale_f16_serial);
+    run_each<f16_k, blend_k, f32_k>("each_blend_f16_serial", nk_each_blend_f16_serial);
+    run_each<f16_k, fma_k, f32_k>("each_fma_f16_serial", nk_each_fma_f16_serial);
+    run_each<bf16_k, sum_k, f32_k>("each_sum_bf16_serial", nk_each_sum_bf16_serial);
+    run_each<bf16_k, scale_k, f32_k>("each_scale_bf16_serial", nk_each_scale_bf16_serial);
+    run_each<bf16_k, blend_k, f32_k>("each_blend_bf16_serial", nk_each_blend_bf16_serial);
+    run_each<bf16_k, fma_k, f32_k>("each_fma_bf16_serial", nk_each_fma_bf16_serial);
+    // Serial fallbacks — e4m3, e5m2, e2m3, e3m2
+    run_each<e4m3_k, sum_k, f32_k>("each_sum_e4m3_serial", nk_each_sum_e4m3_serial);
+    run_each<e4m3_k, scale_k, f32_k>("each_scale_e4m3_serial", nk_each_scale_e4m3_serial);
+    run_each<e4m3_k, blend_k, f32_k>("each_blend_e4m3_serial", nk_each_blend_e4m3_serial);
+    run_each<e4m3_k, fma_k, f32_k>("each_fma_e4m3_serial", nk_each_fma_e4m3_serial);
+    run_each<e5m2_k, sum_k, f32_k>("each_sum_e5m2_serial", nk_each_sum_e5m2_serial);
+    run_each<e5m2_k, scale_k, f32_k>("each_scale_e5m2_serial", nk_each_scale_e5m2_serial);
+    run_each<e5m2_k, blend_k, f32_k>("each_blend_e5m2_serial", nk_each_blend_e5m2_serial);
+    run_each<e5m2_k, fma_k, f32_k>("each_fma_e5m2_serial", nk_each_fma_e5m2_serial);
+    run_each<e2m3_k, sum_k, f32_k>("each_sum_e2m3_serial", nk_each_sum_e2m3_serial);
+    run_each<e2m3_k, scale_k, f32_k>("each_scale_e2m3_serial", nk_each_scale_e2m3_serial);
+    run_each<e2m3_k, blend_k, f32_k>("each_blend_e2m3_serial", nk_each_blend_e2m3_serial);
+    run_each<e2m3_k, fma_k, f32_k>("each_fma_e2m3_serial", nk_each_fma_e2m3_serial);
+    run_each<e3m2_k, sum_k, f32_k>("each_sum_e3m2_serial", nk_each_sum_e3m2_serial);
+    run_each<e3m2_k, scale_k, f32_k>("each_scale_e3m2_serial", nk_each_scale_e3m2_serial);
+    run_each<e3m2_k, blend_k, f32_k>("each_blend_e3m2_serial", nk_each_blend_e3m2_serial);
+    run_each<e3m2_k, fma_k, f32_k>("each_fma_e3m2_serial", nk_each_fma_e3m2_serial);
+    // Serial fallbacks — i8, u8
+    run_each<i8_k, sum_k, f32_k>("each_sum_i8_serial", nk_each_sum_i8_serial);
+    run_each<i8_k, scale_k, f32_k>("each_scale_i8_serial", nk_each_scale_i8_serial);
+    run_each<i8_k, blend_k, f32_k>("each_blend_i8_serial", nk_each_blend_i8_serial);
+    run_each<i8_k, fma_k, f32_k>("each_fma_i8_serial", nk_each_fma_i8_serial);
+    run_each<u8_k, sum_k, f32_k>("each_sum_u8_serial", nk_each_sum_u8_serial);
+    run_each<u8_k, scale_k, f32_k>("each_scale_u8_serial", nk_each_scale_u8_serial);
+    run_each<u8_k, blend_k, f32_k>("each_blend_u8_serial", nk_each_blend_u8_serial);
+    run_each<u8_k, fma_k, f32_k>("each_fma_u8_serial", nk_each_fma_u8_serial);
+    // Serial fallbacks — i16, u16
+    run_each<i16_k, sum_k, f32_k>("each_sum_i16_serial", nk_each_sum_i16_serial);
+    run_each<i16_k, scale_k, f32_k>("each_scale_i16_serial", nk_each_scale_i16_serial);
+    run_each<i16_k, blend_k, f32_k>("each_blend_i16_serial", nk_each_blend_i16_serial);
+    run_each<i16_k, fma_k, f32_k>("each_fma_i16_serial", nk_each_fma_i16_serial);
+    run_each<u16_k, sum_k, f32_k>("each_sum_u16_serial", nk_each_sum_u16_serial);
+    run_each<u16_k, scale_k, f32_k>("each_scale_u16_serial", nk_each_scale_u16_serial);
+    run_each<u16_k, blend_k, f32_k>("each_blend_u16_serial", nk_each_blend_u16_serial);
+    run_each<u16_k, fma_k, f32_k>("each_fma_u16_serial", nk_each_fma_u16_serial);
+    // Serial fallbacks — i32, u32
+    run_each<i32_k, sum_k, f64_k>("each_sum_i32_serial", nk_each_sum_i32_serial);
+    run_each<i32_k, scale_k, f64_k>("each_scale_i32_serial", nk_each_scale_i32_serial);
+    run_each<i32_k, blend_k, f64_k>("each_blend_i32_serial", nk_each_blend_i32_serial);
+    run_each<i32_k, fma_k, f64_k>("each_fma_i32_serial", nk_each_fma_i32_serial);
+    run_each<u32_k, sum_k, f64_k>("each_sum_u32_serial", nk_each_sum_u32_serial);
+    run_each<u32_k, scale_k, f64_k>("each_scale_u32_serial", nk_each_scale_u32_serial);
+    run_each<u32_k, blend_k, f64_k>("each_blend_u32_serial", nk_each_blend_u32_serial);
+    run_each<u32_k, fma_k, f64_k>("each_fma_u32_serial", nk_each_fma_u32_serial);
+    // Serial fallbacks — i64, u64
+    run_each<i64_k, sum_k, f64_k>("each_sum_i64_serial", nk_each_sum_i64_serial);
+    run_each<i64_k, scale_k, f64_k>("each_scale_i64_serial", nk_each_scale_i64_serial);
+    run_each<i64_k, blend_k, f64_k>("each_blend_i64_serial", nk_each_blend_i64_serial);
+    run_each<i64_k, fma_k, f64_k>("each_fma_i64_serial", nk_each_fma_i64_serial);
+    run_each<u64_k, sum_k, f64_k>("each_sum_u64_serial", nk_each_sum_u64_serial);
+    run_each<u64_k, scale_k, f64_k>("each_scale_u64_serial", nk_each_scale_u64_serial);
+    run_each<u64_k, blend_k, f64_k>("each_blend_u64_serial", nk_each_blend_u64_serial);
+    run_each<u64_k, fma_k, f64_k>("each_fma_u64_serial", nk_each_fma_u64_serial);
+    // Serial fallbacks — complex
+    run_each<f32c_k, sum_k, f32c_k>("each_sum_f32c_serial", nk_each_sum_f32c_serial);
+    run_each<f32c_k, scale_k, f32c_k>("each_scale_f32c_serial", nk_each_scale_f32c_serial);
+    run_each<f32c_k, blend_k, f32c_k>("each_blend_f32c_serial", nk_each_blend_f32c_serial);
+    run_each<f32c_k, fma_k, f32c_k>("each_fma_f32c_serial", nk_each_fma_f32c_serial);
+    run_each<f64c_k, sum_k, f64c_k>("each_sum_f64c_serial", nk_each_sum_f64c_serial);
+    run_each<f64c_k, scale_k, f64c_k>("each_scale_f64c_serial", nk_each_scale_f64c_serial);
+    run_each<f64c_k, blend_k, f64c_k>("each_blend_f64c_serial", nk_each_blend_f64c_serial);
+    run_each<f64c_k, fma_k, f64c_k>("each_fma_f64c_serial", nk_each_fma_f64c_serial);
+}

package/numkong/bench/bench_geospatial.cpp

@@ -0,0 +1,112 @@
+/**
+ *  @brief Geospatial distance benchmarks (haversine, vincenty).
+ *  @file bench/bench_geospatial.cpp
+ *  @author Ash Vardanian
+ *  @date March 14, 2023
+ */
+
+#include "numkong/geospatial.h"
+
+#include "bench.hpp"
+
+/**
+ *  @brief Measures the performance of geospatial operations (Haversine/Vincenty) using Google Benchmark.
+ *  @param state The benchmark state object provided by Google Benchmark.
+ *  @param kernel The kernel function to benchmark.
+ *  @param coordinates_count The number of coordinate pairs to process.
+ */
+template <nk_dtype_t dtype_, typename kernel_type_ = void>
+void measure_geospatial(bm::State &state, kernel_type_ kernel, std::size_t coordinates_count) {
+
+    using scalar_t = typename nk::type_for<dtype_>::type;
+    using vector_t = nk::vector<scalar_t>;
+
+    // Preallocate coordinate arrays: latitude1, longitude1, latitude2, longitude2
+    constexpr std::size_t batches_count = 1024;
+    std::vector<vector_t> latitudes_first(batches_count), longitudes_first(batches_count);
+    std::vector<vector_t> latitudes_second(batches_count), longitudes_second(batches_count);
+    auto generator = make_random_engine();
+    double const max_separation_rad = double(bench_config.max_coord_angle) * 3.14159265358979323846 / 180.0;
+    for (std::size_t index = 0; index != batches_count; ++index) {
+        latitudes_first[index] = make_vector<scalar_t>(coordinates_count);
+        longitudes_first[index] = make_vector<scalar_t>(coordinates_count);
+        latitudes_second[index] = make_vector<scalar_t>(coordinates_count);
+        longitudes_second[index] = make_vector<scalar_t>(coordinates_count);
+        nk::fill_coordinates(generator, latitudes_first[index].values_data(), longitudes_first[index].values_data(),
+                             coordinates_count);
+        nk::fill_nearby_coordinates(generator, latitudes_first[index].values_data(),
+                                    longitudes_first[index].values_data(), latitudes_second[index].values_data(),
+                                    longitudes_second[index].values_data(), coordinates_count, max_separation_rad);
+    }
+
+    // Output distances buffer
+    vector_t distances = make_vector<scalar_t>(coordinates_count);
+
+    // Benchmark loop
+    std::size_t iterations = 0;
+    for (auto _ : state) {
+        std::size_t const index = iterations & (batches_count - 1);
+        kernel(latitudes_first[index].raw_values_data(), longitudes_first[index].raw_values_data(),
+               latitudes_second[index].raw_values_data(), longitudes_second[index].raw_values_data(), coordinates_count,
+               distances.raw_values_data());
+        bm::ClobberMemory();
+        iterations++;
+    }
+
+    state.counters["ops"] = bm::Counter(1.0 * iterations * coordinates_count, bm::Counter::kIsRate);
+    state.counters["calls"] = bm::Counter(iterations, bm::Counter::kIsRate);
+}
+
+template <nk_dtype_t dtype_, typename kernel_type_ = void>
+void run_geospatial(std::string name, kernel_type_ *kernel) {
+    std::string bench_name = name + "<" + std::to_string(bench_config.dense_dimensions) + "d," +
+                             std::to_string(static_cast<int>(bench_config.max_coord_angle)) + "°>";
+    bm::RegisterBenchmark(bench_name.c_str(), measure_geospatial<dtype_, kernel_type_ *>, kernel,
+                          bench_config.dense_dimensions);
+}
+
+void bench_geospatial() {
+    constexpr nk_dtype_t f64_k = nk_f64_k;
+    constexpr nk_dtype_t f32_k = nk_f32_k;
+
+#if NK_TARGET_NEON
+    run_geospatial<f32_k>("haversine_f32_neon", nk_haversine_f32_neon);
+    run_geospatial<f64_k>("haversine_f64_neon", nk_haversine_f64_neon);
+    run_geospatial<f32_k>("vincenty_f32_neon", nk_vincenty_f32_neon);
+    run_geospatial<f64_k>("vincenty_f64_neon", nk_vincenty_f64_neon);
+#endif
+
+#if NK_TARGET_HASWELL
+    run_geospatial<f32_k>("haversine_f32_haswell", nk_haversine_f32_haswell);
+    run_geospatial<f64_k>("haversine_f64_haswell", nk_haversine_f64_haswell);
+    run_geospatial<f32_k>("vincenty_f32_haswell", nk_vincenty_f32_haswell);
+    run_geospatial<f64_k>("vincenty_f64_haswell", nk_vincenty_f64_haswell);
+#endif
+
+#if NK_TARGET_SKYLAKE
+    run_geospatial<f32_k>("haversine_f32_skylake", nk_haversine_f32_skylake);
+    run_geospatial<f64_k>("haversine_f64_skylake", nk_haversine_f64_skylake);
+    run_geospatial<f32_k>("vincenty_f32_skylake", nk_vincenty_f32_skylake);
+    run_geospatial<f64_k>("vincenty_f64_skylake", nk_vincenty_f64_skylake);
+#endif
+
+#if NK_TARGET_RVV
+    run_geospatial<f32_k>("haversine_f32_rvv", nk_haversine_f32_rvv);
+    run_geospatial<f64_k>("haversine_f64_rvv", nk_haversine_f64_rvv);
+    run_geospatial<f32_k>("vincenty_f32_rvv", nk_vincenty_f32_rvv);
+    run_geospatial<f64_k>("vincenty_f64_rvv", nk_vincenty_f64_rvv);
+#endif
+
+#if NK_TARGET_V128RELAXED
+    run_geospatial<f32_k>("haversine_f32_v128relaxed", nk_haversine_f32_v128relaxed);
+    run_geospatial<f64_k>("haversine_f64_v128relaxed", nk_haversine_f64_v128relaxed);
+    run_geospatial<f32_k>("vincenty_f32_v128relaxed", nk_vincenty_f32_v128relaxed);
+    run_geospatial<f64_k>("vincenty_f64_v128relaxed", nk_vincenty_f64_v128relaxed);
+#endif
+
+    // Serial fallbacks
+    run_geospatial<f32_k>("haversine_f32_serial", nk_haversine_f32_serial);
+    run_geospatial<f64_k>("haversine_f64_serial", nk_haversine_f64_serial);
+    run_geospatial<f32_k>("vincenty_f32_serial", nk_vincenty_f32_serial);
+    run_geospatial<f64_k>("vincenty_f64_serial", nk_vincenty_f64_serial);
+}