usearch 2.21.4 → 2.25.1

package/numkong/include/numkong/spatial/sve.h

@@ -0,0 +1,246 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for SVE.
+ *  @file include/numkong/spatial/sve.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  @section spatial_sve_instructions ARM SVE Instructions
+ *
+ *      Intrinsic      Instruction                V1
+ *      svld1_f32      LD1W (Z.S, P/Z, [Xn])      4-6cy @ 2p
+ *      svsub_f32_x    FSUB (Z.S, P/M, Z.S, Z.S)  3cy @ 2p
+ *      svmla_f32_x    FMLA (Z.S, P/M, Z.S, Z.S)  4cy @ 2p
+ *      svaddv_f32     FADDV (S, P, Z.S)          6cy @ 1p
+ *      svdupq_n_f32   DUP (Z.S, #imm)            1cy @ 2p
+ *      svwhilelt_b32  WHILELT (P.S, Xn, Xm)      2cy @ 1p
+ *      svptrue_b32    PTRUE (P.S, pattern)       1cy @ 2p
+ *      svcntw         CNTW (Xd)                  1cy @ 2p
+ *      svld1_f64      LD1D (Z.D, P/Z, [Xn])      4-6cy @ 2p
+ *      svsub_f64_x    FSUB (Z.D, P/M, Z.D, Z.D)  3cy @ 2p
+ *      svmla_f64_x    FMLA (Z.D, P/M, Z.D, Z.D)  4cy @ 2p
+ *      svaddv_f64     FADDV (D, P, Z.D)          6cy @ 1p
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  Spatial operations like L2 distance and angular similarity benefit from SVE's fused
+ *  multiply-add instructions. The FADDV reduction dominates the critical path for short vectors.
+ */
+#ifndef NK_SPATIAL_SVE_H
+#define NK_SPATIAL_SVE_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SVE
+
+#include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
+#include "numkong/spatial/neon.h" // `nk_f64_sqrt_neon`
+#include "numkong/dot/sve.h"      // `nk_dot_stable_sum_f64_sve_`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve")
+#endif
+
+/** @brief Reciprocal square root of an f32 SVE vector via estimate + 2 Newton-Raphson steps.
+ *
+ *  Computes 1/sqrt(x) for each active lane. The initial estimate from `svrsqrte_f32`
+ *  has ~8 bits of precision; each Newton-Raphson iteration via `svrsqrts_f32` roughly
+ *  doubles the mantissa bits, giving ~23 bits (~full f32 precision) after 2 steps.
+ *
+ *  Marked `__arm_streaming_compatible` so the helper is callable from both streaming
+ *  (SME) and non-streaming (SVE) contexts without mode transitions.
+ *
+ *  @param predicate Active-lane mask
+ *  @param x         Input vector (must be positive for meaningful results)
+ *  @return          Approximate 1/sqrt(x) with ~23-bit mantissa accuracy
+ */
+NK_INTERNAL svfloat32_t nk_rsqrt_f32x_sve_(svbool_t predicate_b32x, svfloat32_t x) NK_STREAMING_COMPATIBLE_ {
+    svfloat32_t r = svrsqrte_f32(x);
+    r = svmul_f32_x(predicate_b32x, r, svrsqrts_f32(svmul_f32_x(predicate_b32x, x, r), r));
+    r = svmul_f32_x(predicate_b32x, r, svrsqrts_f32(svmul_f32_x(predicate_b32x, x, r), r));
+    return r;
+}
+
+/** @brief Reciprocal square root of an f64 SVE vector via estimate + 3 Newton-Raphson steps.
+ *
+ *  Computes 1/sqrt(x) for each active lane. The initial estimate from `svrsqrte_f64`
+ *  has ~8 bits of precision; three Newton-Raphson iterations via `svrsqrts_f64` yield
+ *  ~52-bit mantissa accuracy (full f64 precision).
+ *
+ *  Marked `__arm_streaming_compatible` so the helper is callable from both streaming
+ *  (SME) and non-streaming (SVE) contexts without mode transitions.
+ *
+ *  @param predicate_b32x Active-lane mask
+ *  @param x         Input vector (must be positive for meaningful results)
+ *  @return          Approximate 1/sqrt(x) with ~52-bit mantissa accuracy
+ */
+NK_INTERNAL svfloat64_t nk_rsqrt_f64x_sve_(svbool_t predicate_b64x, svfloat64_t x) NK_STREAMING_COMPATIBLE_ {
+    svfloat64_t r = svrsqrte_f64(x);
+    r = svmul_f64_x(predicate_b64x, r, svrsqrts_f64(svmul_f64_x(predicate_b64x, x, r), r));
+    r = svmul_f64_x(predicate_b64x, r, svrsqrts_f64(svmul_f64_x(predicate_b64x, x, r), r));
+    r = svmul_f64_x(predicate_b64x, r, svrsqrts_f64(svmul_f64_x(predicate_b64x, x, r), r));
+    return r;
+}
+
+NK_PUBLIC void nk_sqeuclidean_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
+    nk_size_t i = 0;
+    svfloat64_t dist_sq_f64x = svdupq_n_f64(0.0, 0.0);
+    for (; i < n; i += svcntw()) {
+        svbool_t predicate_b32x = svwhilelt_b32_u64(i, n);
+        svfloat32_t a_f32x = svld1_f32(predicate_b32x, a + i);
+        svfloat32_t b_f32x = svld1_f32(predicate_b32x, b + i);
+        nk_size_t remaining = n - i < svcntw() ? n - i : svcntw();
+
+        // svcvt_f64_f32_x widens only even-indexed f32 elements; svext by 1 shifts odd into even.
+        svbool_t pred_even_b64x = svwhilelt_b64_u64(0u, (remaining + 1) / 2);
+        svfloat64_t a_even_f64x = svcvt_f64_f32_x(pred_even_b64x, a_f32x);
+        svfloat64_t b_even_f64x = svcvt_f64_f32_x(pred_even_b64x, b_f32x);
+        svfloat64_t diff_even_f64x = svsub_f64_x(pred_even_b64x, a_even_f64x, b_even_f64x);
+        dist_sq_f64x = svmla_f64_m(pred_even_b64x, dist_sq_f64x, diff_even_f64x, diff_even_f64x);
+
+        svbool_t pred_odd_b64x = svwhilelt_b64_u64(0u, remaining / 2);
+        svfloat64_t a_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(a_f32x, a_f32x, 1));
+        svfloat64_t b_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(b_f32x, b_f32x, 1));
+        svfloat64_t diff_odd_f64x = svsub_f64_x(pred_odd_b64x, a_odd_f64x, b_odd_f64x);
+        dist_sq_f64x = svmla_f64_m(pred_odd_b64x, dist_sq_f64x, diff_odd_f64x, diff_odd_f64x);
+    }
+    nk_f64_t dist_sq_f64 = nk_svaddv_f64_(svptrue_b64(), dist_sq_f64x);
+    *result = dist_sq_f64;
+}
+
+NK_PUBLIC void nk_euclidean_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
+    nk_sqeuclidean_f32_sve(a, b, n, result);
+    *result = nk_f64_sqrt_neon(*result);
+}
+
+NK_PUBLIC void nk_angular_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
+    nk_size_t i = 0;
+    svfloat64_t ab_f64x = svdupq_n_f64(0.0, 0.0);
+    svfloat64_t a2_f64x = svdupq_n_f64(0.0, 0.0);
+    svfloat64_t b2_f64x = svdupq_n_f64(0.0, 0.0);
+    for (; i < n; i += svcntw()) {
+        svbool_t predicate_b32x = svwhilelt_b32_u64(i, n);
+        svfloat32_t a_f32x = svld1_f32(predicate_b32x, a + i);
+        svfloat32_t b_f32x = svld1_f32(predicate_b32x, b + i);
+        nk_size_t remaining = n - i < svcntw() ? n - i : svcntw();
+
+        // svcvt_f64_f32_x widens only even-indexed f32 elements; svext by 1 shifts odd into even.
+        svbool_t pred_even_b64x = svwhilelt_b64_u64(0u, (remaining + 1) / 2);
+        svfloat64_t a_even_f64x = svcvt_f64_f32_x(pred_even_b64x, a_f32x);
+        svfloat64_t b_even_f64x = svcvt_f64_f32_x(pred_even_b64x, b_f32x);
+        ab_f64x = svmla_f64_m(pred_even_b64x, ab_f64x, a_even_f64x, b_even_f64x);
+        a2_f64x = svmla_f64_m(pred_even_b64x, a2_f64x, a_even_f64x, a_even_f64x);
+        b2_f64x = svmla_f64_m(pred_even_b64x, b2_f64x, b_even_f64x, b_even_f64x);
+
+        svbool_t pred_odd_b64x = svwhilelt_b64_u64(0u, remaining / 2);
+        svfloat64_t a_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(a_f32x, a_f32x, 1));
+        svfloat64_t b_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(b_f32x, b_f32x, 1));
+        ab_f64x = svmla_f64_m(pred_odd_b64x, ab_f64x, a_odd_f64x, b_odd_f64x);
+        a2_f64x = svmla_f64_m(pred_odd_b64x, a2_f64x, a_odd_f64x, a_odd_f64x);
+        b2_f64x = svmla_f64_m(pred_odd_b64x, b2_f64x, b_odd_f64x, b_odd_f64x);
+    }
+
+    nk_f64_t ab_f64 = nk_svaddv_f64_(svptrue_b64(), ab_f64x);
+    nk_f64_t a2_f64 = nk_svaddv_f64_(svptrue_b64(), a2_f64x);
+    nk_f64_t b2_f64 = nk_svaddv_f64_(svptrue_b64(), b2_f64x);
+    *result = nk_angular_normalize_f64_neon_(ab_f64, a2_f64, b2_f64);
+}
+
+NK_PUBLIC void nk_sqeuclidean_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result) {
+    // Neumaier compensated summation for numerical stability
+    nk_size_t i = 0;
+    svfloat64_t sum_f64x = svdupq_n_f64(0.0, 0.0);
+    svfloat64_t compensation_f64x = svdupq_n_f64(0.0, 0.0);
+    svbool_t predicate_all_b64x = svptrue_b64();
+    do {
+        svbool_t predicate_b64x = svwhilelt_b64_u64(i, n);
+        svfloat64_t a_f64x = svld1_f64(predicate_b64x, a + i);
+        svfloat64_t b_f64x = svld1_f64(predicate_b64x, b + i);
+        svfloat64_t diff_f64x = svsub_f64_x(predicate_b64x, a_f64x, b_f64x);
+        svfloat64_t diff_sq_f64x = svmul_f64_x(predicate_b64x, diff_f64x, diff_f64x);
+        // Neumaier: t = sum + x
+        svfloat64_t t_f64x = svadd_f64_m(predicate_b64x, sum_f64x, diff_sq_f64x);
+        svfloat64_t abs_sum_f64x = svabs_f64_x(predicate_b64x, sum_f64x);
+        // diff_sq is already non-negative (it's a square), so svabs is unnecessary
+        svbool_t sum_ge_x_b64x = svcmpge_f64(predicate_b64x, abs_sum_f64x, diff_sq_f64x);
+        // When |sum| >= |x|: comp += (sum - t) + x; when |x| > |sum|: comp += (x - t) + sum
+        svfloat64_t comp_sum_large_f64x = svadd_f64_x(predicate_b64x, svsub_f64_x(predicate_b64x, sum_f64x, t_f64x),
+                                                      diff_sq_f64x);
+        svfloat64_t comp_x_large_f64x = svadd_f64_x(predicate_b64x, svsub_f64_x(predicate_b64x, diff_sq_f64x, t_f64x),
+                                                    sum_f64x);
+        svfloat64_t comp_update_f64x = svsel_f64(sum_ge_x_b64x, comp_sum_large_f64x, comp_x_large_f64x);
+        compensation_f64x = svadd_f64_m(predicate_b64x, compensation_f64x, comp_update_f64x);
+        sum_f64x = t_f64x;
+        i += svcntd();
+    } while (i < n);
+    *result = nk_dot_stable_sum_f64_sve_(predicate_all_b64x, sum_f64x, compensation_f64x);
+}
+
+NK_PUBLIC void nk_euclidean_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result) {
+    nk_sqeuclidean_f64_sve(a, b, n, result);
+    *result = nk_f64_sqrt_neon(*result);
+}
+
+NK_PUBLIC void nk_angular_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result) {
+    // Dot2 (Ogita-Rump-Oishi) for cross-product ab (may have cancellation),
+    // simple FMA for self-products a2/b2 (all positive, no cancellation)
+    nk_size_t i = 0;
+    svfloat64_t ab_sum_f64x = svdupq_n_f64(0.0, 0.0);
+    svfloat64_t ab_compensation_f64x = svdupq_n_f64(0.0, 0.0);
+    svfloat64_t a2_f64x = svdupq_n_f64(0.0, 0.0);
+    svfloat64_t b2_f64x = svdupq_n_f64(0.0, 0.0);
+    svbool_t predicate_all_b64x = svptrue_b64();
+    do {
+        svbool_t predicate_b64x = svwhilelt_b64_u64(i, n);
+        svfloat64_t a_f64x = svld1_f64(predicate_b64x, a + i);
+        svfloat64_t b_f64x = svld1_f64(predicate_b64x, b + i);
+        // TwoProd for ab: product = a*b, error = fma(a,b,-product) = -(product - a*b)
+        svfloat64_t product_f64x = svmul_f64_x(predicate_b64x, a_f64x, b_f64x);
+        svfloat64_t product_error_f64x = svneg_f64_x(predicate_b64x,
+                                                     svnmls_f64_x(predicate_b64x, product_f64x, a_f64x, b_f64x));
+        // TwoSum: (tentative_sum, sum_error) = TwoSum(sum, product)
+        svfloat64_t tentative_sum_f64x = svadd_f64_m(predicate_b64x, ab_sum_f64x, product_f64x);
+        svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_b64x, tentative_sum_f64x, ab_sum_f64x);
+        svfloat64_t sum_error_f64x = svadd_f64_x(
+            predicate_b64x,
+            svsub_f64_x(predicate_b64x, ab_sum_f64x,
+                        svsub_f64_x(predicate_b64x, tentative_sum_f64x, virtual_addend_f64x)),
+            svsub_f64_x(predicate_b64x, product_f64x, virtual_addend_f64x));
+        ab_sum_f64x = tentative_sum_f64x;
+        ab_compensation_f64x = svadd_f64_m(predicate_b64x, ab_compensation_f64x,
+                                           svadd_f64_x(predicate_b64x, sum_error_f64x, product_error_f64x));
+        // Simple FMA for self-products (no cancellation)
+        a2_f64x = svmla_f64_m(predicate_b64x, a2_f64x, a_f64x, a_f64x);
+        b2_f64x = svmla_f64_m(predicate_b64x, b2_f64x, b_f64x, b_f64x);
+        i += svcntd();
+    } while (i < n);
+
+    nk_f64_t ab_f64 = nk_dot_stable_sum_f64_sve_(predicate_all_b64x, ab_sum_f64x, ab_compensation_f64x);
+    nk_f64_t a2_f64 = nk_svaddv_f64_(predicate_all_b64x, a2_f64x);
+    nk_f64_t b2_f64 = nk_svaddv_f64_(predicate_all_b64x, b2_f64x);
+    *result = nk_angular_normalize_f64_neon_(ab_f64, a2_f64, b2_f64);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVE
+#endif // NK_TARGET_ARM64_
+#endif // NK_SPATIAL_SVE_H

package/numkong/include/numkong/spatial/svebfdot.h

@@ -0,0 +1,125 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for SVE BF16.
+ *  @file include/numkong/spatial/sve.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  @section spatial_svebfdot_instructions ARM SVE+BF16 Instructions
+ *
+ *      Intrinsic      Instruction                V1
+ *      svld1_bf16     LD1H (Z.H, P/Z, [Xn])      4-6cy @ 2p
+ *      svld1_u16      LD1H (Z.H, P/Z, [Xn])      4-6cy @ 2p
+ *      svbfdot_f32    BFDOT (Z.S, Z.H, Z.H)      4cy @ 2p
+ *      svmla_f32_x    FMLA (Z.S, P/M, Z.S, Z.S)  4cy @ 2p
+ *      svsub_f32_x    FSUB (Z.S, P/M, Z.S, Z.S)  3cy @ 2p
+ *      svaddv_f32     FADDV (S, P, Z.S)          6cy @ 1p
+ *      svunpklo_u32   UUNPKLO (Z.S, Z.H)         2cy @ 2p
+ *      svunpkhi_u32   UUNPKHI (Z.S, Z.H)         2cy @ 2p
+ *      svlsl_n_u32_x  LSL (Z.S, P/M, Z.S, #imm)  2cy @ 2p
+ *      svwhilelt_b16  WHILELT (P.H, Xn, Xm)      2cy @ 1p
+ *      svwhilelt_b32  WHILELT (P.S, Xn, Xm)      2cy @ 1p
+ *      svcnth         CNTH (Xd)                  1cy @ 2p
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  The BFDOT instruction fuses two BF16 multiplications with FP32 accumulation, providing
+ *  efficient BF16 dot products without explicit conversion overhead.
+ */
+#ifndef NK_SPATIAL_SVEBFDOT_H
+#define NK_SPATIAL_SVEBFDOT_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SVEBFDOT
+
+#include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
+#include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve+bf16")
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_bf16_svebfdot(nk_bf16_t const *a_enum, nk_bf16_t const *b_enum, nk_size_t n,
+                                            nk_f32_t *result) {
+    nk_size_t i = 0;
+    svfloat32_t d2_low_f32x = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
+    svfloat32_t d2_high_f32x = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
+    nk_u16_t const *a = (nk_u16_t const *)(a_enum);
+    nk_u16_t const *b = (nk_u16_t const *)(b_enum);
+    do {
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svuint16_t a_u16x = svld1_u16(predicate_b16x, a + i);
+        svuint16_t b_u16x = svld1_u16(predicate_b16x, b + i);
+
+        // There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
+        svbool_t predicate_low_b32x = svwhilelt_b32_u64(i, n);
+        svbool_t predicate_high_b32x = svwhilelt_b32_u64(i + svcnth() / 2, n);
+        svfloat32_t a_low_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_low_b32x, svunpklo_u32(a_u16x), 16));
+        svfloat32_t a_high_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_high_b32x, svunpkhi_u32(a_u16x), 16));
+        svfloat32_t b_low_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_low_b32x, svunpklo_u32(b_u16x), 16));
+        svfloat32_t b_high_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_high_b32x, svunpkhi_u32(b_u16x), 16));
+
+        svfloat32_t a_minus_b_low_f32x = svsub_f32_x(predicate_low_b32x, a_low_f32x, b_low_f32x);
+        svfloat32_t a_minus_b_high_f32x = svsub_f32_x(predicate_high_b32x, a_high_f32x, b_high_f32x);
+        d2_low_f32x = svmla_f32_m(predicate_low_b32x, d2_low_f32x, a_minus_b_low_f32x, a_minus_b_low_f32x);
+        d2_high_f32x = svmla_f32_m(predicate_high_b32x, d2_high_f32x, a_minus_b_high_f32x, a_minus_b_high_f32x);
+        i += svcnth();
+    } while (i < n);
+    nk_f32_t d2_low = nk_svaddv_f32_(svptrue_b32(), d2_low_f32x);
+    nk_f32_t d2_high = nk_svaddv_f32_(svptrue_b32(), d2_high_f32x);
+    nk_f32_t d2 = d2_low + d2_high;
+    *result = d2;
+}
+NK_PUBLIC void nk_euclidean_bf16_svebfdot(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_bf16_svebfdot(a, b, n, result);
+    *result = nk_f32_sqrt_neon(*result);
+}
+
+NK_PUBLIC void nk_angular_bf16_svebfdot(nk_bf16_t const *a_enum, nk_bf16_t const *b_enum, nk_size_t n,
+                                        nk_f32_t *result) {
+    nk_size_t i = 0;
+    svfloat32_t ab_f32x = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
+    svfloat32_t a2_f32x = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
+    svfloat32_t b2_f32x = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
+    nk_bf16_for_arm_simd_t const *a = (nk_bf16_for_arm_simd_t const *)(a_enum);
+    nk_bf16_for_arm_simd_t const *b = (nk_bf16_for_arm_simd_t const *)(b_enum);
+    do {
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svbfloat16_t a_bf16x = svld1_bf16(predicate_b16x, a + i);
+        svbfloat16_t b_bf16x = svld1_bf16(predicate_b16x, b + i);
+        ab_f32x = svbfdot_f32(ab_f32x, a_bf16x, b_bf16x);
+        a2_f32x = svbfdot_f32(a2_f32x, a_bf16x, a_bf16x);
+        b2_f32x = svbfdot_f32(b2_f32x, b_bf16x, b_bf16x);
+        i += svcnth();
+    } while (i < n);
+
+    nk_f32_t ab = nk_svaddv_f32_(svptrue_b32(), ab_f32x);
+    nk_f32_t a2 = nk_svaddv_f32_(svptrue_b32(), a2_f32x);
+    nk_f32_t b2 = nk_svaddv_f32_(svptrue_b32(), b2_f32x);
+    *result = nk_angular_normalize_f32_neon_(ab, a2, b2);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVEBFDOT
+#endif // NK_TARGET_ARM64_
+#endif // NK_SPATIAL_SVEBFDOT_H

package/numkong/include/numkong/spatial/svehalf.h

@@ -0,0 +1,136 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for SVE FP16.
+ *  @file include/numkong/spatial/sve.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  @section spatial_svehalf_instructions ARM SVE+FP16 Instructions
+ *
+ *      Intrinsic      Instruction                V1
+ *      svld1_f16      LD1H (Z.H, P/Z, [Xn])      4-6cy @ 2p
+ *      svsub_f16_x    FSUB (Z.H, P/M, Z.H, Z.H)  3cy @ 2p
+ *      svmla_f16_x    FMLA (Z.H, P/M, Z.H, Z.H)  4cy @ 2p
+ *      svaddv_f16     FADDV (H, P, Z.H)          6cy @ 1p
+ *      svdupq_n_f16   DUP (Z.H, #imm)            1cy @ 2p
+ *      svwhilelt_b16  WHILELT (P.H, Xn, Xm)      2cy @ 1p
+ *      svptrue_b16    PTRUE (P.H, pattern)       1cy @ 2p
+ *      svcnth         CNTH (Xd)                  1cy @ 2p
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  FP16 spatial operations trade precision for throughput, processing twice as many elements
+ *  per cycle. This is particularly effective for embedding similarity in ML applications.
+ */
+#ifndef NK_SPATIAL_SVEHALF_H
+#define NK_SPATIAL_SVEHALF_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SVEHALF
+
+#include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
+#include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve+fp16")
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_f16_svehalf(nk_f16_t const *a_enum, nk_f16_t const *b_enum, nk_size_t n,
+                                          nk_f32_t *result) {
+    nk_size_t i = 0;
+    svfloat32_t d2_f32x = svdup_n_f32(0.0f);
+    nk_f16_for_arm_simd_t const *a = (nk_f16_for_arm_simd_t const *)(a_enum);
+    nk_f16_for_arm_simd_t const *b = (nk_f16_for_arm_simd_t const *)(b_enum);
+    do {
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svfloat16_t a_f16x = svld1_f16(predicate_b16x, a + i);
+        svfloat16_t b_f16x = svld1_f16(predicate_b16x, b + i);
+        nk_size_t remaining = n - i < svcnth() ? n - i : svcnth();
+
+        // SVE `svcvt_f32_f16_x` widens only even-indexed f16 elements (0, 2, 4, ...),
+        // so we need two passes: one on the original vector (even elements) and one on
+        // a vector shifted by one position via `svext` (odd elements become even).
+        svbool_t pred_even_b32x = svwhilelt_b32_u64(0u, (remaining + 1) / 2);
+        svfloat32_t a_even_f32x = svcvt_f32_f16_x(pred_even_b32x, a_f16x);
+        svfloat32_t b_even_f32x = svcvt_f32_f16_x(pred_even_b32x, b_f16x);
+        svfloat32_t diff_even_f32x = svsub_f32_x(pred_even_b32x, a_even_f32x, b_even_f32x);
+        d2_f32x = svmla_f32_m(pred_even_b32x, d2_f32x, diff_even_f32x, diff_even_f32x);
+
+        svbool_t pred_odd_b32x = svwhilelt_b32_u64(0u, remaining / 2);
+        svfloat32_t a_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(a_f16x, a_f16x, 1));
+        svfloat32_t b_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(b_f16x, b_f16x, 1));
+        svfloat32_t diff_odd_f32x = svsub_f32_x(pred_odd_b32x, a_odd_f32x, b_odd_f32x);
+        d2_f32x = svmla_f32_m(pred_odd_b32x, d2_f32x, diff_odd_f32x, diff_odd_f32x);
+
+        i += svcnth();
+    } while (i < n);
+    *result = nk_svaddv_f32_(svptrue_b32(), d2_f32x);
+}
+
+NK_PUBLIC void nk_euclidean_f16_svehalf(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_f16_svehalf(a, b, n, result);
+    *result = nk_f32_sqrt_neon(*result);
+}
+
+NK_PUBLIC void nk_angular_f16_svehalf(nk_f16_t const *a_enum, nk_f16_t const *b_enum, nk_size_t n, nk_f32_t *result) {
+    nk_size_t i = 0;
+    svfloat32_t ab_f32x = svdup_n_f32(0.0f);
+    svfloat32_t a2_f32x = svdup_n_f32(0.0f);
+    svfloat32_t b2_f32x = svdup_n_f32(0.0f);
+    nk_f16_for_arm_simd_t const *a = (nk_f16_for_arm_simd_t const *)(a_enum);
+    nk_f16_for_arm_simd_t const *b = (nk_f16_for_arm_simd_t const *)(b_enum);
+    do {
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svfloat16_t a_f16x = svld1_f16(predicate_b16x, a + i);
+        svfloat16_t b_f16x = svld1_f16(predicate_b16x, b + i);
+        nk_size_t remaining = n - i < svcnth() ? n - i : svcnth();
+
+        // Even-indexed f16 elements (0, 2, 4, ...)
+        svbool_t pred_even_b32x = svwhilelt_b32_u64(0u, (remaining + 1) / 2);
+        svfloat32_t a_even_f32x = svcvt_f32_f16_x(pred_even_b32x, a_f16x);
+        svfloat32_t b_even_f32x = svcvt_f32_f16_x(pred_even_b32x, b_f16x);
+        ab_f32x = svmla_f32_m(pred_even_b32x, ab_f32x, a_even_f32x, b_even_f32x);
+        a2_f32x = svmla_f32_m(pred_even_b32x, a2_f32x, a_even_f32x, a_even_f32x);
+        b2_f32x = svmla_f32_m(pred_even_b32x, b2_f32x, b_even_f32x, b_even_f32x);
+
+        // Odd-indexed f16 elements (1, 3, 5, ...) via svext shift-by-1
+        svbool_t pred_odd_b32x = svwhilelt_b32_u64(0u, remaining / 2);
+        svfloat32_t a_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(a_f16x, a_f16x, 1));
+        svfloat32_t b_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(b_f16x, b_f16x, 1));
+        ab_f32x = svmla_f32_m(pred_odd_b32x, ab_f32x, a_odd_f32x, b_odd_f32x);
+        a2_f32x = svmla_f32_m(pred_odd_b32x, a2_f32x, a_odd_f32x, a_odd_f32x);
+        b2_f32x = svmla_f32_m(pred_odd_b32x, b2_f32x, b_odd_f32x, b_odd_f32x);
+
+        i += svcnth();
+    } while (i < n);
+
+    nk_f32_t ab_f32 = nk_svaddv_f32_(svptrue_b32(), ab_f32x);
+    nk_f32_t a2_f32 = nk_svaddv_f32_(svptrue_b32(), a2_f32x);
+    nk_f32_t b2_f32 = nk_svaddv_f32_(svptrue_b32(), b2_f32x);
+    *result = nk_angular_normalize_f32_neon_(ab_f32, a2_f32, b2_f32);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVEHALF
+#endif // NK_TARGET_ARM64_
+#endif // NK_SPATIAL_SVEHALF_H

package/numkong/include/numkong/spatial/svesdot.h

@@ -0,0 +1,143 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for SVE SDOT.
+ *  @file include/numkong/spatial/svesdot.h
+ *  @author Ash Vardanian
+ *  @date April 3, 2026
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  @section spatial_svesdot_instructions ARM SVE+DotProd Instructions
+ *
+ *      Intrinsic      Instruction              V1
+ *      svld1_s8       LD1B (Z.B, P/Z, [Xn])    4-6cy @ 2p
+ *      svld1_u8       LD1B (Z.B, P/Z, [Xn])    4-6cy @ 2p
+ *      svdot_s32      SDOT (Z.S, Z.B, Z.B)     3cy @ 2p
+ *      svdot_u32      UDOT (Z.S, Z.B, Z.B)     3cy @ 2p
+ *      svabd_s8_x     SABD (Z.B, P/M, Z.B)     3cy @ 2p
+ *      svabd_u8_x     UABD (Z.B, P/M, Z.B)     3cy @ 2p
+ *      svaddv_s32     SADDV (D, P, Z.S)        6cy @ 1p
+ *      svaddv_u32     UADDV (D, P, Z.S)        6cy @ 1p
+ *      svwhilelt_b8   WHILELT (P.B, Xn, Xm)    2cy @ 1p
+ *      svcntb         CNTB (Xd)                1cy @ 2p
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  For L2 distance, SABD/UABD computes |a-b| per byte, then UDOT squares and accumulates.
+ *  Angular distance uses SDOT/UDOT directly for dot product and norm computations.
+ */
+#ifndef NK_SPATIAL_SVESDOT_H
+#define NK_SPATIAL_SVESDOT_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SVESDOT
+
+#include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
+#include "numkong/spatial/neon.h" // `nk_angular_normalize_f32_neon_`, `nk_f32_sqrt_neon`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+dotprod"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve+dotprod")
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_u32_t *result) {
+    nk_size_t i = 0;
+    svuint32_t distance_sq_u32x = svdup_u32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svint8_t a_i8x = svld1_s8(predicate_b8x, a + i);
+        svint8_t b_i8x = svld1_s8(predicate_b8x, b + i);
+        svuint8_t diff_u8x = svreinterpret_u8_s8(svabd_s8_x(predicate_b8x, a_i8x, b_i8x));
+        distance_sq_u32x = svdot_u32(distance_sq_u32x, diff_u8x, diff_u8x);
+        i += svcntb();
+    } while (i < n);
+    *result = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), distance_sq_u32x);
+}
+NK_PUBLIC void nk_euclidean_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_u32_t distance_sq_u32;
+    nk_sqeuclidean_i8_svesdot(a, b, n, &distance_sq_u32);
+    *result = nk_f32_sqrt_neon((nk_f32_t)distance_sq_u32);
+}
+
+NK_PUBLIC void nk_angular_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_size_t i = 0;
+    svint32_t ab_i32x = svdup_s32(0);
+    svint32_t a2_i32x = svdup_s32(0);
+    svint32_t b2_i32x = svdup_s32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svint8_t a_i8x = svld1_s8(predicate_b8x, a + i);
+        svint8_t b_i8x = svld1_s8(predicate_b8x, b + i);
+        ab_i32x = svdot_s32(ab_i32x, a_i8x, b_i8x);
+        a2_i32x = svdot_s32(a2_i32x, a_i8x, a_i8x);
+        b2_i32x = svdot_s32(b2_i32x, b_i8x, b_i8x);
+        i += svcntb();
+    } while (i < n);
+
+    nk_i32_t ab = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), ab_i32x);
+    nk_i32_t a2 = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), a2_i32x);
+    nk_i32_t b2 = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), b2_i32x);
+    *result = nk_angular_normalize_f32_neon_((nk_f32_t)ab, (nk_f32_t)a2, (nk_f32_t)b2);
+}
+
+NK_PUBLIC void nk_sqeuclidean_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u32_t *result) {
+    nk_size_t i = 0;
+    svuint32_t distance_sq_u32x = svdup_u32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svuint8_t a_u8x = svld1_u8(predicate_b8x, a + i);
+        svuint8_t b_u8x = svld1_u8(predicate_b8x, b + i);
+        svuint8_t diff_u8x = svabd_u8_x(predicate_b8x, a_u8x, b_u8x);
+        distance_sq_u32x = svdot_u32(distance_sq_u32x, diff_u8x, diff_u8x);
+        i += svcntb();
+    } while (i < n);
+    *result = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), distance_sq_u32x);
+}
+NK_PUBLIC void nk_euclidean_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_u32_t distance_sq_u32;
+    nk_sqeuclidean_u8_svesdot(a, b, n, &distance_sq_u32);
+    *result = nk_f32_sqrt_neon((nk_f32_t)distance_sq_u32);
+}
+
+NK_PUBLIC void nk_angular_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_size_t i = 0;
+    svuint32_t ab_u32x = svdup_u32(0);
+    svuint32_t a2_u32x = svdup_u32(0);
+    svuint32_t b2_u32x = svdup_u32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svuint8_t a_u8x = svld1_u8(predicate_b8x, a + i);
+        svuint8_t b_u8x = svld1_u8(predicate_b8x, b + i);
+        ab_u32x = svdot_u32(ab_u32x, a_u8x, b_u8x);
+        a2_u32x = svdot_u32(a2_u32x, a_u8x, a_u8x);
+        b2_u32x = svdot_u32(b2_u32x, b_u8x, b_u8x);
+        i += svcntb();
+    } while (i < n);
+
+    nk_u32_t ab = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), ab_u32x);
+    nk_u32_t a2 = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), a2_u32x);
+    nk_u32_t b2 = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), b2_u32x);
+    *result = nk_angular_normalize_f32_neon_((nk_f32_t)ab, (nk_f32_t)a2, (nk_f32_t)b2);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVESDOT
+#endif // NK_TARGET_ARM64_
+#endif // NK_SPATIAL_SVESDOT_H