usearch 2.9.0 → 2.9.2

package/simsimd/include/simsimd/complex.h

@@ -0,0 +1,174 @@
+/**
+ *  @brief      SIMD-accelerated Similarity Measures for Complex Vectors.
+ *  @author     Ash Vardanian
+ *  @date       February 24, 2024
+ *
+ *  Contains:
+ *  - Inner Product
+ *  - Cosine Distance
+ *
+ *  For datatypes:
+ *  - 32-bit floating point numbers
+ *  - 16-bit floating point numbers
+ *
+ *  For hardware architectures:
+ *  - Arm (NEON, SVE?)
+ *  - x86 (AVX2?, AVX512?)
+ *
+ *  x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
+ *  Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
+ */
+#ifndef SIMSIMD_COMPLEX_H
+#define SIMSIMD_COMPLEX_H
+
+#include "types.h"
+
+#define SIMSIMD_MAKE_COMPLEX_IP(name, input_type, accumulator_type, converter, epsilon)                                \
+    inline static simsimd_f32_t simsimd_##name##_##input_type##_complex_ip(                                            \
+        simsimd_##input_type##_t const* a, simsimd_##input_type##_t const* b, simsimd_size_t n,                        \
+        simsimd_f32_t* real_out, simsimd_f32_t* imag_out) {                                                            \
+        simsimd_##accumulator_type##_t ab_real, ab_imag;                                                               \
+        for (simsimd_size_t i = 0; i + 2 <= n; i += 2) {                                                               \
+            simsimd_##accumulator_type##_t ar = converter(a[i]);                                                       \
+            simsimd_##accumulator_type##_t br = converter(b[i]);                                                       \
+            simsimd_##accumulator_type##_t ai = converter(a[i + 1]);                                                   \
+            simsimd_##accumulator_type##_t bi = converter(b[i + 1]);                                                   \
+            ab_real += ar * br + ai * bi;                                                                              \
+            ab_imag += ai * br - ar * bi;                                                                              \
+        }                                                                                                              \
+        *real_out = ab_real;                                                                                           \
+        *imag_out = ab_imag;                                                                                           \
+        return 1 - ab_real;                                                                                            \
+    }
+
+#define SIMSIMD_MAKE_COMPLEX_COS(name, input_type, accumulator_type, converter, epsilon)                               \
+    inline static simsimd_f32_t simsimd_##name##_##input_type##_complex_cos(                                           \
+        simsimd_##input_type##_t const* a, simsimd_##input_type##_t const* b, simsimd_size_t n,                        \
+        simsimd_f32_t* real_out, simsimd_f32_t* imag_out) {                                                            \
+        simsimd_##accumulator_type##_t ab_real, ab_imag, a2_sum, b2_sum;                                               \
+        for (simsimd_size_t i = 0; i + 2 <= n; i += 2) {                                                               \
+            simsimd_##accumulator_type##_t ar = converter(a[i]);                                                       \
+            simsimd_##accumulator_type##_t br = converter(b[i]);                                                       \
+            simsimd_##accumulator_type##_t ai = converter(a[i + 1]);                                                   \
+            simsimd_##accumulator_type##_t bi = converter(b[i + 1]);                                                   \
+            ab_real += ar * br + ai * bi;                                                                              \
+            ab_imag += ai * br - ar * bi;                                                                              \
+            a2_sum += ar * ar + ai * ai;                                                                               \
+            b2_sum += br * br + bi * bi;                                                                               \
+        }                                                                                                              \
+        *real_out = ab_real;                                                                                           \
+        *imag_out = ab_imag;                                                                                           \
+        simsimd_##accumulator_type##_t ab_angle = ab_real * SIMSIMD_RSQRT(a2_sum) * SIMSIMD_RSQRT(b2_sum);             \
+        return 1 - ab_angle;                                                                                           \
+    }
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+SIMSIMD_MAKE_COMPLEX_IP(serial, f64, f64, SIMSIMD_IDENTIFY,
+                        SIMSIMD_F32_DIVISION_EPSILON) // simsimd_serial_f64_complex_ip
+SIMSIMD_MAKE_COMPLEX_COS(serial, f64, f64, SIMSIMD_IDENTIFY,
+                         SIMSIMD_F32_DIVISION_EPSILON) // simsimd_serial_f64_complex_cos
+
+SIMSIMD_MAKE_COMPLEX_IP(serial, f32, f32, SIMSIMD_IDENTIFY,
+                        SIMSIMD_F32_DIVISION_EPSILON) // simsimd_serial_f32_complex_ip
+SIMSIMD_MAKE_COMPLEX_COS(serial, f32, f32, SIMSIMD_IDENTIFY,
+                         SIMSIMD_F32_DIVISION_EPSILON) // simsimd_serial_f32_complex_cos
+
+SIMSIMD_MAKE_COMPLEX_IP(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16,
+                        SIMSIMD_F32_DIVISION_EPSILON) // simsimd_serial_f16_complex_ip
+SIMSIMD_MAKE_COMPLEX_COS(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16,
+                         SIMSIMD_F32_DIVISION_EPSILON) // simsimd_serial_f16_complex_cos
+
+SIMSIMD_MAKE_COMPLEX_IP(accurate, f32, f64, SIMSIMD_IDENTIFY,
+                        SIMSIMD_F32_DIVISION_EPSILON) // simsimd_accurate_f32_complex_ip
+SIMSIMD_MAKE_COMPLEX_COS(accurate, f32, f64, SIMSIMD_IDENTIFY,
+                         SIMSIMD_F32_DIVISION_EPSILON) // simsimd_accurate_f32_complex_cos
+
+SIMSIMD_MAKE_COMPLEX_IP(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16,
+                        SIMSIMD_F32_DIVISION_EPSILON) // simsimd_accurate_f16_complex_ip
+SIMSIMD_MAKE_COMPLEX_COS(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16,
+                         SIMSIMD_F32_DIVISION_EPSILON) // simsimd_accurate_f16_complex_cos
+
+#if SIMSIMD_TARGET_ARM
+#if SIMSIMD_TARGET_ARM_NEON
+
+/*
+ *  @file   arm_neon_f32.h
+ *  @brief  Arm NEON implementation of the most common similarity metrics for 32-bit floating point numbers.
+ *  @author Ash Vardanian
+ *
+ *  - Implements: Kullback-Leibler and Jensen–Shannon divergence.
+ *  - Uses `f32` for storage and `f32` for accumulation.
+ *  - Requires compiler capabilities: +simd.
+ */
+
+__attribute__((target("+simd"))) //
+inline static simsimd_f32_t
+simsimd_neon_f32_complex_cos(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, //
+                             simsimd_f32_t* real_out, simsimd_f32_t* imag_out) {
+    float32x4_t ab_real_vec = vdupq_n_f32(0);
+    float32x4_t ab_imag_vec = vdupq_n_f32(0);
+    float32x4_t a2_vec = vdupq_n_f32(0);
+    float32x4_t b2_vec = vdupq_n_f32(0);
+    simsimd_size_t i = 0;
+    for (; i + 8 <= n; i += 8) {
+        // Unpack the input arrays into real and imaginary parts:
+        // a_real = a[::2]
+        // a_imag = a[1::2]
+        // b_real = b[::2]
+        // b_imag = b[1::2]
+        float32x4x2_t a_vec = vld2q_f32(a + i);
+        float32x4x2_t b_vec = vld2q_f32(b + i);
+        float32x4_t a_real_vec = a_vec.val[0];
+        float32x4_t a_imag_vec = a_vec.val[1];
+        float32x4_t b_real_vec = b_vec.val[0];
+        float32x4_t b_imag_vec = b_vec.val[1];
+
+        // Compute the dot product:
+        // ab_real = sum(ar * br + ai * bi for ar, ai, br, bi in zip(a_real, a_imag, b_real, b_imag))
+        // ab_imag = sum(ai * br - ar * bi for ar, ai, br, bi in zip(a_real, a_imag, b_real, b_imag))
+        ab_real_vec = vfmaq_f32(ab_real_vec, a_real_vec, b_real_vec);
+        ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
+        ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
+        ab_imag_vec = vfmsq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
+
+        // Compute the vector norms:
+        // a2 = sum(ar * ar + ai * ai for ar, ai in zip(a_real, a_imag))
+        // b2 = sum(br * br + bi * bi for br, bi in zip(b_real, b_imag))
+        a2_vec = vfmaq_f32(a2_vec, a_real_vec, a_real_vec);
+        a2_vec = vfmaq_f32(a2_vec, a_imag_vec, a_imag_vec);
+        b2_vec = vfmaq_f32(b2_vec, b_real_vec, b_real_vec);
+        b2_vec = vfmaq_f32(b2_vec, b_imag_vec, b_imag_vec);
+    }
+
+    // Reduce horizontal sums:
+    simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
+    simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
+    simsimd_f32_t a2 = vaddvq_f32(a2_vec);
+    simsimd_f32_t b2 = vaddvq_f32(b2_vec);
+
+    // Handle the tail:
+    for (; i < n; i += 2) {
+        simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
+        ab_real += ar * br + ai * bi, ab_imag += ai * br - ar * bi;
+        a2 += ar * ar + ai * ai, b2 += br * br + bi * bi;
+    }
+    *real_out = ab_real;
+    *imag_out = ab_imag;
+
+    // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
+    simsimd_f32_t a2_b2_arr[2] = {a2, b2};
+    vst1_f32(a2_b2_arr, vrsqrte_f32(vld1_f32(a2_b2_arr)));
+    return ab_real != 0 ? 1 - ab_real * a2_b2_arr[0] * a2_b2_arr[1] : 1;
+}
+
+#endif // SIMSIMD_TARGET_ARM_NEON
+#endif // SIMSIMD_TARGET_ARM
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

package/simsimd/include/simsimd/simsimd.h

@@ -15,10 +15,11 @@
 #define SIMSIMD_H
 
 #define SIMSIMD_VERSION_MAJOR 3
-#define SIMSIMD_VERSION_MINOR 8
+#define SIMSIMD_VERSION_MINOR 9
 #define SIMSIMD_VERSION_PATCH 0
 
 #include "binary.h"      // Hamming, Jaccard
+#include "complex.h"     // Complex numbers
 #include "probability.h" // Kullback-Leibler, Jensen–Shannon
 #include "spatial.h"     // L2, Inner Product, Cosine
 
@@ -388,14 +389,6 @@ inline static void simsimd_find_metric_punned( //
     // Binary vectors
     case simsimd_datatype_b8_k:
 
-    #if SIMSIMD_TARGET_ARM_NEON
-        if (viable & simsimd_cap_arm_neon_k)
-            switch (kind) {
-            case simsimd_metric_hamming_k: *m = (simsimd_metric_punned_t)&simsimd_neon_b8_hamming, *c = simsimd_cap_arm_neon_k; return;
-            case simsimd_metric_jaccard_k: *m = (simsimd_metric_punned_t)&simsimd_neon_b8_jaccard, *c = simsimd_cap_arm_neon_k; return;
-            default: break;
-            }
-    #endif
     #if SIMSIMD_TARGET_ARM_SVE
         if (viable & simsimd_cap_arm_sve_k)
             switch (kind) {
@@ -404,6 +397,14 @@ inline static void simsimd_find_metric_punned( //
             default: break;
             }
     #endif
+    #if SIMSIMD_TARGET_ARM_NEON
+        if (viable & simsimd_cap_arm_neon_k)
+            switch (kind) {
+            case simsimd_metric_hamming_k: *m = (simsimd_metric_punned_t)&simsimd_neon_b8_hamming, *c = simsimd_cap_arm_neon_k; return;
+            case simsimd_metric_jaccard_k: *m = (simsimd_metric_punned_t)&simsimd_neon_b8_jaccard, *c = simsimd_cap_arm_neon_k; return;
+            default: break;
+            }
+    #endif
     #if SIMSIMD_TARGET_X86_AVX512
         if (viable & simsimd_cap_x86_avx512vpopcntdq_k)
             switch (kind) {
@@ -412,6 +413,14 @@ inline static void simsimd_find_metric_punned( //
             default: break;
             }
     #endif
+    #if SIMSIMD_TARGET_X86_AVX2
+        if (viable & simsimd_cap_x86_avx2_k)
+            switch (kind) {
+            case simsimd_metric_hamming_k: *m = (simsimd_metric_punned_t)&simsimd_avx2_b8_hamming, *c = simsimd_cap_x86_avx2_k; return;
+            case simsimd_metric_jaccard_k: *m = (simsimd_metric_punned_t)&simsimd_avx2_b8_jaccard, *c = simsimd_cap_x86_avx2_k; return;
+            default: break;
+            }
+    #endif
 
         if (viable & simsimd_cap_serial_k)
             switch (kind) {

package/simsimd/include/simsimd/spatial.h

@@ -194,11 +194,25 @@ simsimd_neon_f16_l2sq(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_si
         float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
         sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
     }
-    simsimd_f32_t sum = vaddvq_f32(sum_vec);
-    for (; i < n; ++i) {
-        simsimd_f32_t diff = SIMSIMD_UNCOMPRESS_F16(a[i]) - SIMSIMD_UNCOMPRESS_F16(b[i]);
-        sum += diff * diff;
+
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            float16x4_t f16_vec;
+            simsimd_f16_t f16[4];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 4; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        float32x4_t diff_vec = vsubq_f32(vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
+        sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
     }
+
+    simsimd_f32_t sum = vaddvq_f32(sum_vec);
     return sum;
 }
 
@@ -212,9 +226,23 @@ simsimd_neon_f16_ip(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
         float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
         ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
     }
+
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            float16x4_t f16_vec;
+            simsimd_f16_t f16[4];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 4; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        ab_vec = vfmaq_f32(ab_vec, vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
+    }
     simsimd_f32_t ab = vaddvq_f32(ab_vec);
-    for (; i < n; ++i)
-        ab += SIMSIMD_UNCOMPRESS_F16(a[i]) * SIMSIMD_UNCOMPRESS_F16(b[i]);
     return 1 - ab;
 }
 
@@ -230,13 +258,29 @@ simsimd_neon_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_siz
         a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
         b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
     }
-    simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
-    for (; i < n; ++i) {
-        simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]), bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
-        ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
+
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            float16x4_t f16_vec;
+            simsimd_f16_t f16[4];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 4; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        float32x4_t a_vec = vcvt_f32_f16(a_padded_tail.f16_vec);
+        float32x4_t b_vec = vcvt_f32_f16(b_padded_tail.f16_vec);
+        ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
+        a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
+        b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
     }
 
     // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
+    simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
     simsimd_f32_t a2_b2_arr[2] = {a2, b2};
     float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
     a2_b2 = vrsqrte_f32(a2_b2);
@@ -582,18 +626,31 @@ simsimd_avx2_f16_l2sq(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_si
         d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
     }
 
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            __m128i f16_vec;
+            simsimd_f16_t f16[8];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 8; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
+        __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
+        __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
+        d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
+    }
+
     d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
     d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
     d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
 
     simsimd_f32_t result;
     _mm_store_ss(&result, _mm256_castps256_ps128(d2_vec));
-
-    // Accumulate the tail:
-    for (; i < n; ++i) {
-        simsimd_f32_t n = SIMSIMD_UNCOMPRESS_F16(a[i]) - SIMSIMD_UNCOMPRESS_F16(b[i]);
-        result += n * n;
-    }
     return result;
 }
 
@@ -608,18 +665,33 @@ simsimd_avx2_f16_ip(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
         ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
     }
 
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            __m128i f16_vec;
+            simsimd_f16_t f16[8];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 8; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
+        __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
+        ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
+    }
+
     ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
 
     simsimd_f32_t result;
     _mm_store_ss(&result, _mm256_castps256_ps128(ab_vec));
-
-    // Accumulate the tail:
-    for (; i < n; ++i)
-        result += SIMSIMD_UNCOMPRESS_F16(a[i]) * SIMSIMD_UNCOMPRESS_F16(b[i]);
     return 1 - result;
 }
+
 __attribute__((target("avx2,f16c,fma"))) //
 inline static simsimd_f32_t
 simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
@@ -634,6 +706,27 @@ simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_siz
         b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
     }
 
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            __m128i f16_vec;
+            simsimd_f16_t f16[8];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 8; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
+        __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
+        ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
+        a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
+        b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
+    }
+
+    // Horizontal reductions:
     ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
@@ -651,12 +744,6 @@ simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_siz
     _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
     _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
 
-    // Accumulate the tail:
-    for (; i < n; ++i) {
-        simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]), bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
-        ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
-    }
-
     // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
     __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
     __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
@@ -1037,19 +1124,24 @@ simsimd_avx512_i8_cos(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size
     __m512i a_vec, b_vec;
 
 simsimd_avx512_i8_cos_cycle:
-    if (n < 64) {
-        __mmask64 mask = _bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
-        a_vec = _mm512_maskz_loadu_epi8(mask, a);
-        b_vec = _mm512_maskz_loadu_epi8(mask, b);
+    if (n < 32) {
+        __mmask32 mask = _bzhi_u32(0xFFFFFFFF, n);
+        a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
+        b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
         n = 0;
     } else {
-        a_vec = _mm512_loadu_epi8(a);
-        b_vec = _mm512_loadu_epi8(b);
-        a += 64, b += 64, n -= 64;
+        a_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(a));
+        b_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(b));
+        a += 32, b += 32, n -= 32;
     }
-    ab_i32s_vec = _mm512_dpbusd_epi32(ab_i32s_vec, a_vec, b_vec);
-    a2_i32s_vec = _mm512_dpbusd_epi32(a2_i32s_vec, a_vec, a_vec);
-    b2_i32s_vec = _mm512_dpbusd_epi32(b2_i32s_vec, b_vec, b_vec);
+    // Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
+    // as it's assymetric with respect to the sign of the input arguments:
+    //      Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
+    // So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
+    // to 16-bit beforehand.
+    ab_i32s_vec = _mm512_dpwssd_epi32(ab_i32s_vec, a_vec, b_vec);
+    a2_i32s_vec = _mm512_dpwssd_epi32(a2_i32s_vec, a_vec, a_vec);
+    b2_i32s_vec = _mm512_dpwssd_epi32(b2_i32s_vec, b_vec, b_vec);
     if (n)
         goto simsimd_avx512_i8_cos_cycle;
 

package/simsimd/include/simsimd/types.h

@@ -84,11 +84,11 @@
 #endif
 
 #ifndef SIMSIMD_F32_DIVISION_EPSILON
-#define SIMSIMD_F32_DIVISION_EPSILON 1e-7
+#define SIMSIMD_F32_DIVISION_EPSILON (1e-7)
 #endif
 
 #ifndef SIMSIMD_F16_DIVISION_EPSILON
-#define SIMSIMD_F16_DIVISION_EPSILON 1e-3
+#define SIMSIMD_F16_DIVISION_EPSILON (1e-3)
 #endif
 
 #ifdef __cplusplus
@@ -100,6 +100,7 @@ typedef float simsimd_f32_t;
 typedef double simsimd_f64_t;
 typedef signed char simsimd_i8_t;
 typedef unsigned char simsimd_b8_t;
+typedef unsigned long long simsimd_u64_t;
 typedef unsigned long long simsimd_size_t;
 
 #if !defined(SIMSIMD_NATIVE_F16) || SIMSIMD_NATIVE_F16

package/simsimd/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "simsimd",
-  "version": "3.6.4",
+  "version": "3.8.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "simsimd",
-      "version": "3.6.4",
+      "version": "3.8.1",
       "hasInstallScript": true,
       "license": "Apache 2.0",
       "dependencies": {
@@ -627,9 +627,9 @@
       "dev": true
     },
     "node_modules/ip": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
-      "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz",
+      "integrity": "sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==",
       "dev": true
     },
     "node_modules/is-fullwidth-code-point": {

package/simsimd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simsimd",
-  "version": "3.8.0",
+  "version": "3.9.0",
   "description": "Fastest SIMD-Accelerated Vector Similarity Functions for x86 and Arm",
   "homepage": "https://github.com/ashvardanian/simsimd",
   "author": "Ash Vardanian",

package/simsimd/rust/benches/native.rs

@@ -10,18 +10,21 @@ pub(crate) fn cosine_similarity_cpu(a: &[f32], b: &[f32]) -> Option<f32> {
         return None;
     }
 
-    let dot_product: f32 = a.iter().zip(b.iter()).map(|(a, b)| a * b).sum();
-    let norm_a: f32 = a.iter().map(|a| a.powf(2.0)).sum();
-    let norm_b: f32 = b.iter().map(|b| b.powf(2.0)).sum();
+    let (dot_product, norm_a, norm_b) = a
+        .iter()
+        .zip(b)
+        .map(|(a, b)| (a * b, a * a, b * b))
+        .fold((0.0, 0.0, 0.0), |acc, x| {
+            (acc.0 + x.0, acc.1 + x.1, acc.2 + x.2)
+        });
 
     Some(1.0 - (dot_product / (norm_a.sqrt() * norm_b.sqrt())))
 }
 
-pub(crate) fn squared_euclidean_cpu(a: &[f32], b: &[f32]) -> f32 {
-    assert_eq!(a.len(), b.len());
+pub(crate) fn squared_euclidean_cpu(a: &[f32], b: &[f32]) -> Option<f32> {
+    if a.len() != b.len() {
+        return None;
+    }
 
-    a.iter()
-        .zip(b.iter())
-        .map(|(x, y)| ((*x) - (*y)) * ((*x) - (*y)))
-        .fold(0.0, ::std::ops::Add::add)
+    Some(a.iter().zip(b).map(|(x, y)| (x - y).powi(2)).sum())
 }

package/simsimd/rust/lib.c

@@ -7,62 +7,39 @@ simsimd_capability_t cached_capabilities(void) {
     return static_capabilities;
 }
 
-simsimd_f32_t cosine_i8(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_cosine_k, simsimd_datatype_i8_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
+#define SIMSIMD_METRIC_DECLARATION(name, type)                                                                         \
+    simsimd_f32_t name##_##type(simsimd_##type##_t const* a, simsimd_##type##_t const* b, simsimd_size_t d) {          \
+        static simsimd_metric_punned_t metric = 0;                                                                     \
+        if (metric == 0) {                                                                                             \
+            simsimd_capability_t used_capability;                                                                      \
+            simsimd_find_metric_punned(simsimd_metric_##name##_k, simsimd_datatype_##type##_k, cached_capabilities(),  \
+                                       simsimd_cap_any_k, &metric, &used_capability);                                  \
+        }                                                                                                              \
+        return metric(a, b, d, d);                                                                                     \
     }
-    return metric(a, b, d, d);
-}
 
-simsimd_f32_t cosine_f32(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_cosine_k, simsimd_datatype_f32_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
+// Spatial distances
+SIMSIMD_METRIC_DECLARATION(cosine, i8)
+SIMSIMD_METRIC_DECLARATION(cosine, f16)
+SIMSIMD_METRIC_DECLARATION(cosine, f32)
+SIMSIMD_METRIC_DECLARATION(cosine, f64)
+SIMSIMD_METRIC_DECLARATION(inner, i8)
+SIMSIMD_METRIC_DECLARATION(inner, f16)
+SIMSIMD_METRIC_DECLARATION(inner, f32)
+SIMSIMD_METRIC_DECLARATION(inner, f64)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, i8)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, f16)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, f32)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, f64)
 
-simsimd_f32_t inner_i8(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_inner_k, simsimd_datatype_i8_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
+// Binary distances
+SIMSIMD_METRIC_DECLARATION(hamming, b8)
+SIMSIMD_METRIC_DECLARATION(jaccard, b8)
 
-simsimd_f32_t inner_f32(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_inner_k, simsimd_datatype_f32_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
-
-simsimd_f32_t sqeuclidean_i8(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_sqeuclidean_k, simsimd_datatype_i8_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
-
-simsimd_f32_t sqeuclidean_f32(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_sqeuclidean_k, simsimd_datatype_f32_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
+// Probability distributions
+SIMSIMD_METRIC_DECLARATION(kl, f16)
+SIMSIMD_METRIC_DECLARATION(kl, f32)
+SIMSIMD_METRIC_DECLARATION(kl, f64)
+SIMSIMD_METRIC_DECLARATION(js, f16)
+SIMSIMD_METRIC_DECLARATION(js, f32)
+SIMSIMD_METRIC_DECLARATION(js, f64)