usearch 2.9.1 → 2.9.2

package/simsimd/include/simsimd/spatial.h

@@ -194,11 +194,25 @@ simsimd_neon_f16_l2sq(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_si
         float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
         sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
     }
-    simsimd_f32_t sum = vaddvq_f32(sum_vec);
-    for (; i < n; ++i) {
-        simsimd_f32_t diff = SIMSIMD_UNCOMPRESS_F16(a[i]) - SIMSIMD_UNCOMPRESS_F16(b[i]);
-        sum += diff * diff;
+
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            float16x4_t f16_vec;
+            simsimd_f16_t f16[4];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 4; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        float32x4_t diff_vec = vsubq_f32(vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
+        sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
     }
+
+    simsimd_f32_t sum = vaddvq_f32(sum_vec);
     return sum;
 }
 
@@ -212,9 +226,23 @@ simsimd_neon_f16_ip(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
         float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
         ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
     }
+
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            float16x4_t f16_vec;
+            simsimd_f16_t f16[4];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 4; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        ab_vec = vfmaq_f32(ab_vec, vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
+    }
     simsimd_f32_t ab = vaddvq_f32(ab_vec);
-    for (; i < n; ++i)
-        ab += SIMSIMD_UNCOMPRESS_F16(a[i]) * SIMSIMD_UNCOMPRESS_F16(b[i]);
     return 1 - ab;
 }
 
@@ -230,13 +258,29 @@ simsimd_neon_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_siz
         a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
         b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
     }
-    simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
-    for (; i < n; ++i) {
-        simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]), bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
-        ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
+
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            float16x4_t f16_vec;
+            simsimd_f16_t f16[4];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 4; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        float32x4_t a_vec = vcvt_f32_f16(a_padded_tail.f16_vec);
+        float32x4_t b_vec = vcvt_f32_f16(b_padded_tail.f16_vec);
+        ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
+        a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
+        b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
     }
 
     // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
+    simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
     simsimd_f32_t a2_b2_arr[2] = {a2, b2};
     float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
     a2_b2 = vrsqrte_f32(a2_b2);
@@ -582,18 +626,31 @@ simsimd_avx2_f16_l2sq(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_si
         d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
     }
 
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            __m128i f16_vec;
+            simsimd_f16_t f16[8];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 8; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
+        __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
+        __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
+        d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
+    }
+
     d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
     d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
     d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
 
     simsimd_f32_t result;
     _mm_store_ss(&result, _mm256_castps256_ps128(d2_vec));
-
-    // Accumulate the tail:
-    for (; i < n; ++i) {
-        simsimd_f32_t n = SIMSIMD_UNCOMPRESS_F16(a[i]) - SIMSIMD_UNCOMPRESS_F16(b[i]);
-        result += n * n;
-    }
     return result;
 }
 
@@ -608,18 +665,33 @@ simsimd_avx2_f16_ip(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
         ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
     }
 
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            __m128i f16_vec;
+            simsimd_f16_t f16[8];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 8; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
+        __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
+        ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
+    }
+
     ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
 
     simsimd_f32_t result;
     _mm_store_ss(&result, _mm256_castps256_ps128(ab_vec));
-
-    // Accumulate the tail:
-    for (; i < n; ++i)
-        result += SIMSIMD_UNCOMPRESS_F16(a[i]) * SIMSIMD_UNCOMPRESS_F16(b[i]);
     return 1 - result;
 }
+
 __attribute__((target("avx2,f16c,fma"))) //
 inline static simsimd_f32_t
 simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
@@ -634,6 +706,27 @@ simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_siz
         b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
     }
 
+    // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
+    // function will run. It is extremely slow, so even for the tail, let's combine serial
+    // loads and stores with vectorized math.
+    if (i < n) {
+        union {
+            __m128i f16_vec;
+            simsimd_f16_t f16[8];
+        } a_padded_tail, b_padded_tail;
+        simsimd_size_t j = 0;
+        for (; i < n; ++i, ++j)
+            a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
+        for (; j < 8; ++j)
+            a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
+        __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
+        __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
+        ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
+        a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
+        b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
+    }
+
+    // Horizontal reductions:
     ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
     ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
@@ -651,12 +744,6 @@ simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_siz
     _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
     _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
 
-    // Accumulate the tail:
-    for (; i < n; ++i) {
-        simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]), bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
-        ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
-    }
-
     // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
     __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
     __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
@@ -1037,19 +1124,24 @@ simsimd_avx512_i8_cos(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size
     __m512i a_vec, b_vec;
 
 simsimd_avx512_i8_cos_cycle:
-    if (n < 64) {
-        __mmask64 mask = _bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
-        a_vec = _mm512_maskz_loadu_epi8(mask, a);
-        b_vec = _mm512_maskz_loadu_epi8(mask, b);
+    if (n < 32) {
+        __mmask32 mask = _bzhi_u32(0xFFFFFFFF, n);
+        a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
+        b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
         n = 0;
     } else {
-        a_vec = _mm512_loadu_epi8(a);
-        b_vec = _mm512_loadu_epi8(b);
-        a += 64, b += 64, n -= 64;
+        a_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(a));
+        b_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(b));
+        a += 32, b += 32, n -= 32;
     }
-    ab_i32s_vec = _mm512_dpbusd_epi32(ab_i32s_vec, a_vec, b_vec);
-    a2_i32s_vec = _mm512_dpbusd_epi32(a2_i32s_vec, a_vec, a_vec);
-    b2_i32s_vec = _mm512_dpbusd_epi32(b2_i32s_vec, b_vec, b_vec);
+    // Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
+    // as it's assymetric with respect to the sign of the input arguments:
+    //      Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
+    // So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
+    // to 16-bit beforehand.
+    ab_i32s_vec = _mm512_dpwssd_epi32(ab_i32s_vec, a_vec, b_vec);
+    a2_i32s_vec = _mm512_dpwssd_epi32(a2_i32s_vec, a_vec, a_vec);
+    b2_i32s_vec = _mm512_dpwssd_epi32(b2_i32s_vec, b_vec, b_vec);
     if (n)
         goto simsimd_avx512_i8_cos_cycle;
 

package/simsimd/include/simsimd/types.h

@@ -84,11 +84,11 @@
 #endif
 
 #ifndef SIMSIMD_F32_DIVISION_EPSILON
-#define SIMSIMD_F32_DIVISION_EPSILON 1e-7
+#define SIMSIMD_F32_DIVISION_EPSILON (1e-7)
 #endif
 
 #ifndef SIMSIMD_F16_DIVISION_EPSILON
-#define SIMSIMD_F16_DIVISION_EPSILON 1e-3
+#define SIMSIMD_F16_DIVISION_EPSILON (1e-3)
 #endif
 
 #ifdef __cplusplus
@@ -100,6 +100,7 @@ typedef float simsimd_f32_t;
 typedef double simsimd_f64_t;
 typedef signed char simsimd_i8_t;
 typedef unsigned char simsimd_b8_t;
+typedef unsigned long long simsimd_u64_t;
 typedef unsigned long long simsimd_size_t;
 
 #if !defined(SIMSIMD_NATIVE_F16) || SIMSIMD_NATIVE_F16

package/simsimd/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "simsimd",
-  "version": "3.6.4",
+  "version": "3.8.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "simsimd",
-      "version": "3.6.4",
+      "version": "3.8.1",
       "hasInstallScript": true,
       "license": "Apache 2.0",
       "dependencies": {
@@ -627,9 +627,9 @@
       "dev": true
     },
     "node_modules/ip": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
-      "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz",
+      "integrity": "sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==",
       "dev": true
     },
     "node_modules/is-fullwidth-code-point": {

package/simsimd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "simsimd",
-  "version": "3.8.0",
+  "version": "3.9.0",
   "description": "Fastest SIMD-Accelerated Vector Similarity Functions for x86 and Arm",
   "homepage": "https://github.com/ashvardanian/simsimd",
   "author": "Ash Vardanian",

package/simsimd/rust/benches/native.rs

@@ -10,18 +10,21 @@ pub(crate) fn cosine_similarity_cpu(a: &[f32], b: &[f32]) -> Option<f32> {
         return None;
     }
 
-    let dot_product: f32 = a.iter().zip(b.iter()).map(|(a, b)| a * b).sum();
-    let norm_a: f32 = a.iter().map(|a| a.powf(2.0)).sum();
-    let norm_b: f32 = b.iter().map(|b| b.powf(2.0)).sum();
+    let (dot_product, norm_a, norm_b) = a
+        .iter()
+        .zip(b)
+        .map(|(a, b)| (a * b, a * a, b * b))
+        .fold((0.0, 0.0, 0.0), |acc, x| {
+            (acc.0 + x.0, acc.1 + x.1, acc.2 + x.2)
+        });
 
     Some(1.0 - (dot_product / (norm_a.sqrt() * norm_b.sqrt())))
 }
 
-pub(crate) fn squared_euclidean_cpu(a: &[f32], b: &[f32]) -> f32 {
-    assert_eq!(a.len(), b.len());
+pub(crate) fn squared_euclidean_cpu(a: &[f32], b: &[f32]) -> Option<f32> {
+    if a.len() != b.len() {
+        return None;
+    }
 
-    a.iter()
-        .zip(b.iter())
-        .map(|(x, y)| ((*x) - (*y)) * ((*x) - (*y)))
-        .fold(0.0, ::std::ops::Add::add)
+    Some(a.iter().zip(b).map(|(x, y)| (x - y).powi(2)).sum())
 }

package/simsimd/rust/lib.c

@@ -7,62 +7,39 @@ simsimd_capability_t cached_capabilities(void) {
     return static_capabilities;
 }
 
-simsimd_f32_t cosine_i8(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_cosine_k, simsimd_datatype_i8_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
+#define SIMSIMD_METRIC_DECLARATION(name, type)                                                                         \
+    simsimd_f32_t name##_##type(simsimd_##type##_t const* a, simsimd_##type##_t const* b, simsimd_size_t d) {          \
+        static simsimd_metric_punned_t metric = 0;                                                                     \
+        if (metric == 0) {                                                                                             \
+            simsimd_capability_t used_capability;                                                                      \
+            simsimd_find_metric_punned(simsimd_metric_##name##_k, simsimd_datatype_##type##_k, cached_capabilities(),  \
+                                       simsimd_cap_any_k, &metric, &used_capability);                                  \
+        }                                                                                                              \
+        return metric(a, b, d, d);                                                                                     \
     }
-    return metric(a, b, d, d);
-}
 
-simsimd_f32_t cosine_f32(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_cosine_k, simsimd_datatype_f32_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
+// Spatial distances
+SIMSIMD_METRIC_DECLARATION(cosine, i8)
+SIMSIMD_METRIC_DECLARATION(cosine, f16)
+SIMSIMD_METRIC_DECLARATION(cosine, f32)
+SIMSIMD_METRIC_DECLARATION(cosine, f64)
+SIMSIMD_METRIC_DECLARATION(inner, i8)
+SIMSIMD_METRIC_DECLARATION(inner, f16)
+SIMSIMD_METRIC_DECLARATION(inner, f32)
+SIMSIMD_METRIC_DECLARATION(inner, f64)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, i8)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, f16)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, f32)
+SIMSIMD_METRIC_DECLARATION(sqeuclidean, f64)
 
-simsimd_f32_t inner_i8(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_inner_k, simsimd_datatype_i8_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
+// Binary distances
+SIMSIMD_METRIC_DECLARATION(hamming, b8)
+SIMSIMD_METRIC_DECLARATION(jaccard, b8)
 
-simsimd_f32_t inner_f32(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_inner_k, simsimd_datatype_f32_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
-
-simsimd_f32_t sqeuclidean_i8(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_sqeuclidean_k, simsimd_datatype_i8_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
-
-simsimd_f32_t sqeuclidean_f32(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t d) {
-    static simsimd_metric_punned_t metric = 0;
-    if (metric == 0) {
-        simsimd_capability_t used_capability;
-        simsimd_find_metric_punned(simsimd_metric_sqeuclidean_k, simsimd_datatype_f32_k, cached_capabilities(),
-                                   simsimd_cap_any_k, &metric, &used_capability);
-    }
-    return metric(a, b, d, d);
-}
+// Probability distributions
+SIMSIMD_METRIC_DECLARATION(kl, f16)
+SIMSIMD_METRIC_DECLARATION(kl, f32)
+SIMSIMD_METRIC_DECLARATION(kl, f64)
+SIMSIMD_METRIC_DECLARATION(js, f16)
+SIMSIMD_METRIC_DECLARATION(js, f32)
+SIMSIMD_METRIC_DECLARATION(js, f64)