usearch 2.1.2 → 2.2.0

package/simsimd/include/simsimd/simsimd_chem.h

@@ -26,7 +26,8 @@ extern "C" {
  *          forming 21 incomplete bytes for the MACCS fingerprints used in
  *          computation chemistry.
  */
-inline static simsimd_f32_t simsimd_tanimoto_maccs_naive(uint8_t const* a_chars, uint8_t const* b_chars) {
+inline static simsimd_f32_t //
+simsimd_tanimoto_maccs_naive(uint8_t const* a_chars, uint8_t const* b_chars) {
     unsigned long a[3] = {0};
     unsigned long b[3] = {0};
     memcpy(&a[0], a_chars, 21);
@@ -36,13 +37,44 @@ inline static simsimd_f32_t simsimd_tanimoto_maccs_naive(uint8_t const* a_chars,
     return 1 - and_count / or_count;
 }
 
+/**
+ *  @brief  Weird Tanimoto distance implementation for concatenated MACCS and ECFP4
+ *          representations, forming a grand-total of 56 bytes. The last chunk is evaluated
+ *          if only the first one matches well.
+ *
+ *              - First 166 bits in 21 bytes is a MACCS vector.
+ *              - After that 3 empty bytes of a padding are expected.
+ *              - After which, 256 bytes (32 words) of ECFP4 follow.
+ */
+inline static simsimd_f32_t //
+simsimd_tanimoto_conditional_naive(uint8_t const* a_chars, uint8_t const* b_chars) {
+    float const threshold = 0.2;
+    unsigned long const* a = (unsigned long const*)(a_chars);
+    unsigned long const* b = (unsigned long const*)(b_chars);
+    float and_count = popcount64(a[0] & b[0]) + popcount64(a[1] & b[1]) + popcount64(a[2] & b[2]);
+    float or_count = popcount64(a[0] | b[0]) + popcount64(a[1] | b[1]) + popcount64(a[2] | b[2]);
+    float result = 1 - and_count / or_count;
+    if (result > threshold)
+        return result;
+
+    // Start comparing ECFP4
+    and_count = 0, or_count = 0;
+    for (int i = 0; i != 32; ++i)
+        and_count += popcount64(a[3 + i] & b[3 + i]), or_count += popcount64(a[3 + i] | b[3 + i]);
+    result = 1 - and_count / or_count;
+    return result * threshold;
+}
+
+#if SIMSIMD_TARGET_ARM
+#if SIMSIMD_TARGET_ARM_NEON
+
 /**
  *  @brief  Optimized version for Tanimoto distance on @b exactly 166 bits,
  *          forming 21 incomplete bytes for the MACCS fingerprints used in
  *          computation chemistry, accelerated for Arm NEON ISA.
  */
-inline static simsimd_f32_t simsimd_tanimoto_maccs_neon(uint8_t const* a_chars, uint8_t const* b_chars) {
-#if defined(__ARM_NEON)
+inline static simsimd_f32_t //
+simsimd_tanimoto_maccs_neon(uint8_t const* a_chars, uint8_t const* b_chars) {
     unsigned char a[32] = {0};
     unsigned char b[32] = {0};
     memcpy(&a[0], a_chars, 21);
@@ -58,19 +90,19 @@ inline static simsimd_f32_t simsimd_tanimoto_maccs_neon(uint8_t const* a_chars,
     float and_populations = vaddvq_u8(vaddq_u8(vcntq_u8(a_and_b_first), vcntq_u8(a_and_b_second)));
     float or_populations = vaddvq_u8(vaddq_u8(vcntq_u8(a_or_b_first), vcntq_u8(a_or_b_second)));
     return 1 - and_populations / or_populations;
-#else
-    (void)a_chars, (void)b_chars;
-    return 0;
-#endif
 }
 
+#endif // SIMSIMD_TARGET_ARM_NEON
+
+#if SIMSIMD_TARGET_ARM_SVE
+
 /**
  *  @brief  Optimized version for Tanimoto distance on @b exactly 166 bits,
  *          forming 21 incomplete bytes for the MACCS fingerprints used in
  *          computation chemistry, accelerated for Arm NEON ISA.
  */
-inline static simsimd_f32_t simsimd_tanimoto_maccs_sve(uint8_t const* a_chars, uint8_t const* b_chars) {
-#if defined(__ARM_FEATURE_SVE)
+__attribute__((target("+sve"))) inline static simsimd_f32_t //
+simsimd_tanimoto_maccs_sve(uint8_t const* a_chars, uint8_t const* b_chars) {
     svbool_t pg_vec = svwhilelt_b8(0ul, 21ul);
     svuint8_t a_vec = svld1_u8(pg_vec, (uint8_t const*)a_chars);
     svuint8_t b_vec = svld1_u8(pg_vec, (uint8_t const*)b_chars);
@@ -92,11 +124,12 @@ inline static simsimd_f32_t simsimd_tanimoto_maccs_sve(uint8_t const* a_chars, u
     float and_populations = svaddv_u8(svptrue_b8(), and_populations_vec);
     float or_populations = svaddv_u8(svptrue_b8(), or_populations_vec);
     return 1 - and_populations / or_populations;
-#else
-    (void)a_chars, (void)b_chars;
-    return 0;
-#endif
 }
+#endif
+#endif // SIMSIMD_TARGET_ARM
+
+#if SIMSIMD_TARGET_X86
+#if SIMSIMD_TARGET_X86_AVX512
 
 /**
  *  @brief  Optimized version for Tanimoto distance on @b exactly 166 bits,
@@ -104,21 +137,29 @@ inline static simsimd_f32_t simsimd_tanimoto_maccs_sve(uint8_t const* a_chars, u
  *          computation chemistry, accelerated with AVX-512 population count
  *          instructions.
  */
-inline static simsimd_f32_t simsimd_tanimoto_maccs_avx512(uint8_t const* a, uint8_t const* b) {
-#if defined(__AVX512VPOPCNTDQ__)
+__attribute__((target("avx512vpopcntdq")))                     //
+__attribute__((target("avx512vl")))                            //
+__attribute__((target("avx512bw")))                            //
+__attribute__((target("avx512f"))) inline static simsimd_f32_t //
+simsimd_tanimoto_maccs_avx512(uint8_t const* a, uint8_t const* b) {
     __m256i a_vec = _mm256_maskz_loadu_epi8(0b11111111111111111111100000000000, a);
     __m256i b_vec = _mm256_maskz_loadu_epi8(0b11111111111111111111100000000000, b);
-    __m256i and_vec _mm256_and_si256(a_vec, b_vec);
-    __m256i or_vec _mm256_or_si256(a_vec, b_vec);
-    __m256i and_counts = _mm256_popcnt_epi8(and_vec);
-    __m256i or_counts = _mm256_popcnt_epi8(or_vec);
-    return 1 - float(_mm256_reduce_add_epi8(and_counts)) / _mm256_reduce_add_epi8(or_counts);
-#else
-    (void)a, (void)b;
-    return 0;
-#endif
+    __m256i and_vec = _mm256_and_si256(a_vec, b_vec);
+    __m256i or_vec = _mm256_or_si256(a_vec, b_vec);
+    __m256i and_counts_vec = _mm256_popcnt_epi64(and_vec);
+    __m256i or_counts_vec = _mm256_popcnt_epi64(or_vec);
+
+    float and_counts = _mm256_extract_epi64(and_counts_vec, 0) + _mm256_extract_epi64(and_counts_vec, 1) +
+                       _mm256_extract_epi64(and_counts_vec, 2) + _mm256_extract_epi64(and_counts_vec, 3);
+    float or_counts = _mm256_extract_epi64(or_counts_vec, 0) + _mm256_extract_epi64(or_counts_vec, 1) +
+                      _mm256_extract_epi64(or_counts_vec, 2) + _mm256_extract_epi64(or_counts_vec, 3);
+
+    return 1 - and_counts / or_counts;
 }
 
+#endif // SIMSIMD_TARGET_X86_AVX512
+#endif // SIMSIMD_TARGET_X86
+
 #undef popcount64
 
 #ifdef __cplusplus

package/simsimd/include/simsimd/simsimd_text.h

@@ -0,0 +1,92 @@
+/**
+ *  @brief Collection of Biology-oriented Similarity Measures, SIMD-accelerated with SSE, AVX, NEON, SVE.
+ *
+ *  @author Ash Vardanian
+ *  @date August 8, 2023
+ */
+
+#pragma once
+
+#include <simsimd/simsimd_chem.h>
+
+#ifdef _MSC_VER
+#define hash32
+#define hash64
+#else
+#define hash32
+#define hash64
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+inline static void simsimd_4gram_hash_string( //
+    char const* chars, size_t chars_length,   //
+    uint8_t* hash, size_t hash_length) {
+
+    // Skip misaligned addresses in a naive fashion
+    char const* const chars_end = chars + chars_length;
+    for (; uintptr_t(chars) % 4 && chars != chars_end; ++chars) {
+        uint32_t word;
+        memcpy(&word, chars, 4);
+        hash;
+    }
+
+    uint32_t last_word = *a_chars;
+
+    for (size_t i = 0; i + 8 <= a_length; i += 8) {
+        uint32_t new_word = *a_chars;
+
+        size_t hash_first = hash(last_word);
+        size_t hash_second = hash((last_word << 8) | (new_word >> 24));
+        size_t hash_third = hash((last_word << 16) | (new_word >> 16));
+        size_t hash_fourth = hash((last_word << 16) | (new_word >> 16));
+
+        uint64_t a_word;
+        memcpy(&a_word, a_chars, 8);
+    }
+}
+
+/**
+ *  @brief  Similarity measure for different length strings, that compares them
+ *          by hashing the N-grams first and computing the Tenimoto distance of
+ *          hashes.
+ *
+ *  By design uses a separate bit-set for every string. Those have to be of identical
+ *  length derived from `l = max(a_length, b_length)`, as `(l + N - 1) * 2` for any
+ *  N-gram, to have less than half of bits populated.
+ *
+ *  Inspired by Rabin-Karp algorithm, and continues the approaches implemented in Stringzilla.
+ */
+inline static simsimd_f32_t simsimd_4gram_hashes_naive( //
+    uint8_t const* a_chars, uint8_t const* b_chars,     //
+    size_t a_length, size_t b_length,                   //
+    uint8_t* a_hash, uint8_t* b_hash, size_t length_hash) {
+
+    // Skip thrrough misaligned part
+
+    uint32_t last_word = *a_chars;
+
+    for (size_t i = 0; i + 8 <= a_length; i += 8) {
+        uint32_t new_word = *a_chars;
+
+        size_t hash_first = hash(last_word);
+        size_t hash_second = hash((last_word << 8) | (new_word >> 24));
+        size_t hash_third = hash((last_word << 16) | (new_word >> 16));
+        size_t hash_fourth = hash((last_word << 16) | (new_word >> 16));
+
+        uint64_t a_word;
+        memcpy(&a_word, a_chars, 8);
+    }
+
+    size_t b_words = (b_length + 7) / 8;
+
+    return simsimd_tanimoto_b1x8_naive(a_hash, b_hash, length_hash);
+}
+
+#undef popcount64
+
+#ifdef __cplusplus
+}
+#endif

package/simsimd/package.json

@@ -3,6 +3,6 @@
     "@semantic-release/exec": "^6.0.3",
     "@semantic-release/git": "^10.0.1",
     "conventional-changelog-eslint": "^4.0.0",
-    "semantic-release": "^21.0.7"
+    "semantic-release": "^21.1.1"
   }
 }

package/simsimd/pyproject.toml

@@ -30,8 +30,9 @@ repair-wheel-command = "auditwheel repair --lib-sdir . -w {dest_dir} {wheel}"
 [tool.cibuildwheel.macos]
 before-build = [
     "rm -rf {project}/build",
+    "brew install gcc@12",
 ]
-archs = ["x86_64", "universal2", "arm64"]
+archs = ["x86_64", "arm64"]
 repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"
 
 [tool.cibuildwheel.windows]

package/simsimd/python/{python.c → lib.c}

@@ -1,11 +1,17 @@
 /**
  *  @file python.c
- *  @author Ashot Vardanian
+ *  @author Ash Vardanian
  *  @date 2023-01-30
  *  @copyright Copyright (c) 2023
  *
- *  @brief Pure CPython bindings for simsimd.
+ *  @brief Pure CPython bindings for SimSIMD.
  */
+#define SIMSIMD_TARGET_ARM_NEON 1
+#define SIMSIMD_TARGET_X86_AVX2 1
+#if __linux__
+#define SIMSIMD_TARGET_ARM_SVE 1
+#define SIMSIMD_TARGET_X86_AVX512 1
+#endif
 
 #include "simsimd/simsimd.h"
 #include "simsimd/simsimd_chem.h"
@@ -51,31 +57,54 @@ PyMODINIT_FUNC PyInit_simsimd(void) {
     if (m == NULL)
         return NULL;
 
-    PyModule_AddObject(m, "dot_f32_sve", distance(&simsimd_dot_f32_sve));
-    PyModule_AddObject(m, "dot_f32x4_neon", distance(&simsimd_dot_f32x4_neon));
+    PyModule_AddObject(m, "tanimoto_b1x8_naive", distance(&simsimd_tanimoto_b1x8_naive));
 
-    PyModule_AddObject(m, "cos_f32_sve", distance(&simsimd_cos_f32_sve));
-    PyModule_AddObject(m, "cos_f16_sve", distance(&simsimd_cos_f16_sve));
+    // Specialized:
+    PyModule_AddObject(m, "tanimoto_maccs_naive", distance(&simsimd_tanimoto_maccs_naive));
+    PyModule_AddObject(m, "tanimoto_conditional_naive", distance(&simsimd_tanimoto_conditional_naive));
+
+#if SIMSIMD_TARGET_ARM
+    PyModule_AddObject(m, "dot_f32x4_neon", distance(&simsimd_dot_f32x4_neon));
     PyModule_AddObject(m, "cos_f16x4_neon", distance(&simsimd_cos_f16x4_neon));
     PyModule_AddObject(m, "cos_i8x16_neon", distance(&simsimd_cos_i8x16_neon));
     PyModule_AddObject(m, "cos_f32x4_neon", distance(&simsimd_cos_f32x4_neon));
-    PyModule_AddObject(m, "cos_f16x16_avx512", distance(&simsimd_cos_f16x16_avx512));
-    PyModule_AddObject(m, "cos_f32x4_avx2", distance(&simsimd_cos_f32x4_avx2));
 
+    // Specialized:
+    PyModule_AddObject(m, "tanimoto_maccs_neon", distance(&simsimd_tanimoto_maccs_neon));
+
+#if SIMSIMD_TARGET_ARM_SVE
+    PyModule_AddObject(m, "dot_f32_sve", distance(&simsimd_dot_f32_sve));
+    PyModule_AddObject(m, "cos_f32_sve", distance(&simsimd_cos_f32_sve));
+    PyModule_AddObject(m, "cos_f16_sve", distance(&simsimd_cos_f16_sve));
     PyModule_AddObject(m, "l2sq_f32_sve", distance(&simsimd_l2sq_f32_sve));
     PyModule_AddObject(m, "l2sq_f16_sve", distance(&simsimd_l2sq_f16_sve));
-
     PyModule_AddObject(m, "hamming_b1x8_sve", distance(&simsimd_hamming_b1x8_sve));
     PyModule_AddObject(m, "hamming_b1x128_sve", distance(&simsimd_hamming_b1x128_sve));
-    PyModule_AddObject(m, "hamming_b1x128_avx512", distance(&simsimd_hamming_b1x128_avx512));
 
-    PyModule_AddObject(m, "tanimoto_b1x8_naive", distance(&simsimd_tanimoto_b1x8_naive));
-
-    // Specific vector sizes for chemistry
-    PyModule_AddObject(m, "tanimoto_maccs_naive", distance(&simsimd_tanimoto_maccs_naive));
-    PyModule_AddObject(m, "tanimoto_maccs_neon", distance(&simsimd_tanimoto_maccs_neon));
+    // Specialized:
     PyModule_AddObject(m, "tanimoto_maccs_sve", distance(&simsimd_tanimoto_maccs_sve));
+#endif
+#endif
+
+#if SIMSIMD_TARGET_X86
+    PyModule_AddObject(m, "dot_f32x4_avx2", distance(&simsimd_dot_f32x4_avx2));
+    PyModule_AddObject(m, "cos_f32x4_avx2", distance(&simsimd_cos_f32x4_avx2));
+    PyModule_AddObject(m, "l2sq_f32x4_avx2", distance(&simsimd_l2sq_f32x4_avx2));
+    PyModule_AddObject(m, "dot_f16x8_avx2", distance(&simsimd_dot_f16x8_avx2));
+    PyModule_AddObject(m, "cos_f16x8_avx2", distance(&simsimd_cos_f16x8_avx2));
+    PyModule_AddObject(m, "l2sq_f16x8_avx2", distance(&simsimd_l2sq_f16x8_avx2));
+
+#if SIMSIMD_TARGET_X86_AVX512
+    PyModule_AddObject(m, "dot_f16x16_avx512", distance(&simsimd_dot_f16x16_avx512));
+    PyModule_AddObject(m, "cos_f16x16_avx512", distance(&simsimd_cos_f16x16_avx512));
+    PyModule_AddObject(m, "l2sq_f16x16_avx512", distance(&simsimd_l2sq_f16x16_avx512));
+
+    PyModule_AddObject(m, "hamming_b1x128_avx512", distance(&simsimd_hamming_b1x128_avx512));
+
+    // Specialized:
     PyModule_AddObject(m, "tanimoto_maccs_avx512", distance(&simsimd_tanimoto_maccs_avx512));
+#endif
+#endif
 
     return m;
 }

package/simsimd/python/test.py

@@ -1,23 +1,38 @@
-import simsimd as sisi
+import platform
+
+import simsimd as simd
 
 
 def test_non_null():
-    assert sisi.to_int(sisi.dot_f32_sve) != 0
-    assert sisi.to_int(sisi.dot_f32x4_neon) != 0
-    assert sisi.to_int(sisi.cos_f32_sve) != 0
-    assert sisi.to_int(sisi.cos_f16_sve) != 0
-    assert sisi.to_int(sisi.cos_f16x4_neon) != 0
-    assert sisi.to_int(sisi.cos_i8x16_neon) != 0
-    assert sisi.to_int(sisi.cos_f32x4_neon) != 0
-    assert sisi.to_int(sisi.cos_f16x16_avx512) != 0
-    assert sisi.to_int(sisi.cos_f32x4_avx2) != 0
-    assert sisi.to_int(sisi.l2sq_f32_sve) != 0
-    assert sisi.to_int(sisi.l2sq_f16_sve) != 0
-    assert sisi.to_int(sisi.hamming_b1x8_sve) != 0
-    assert sisi.to_int(sisi.hamming_b1x128_sve) != 0
-    assert sisi.to_int(sisi.hamming_b1x128_avx512) != 0
-    assert sisi.to_int(sisi.tanimoto_b1x8_naive) != 0
-    assert sisi.to_int(sisi.tanimoto_maccs_naive) != 0
-    assert sisi.to_int(sisi.tanimoto_maccs_neon) != 0
-    assert sisi.to_int(sisi.tanimoto_maccs_sve) != 0
-    assert sisi.to_int(sisi.tanimoto_maccs_avx512) != 0
+    # Some distance functions are provided for every platform
+    assert simd.to_int(simd.tanimoto_b1x8_naive) != 0
+    assert simd.to_int(simd.tanimoto_maccs_naive) != 0
+
+    # Arm Neon variants should be precompiled for any 64-bit Arm machine:
+    if platform.machine() == "arm64":
+        assert simd.to_int(simd.cos_f16x4_neon) != 0
+        assert simd.to_int(simd.cos_i8x16_neon) != 0
+        assert simd.to_int(simd.cos_f32x4_neon) != 0
+        assert simd.to_int(simd.dot_f32x4_neon) != 0
+        assert simd.to_int(simd.tanimoto_maccs_neon) != 0
+
+    # Arm SVE variants should be precompiled for 64-bit Arm machines running Linux:
+    if platform.machine() == "arm64" and platform.system().lower() == "linux":
+        assert simd.to_int(simd.dot_f32_sve) != 0
+        assert simd.to_int(simd.cos_f32_sve) != 0
+        assert simd.to_int(simd.cos_f16_sve) != 0
+        assert simd.to_int(simd.l2sq_f32_sve) != 0
+        assert simd.to_int(simd.l2sq_f16_sve) != 0
+        assert simd.to_int(simd.hamming_b1x8_sve) != 0
+        assert simd.to_int(simd.hamming_b1x128_sve) != 0
+        assert simd.to_int(simd.tanimoto_maccs_sve) != 0
+
+    # x86 AVX2 variants are precompiled for every 64-bit x86 platform:
+    if platform.machine() == "x86_64":
+        assert simd.to_int(simd.cos_f32x4_avx2) != 0
+
+    # x86 AVX-512 variants are precompiled for 64-bit x86 machines running Linux:
+    if platform.machine() == "x86_64" and platform.system().lower() == "linux":
+        assert simd.to_int(simd.cos_f16x16_avx512) != 0
+        assert simd.to_int(simd.hamming_b1x128_avx512) != 0
+        assert simd.to_int(simd.tanimoto_maccs_avx512) != 0

package/simsimd/setup.py

@@ -10,13 +10,10 @@ this_directory = os.path.abspath(dirname(__file__))
 with open(os.path.join(this_directory, "README.md"), "r", encoding="utf8") as f:
     long_description = f.read()
 
-"-march=armv8.2-a+sve"
-"-march=sapphirerapids"
-
 setup(
     name=__lib_name__,
     version=__version__,
-    author="Ashot Vardanian",
+    author="Ash Vardanian",
     author_email="1983160+ashvardanian@users.noreply.github.com",
     url="https://github.com/ashvardanian/simsimd",
     description="SIMD-accelerated similarity measures for x86 and Arm",
@@ -29,14 +26,13 @@ setup(
         "Intended Audience :: Developers",
         "Intended Audience :: Information Technology",
         "License :: OSI Approved :: Apache Software License",
-        "Operating System :: Linux",
         "Programming Language :: C",
         "Programming Language :: Python :: Implementation :: CPython",
     ],
     ext_modules=[
         Extension(
             "simsimd",
-            sources=["python/python.c"],
+            sources=["python/lib.c"],
             include_dirs=["include"],
         )
     ],

package/wasmer.toml

@@ -1,6 +1,6 @@
 [package]
 name="unum/usearch"
-version="2.1.2"
+version="2.2.0"
 description="Smaller & Faster Single-File Vector Search Engine from Unum"
 license="Apache-2.0"
 readme="README.md"