usearch 0.19.2 → 0.20.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "0.19.2"
+version = "0.20.0"
 authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/README.md

@@ -167,7 +167,7 @@ index.add(/* label: */ 42, /* vector: */ {&vec[0], 3});
 auto results = index.search(/* query: */ {&vec[0], 3}, 5 /* neighbors */);
 
 for (std::size_t i = 0; i != results.size(); ++i)
-    results[i].element.label, results[i].element.vector, results[i].distance;
+    results[i].member.label, results[i].member.vector, results[i].member.id, results[i].distance;
 ```
 
 The `add` is thread-safe for concurrent index construction.

package/c/lib.cpp

@@ -15,8 +15,7 @@ using index_t = index_punned_dense_gt<label_t>;
 using add_result_t = index_t::add_result_t;
 using search_result_t = index_t::search_result_t;
 using serialization_result_t = index_t::serialization_result_t;
-
-using span_t = span_gt<float>;
+using vector_view_t = span_gt<float>;
 
 // helper functions that are not part of the C ABI
 metric_kind_t to_native_metric(usearch_metric_kind_t kind) {

package/c/test.c

@@ -1,182 +1,294 @@
-#include <assert.h>
+#include <stdio.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 #include <sys/stat.h>
-#include <time.h>
 
 #include "usearch.h"
 
-float* read_fvecs_file(const char* filename, size_t* num_vectors, size_t* vector_dim) {
-    struct stat st;
-    FILE* file;
-    size_t cnt;
-    float skip;
-
-    ;
-    if (stat(filename, &st) == -1) {
-        printf("Failed to read file: %s err: %d: %s\n", filename, errno, strerror(errno));
-        return NULL;
+#define ASSERT(must_be_true, message)           \
+    if (!(must_be_true)) {                      \
+        printf("Assert: %s\n", message);        \
+        exit(-1);                               \
     }
 
-    file = fopen(filename, "rb");
-    if (file == NULL) {
-        printf("Failed to open file: %s\n", filename);
-        return NULL;
-    }
 
-    // Read the number of vectors and vector dimension from the file
-    cnt = fread(vector_dim, sizeof(int), 1, file);
-    assert(cnt == 1);
-    if (st.st_size % ((*vector_dim + 1) * sizeof(float)) != 0) {
-        printf("File does not contain a whole number of vectors.\n Vector dim: %ld, file size: %ld\n", *vector_dim,
-               st.st_size);
-        fclose(file);
-        return NULL;
-    }
+void parse_vectors_from_file(char const* file_path, size_t* count, size_t* dimension, float** data) {
 
-    *num_vectors = st.st_size / ((*vector_dim + 1) * sizeof(float));
+    // Open the file
+    struct stat st = {};
+    ASSERT(stat(file_path, &st) == 0, "Failed to get metadata of the file");
+    FILE* file = fopen(file_path, "rb");
+    ASSERT(file, "Failed to open the file");
 
-    // Allocate memory for the array
-    float* data = (float*)malloc(*num_vectors * *vector_dim * sizeof(float));
-    if (data == NULL) {
-        printf("Memory allocation failed.\n");
-        fclose(file);
-        return NULL;
-    }
+    // Read vectors dimension and calculate count
+    int dim = 0;
+    size_t read = fread(&dim, sizeof(int), 1, file);
+    ASSERT(read == 1, "Failed to read from the file");
+    ASSERT(st.st_size % ((dim + 1) * sizeof(float)) == 0, "File does not contain a whole number of vectors");
+    *dimension = (size_t)dim;
+    *count = st.st_size / ((dim + 1) * sizeof(float));
 
-    // Read the data from the file into the array
-    for (size_t i = 0; i < *num_vectors; i++) {
-        cnt = fread(data + i * *vector_dim, sizeof(float), *vector_dim, file);
-        assert(cnt == *vector_dim);
-        if (i == *num_vectors - 1) {
+    // Allocate memory for the vectors' data
+    *data = (float*)malloc(*count * *dimension * sizeof(float));
+    ASSERT(*data, "Failed to allocate memory");
+
+    // Read the data
+    for (size_t i = 0; i < *count; ++i) {
+        read = fread(*data + i * *dimension, sizeof(float), *dimension, file);
+        ASSERT(read == *dimension, "Failed to read from the file");
+         if (i == *count - 1)
             break;
-        }
-        cnt = fread(&skip, sizeof(float), 1, file);
-        assert(cnt == 1);
+
+        // Skip
+        float skip;
+        read = fread(&skip, sizeof(float), 1, file);
+        ASSERT(read == 1, "Failed to read from the file");
     }
 
     fclose(file);
-    return data;
 }
 
-// log every this many iterations in test loops
-const int LOG_GRANULARITY = 10000;
-
-void test_index(float* data, size_t num_vectors, size_t vector_dim) {
+usearch_init_options_t create_options(size_t vector_dimension) {
     usearch_init_options_t opts;
-    usearch_index_t idx;
-    usearch_error_t error = NULL;
-    clock_t start;
     opts.connectivity = 2; // 32 in faiss
-    opts.dimensions = vector_dim;
+    opts.dimensions = vector_dimension;
     opts.expansion_add = 40;    // 40 in faiss
     opts.expansion_search = 16; // 10 in faiss
     opts.metric_kind = usearch_metric_ip_k;
     opts.metric = NULL;
     opts.quantization = usearch_scalar_f32_k;
+    return opts;
+}
+
+void test_init(size_t vectors_count, size_t vector_dimension) {
+    printf("Test: Index Initialization...\n");
+
+    // Init index
+    usearch_index_t idx = NULL;
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(vector_dimension);
+    idx = usearch_init(&opts, &error);
+    ASSERT(!error, error);
+    usearch_free(idx, &error);
+    ASSERT(!error, error);
+
+    // Init second time
+    idx = usearch_init(&opts, &error);
+    ASSERT(!error, error);
+
+    ASSERT(usearch_size(idx, &error) == 0, error);
+    ASSERT(usearch_capacity(idx, &error) == 0, error);
+    ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
+    ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
+
+    // Reserve
+    usearch_reserve(idx, vectors_count, &error);
+    ASSERT(!error, error);
+    ASSERT(usearch_size(idx, &error) == 0, error);
+    ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
+    ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
+    ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
+
+    usearch_free(idx, &error);
+    ASSERT(!error, error);
+
+    printf("Test: Index Initialization - PASSED\n");
+}
 
-    // results arrays
-    int k = 10;
-    num_vectors /= 10;
-    usearch_label_t* labels = (usearch_label_t*)calloc(k, sizeof(usearch_label_t));
-    float* distances = (float*)calloc(k, sizeof(float));
+void test_add_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
+    printf("Test: Add Vector...\n");
 
-    fprintf(stderr, "read %ld vectors of dimension %ld\n", num_vectors, vector_dim);
-    fprintf(stderr, "initializing usearch...\n");
+    usearch_index_t idx = NULL;
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(vector_dimension);
     idx = usearch_init(&opts, &error);
-    if (error != NULL) {
-        fprintf(stderr, "usearch error: %s\n", error);
-        assert(false);
+    usearch_reserve(idx, vectors_count, &error);
+
+    // Add vectors
+    for (size_t i = 0; i < vectors_count; ++i) {
+        usearch_label_t label = i;
+        usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        ASSERT(!error, error);
     }
 
-    fprintf(stderr, "reserving capacity...\n");
-    usearch_reserve(idx, num_vectors, &error);
-    assert(!error);
+    ASSERT(usearch_size(idx, &error) == vectors_count, error);
+    ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
 
-    fprintf(stderr, "adding vectors...\n");
-    for (size_t i = 0; i < num_vectors; i++) {
-        if (i % LOG_GRANULARITY == 0) {
-            fprintf(stderr, "added %ld vectors...\n", i);
-        }
+    // Check vectors in the index
+    for (size_t i = 0; i < vectors_count; ++i) {
         usearch_label_t label = i;
-        usearch_add(idx, label, data + i * vector_dim, usearch_scalar_f32_k, &error);
-        assert(!error);
+        ASSERT(usearch_contains(idx, label, &error), error);
     }
+    ASSERT(!usearch_contains(idx, -1, &error), error); // Non existing label
+
+    usearch_free(idx, &error);
+    printf("Test: Add Vector - PASSED\n");
+}
+
+void test_find_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
+    printf("Test: Find Vector...\n");
 
-    fprintf(stderr, "checking containment of vectors...\n");
-    for (size_t i = 0; i < num_vectors; i++) {
-        if (i % LOG_GRANULARITY == 0) {
-            fprintf(stderr, "checked %ld vectors...\n", i);
-        }
+    usearch_index_t idx = NULL;
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(vector_dimension);
+    idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, vectors_count, &error);
+
+    // Create result buffers
+    int results_count = 10;
+    usearch_label_t* labels = (usearch_label_t*)malloc(results_count * sizeof(usearch_label_t));
+    float* distances = (float*)malloc(results_count * sizeof(float));
+    ASSERT(labels && distances, "Failed to allocate memory");
+
+    // Add vectors
+    for (size_t i = 0; i < vectors_count; ++i) {
         usearch_label_t label = i;
-        assert(usearch_contains(idx, label, &error));
-        assert(!error);
+        usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        ASSERT(!error, error);
     }
 
-    assert(!usearch_contains(idx, -1, &error));
-    assert(!error);
-
-    fprintf(stderr, "searching vectors...\n");
-    for (size_t i = 0; i < num_vectors; i++) {
-        if (i % LOG_GRANULARITY == 0) {
-            fprintf(stderr, "searched %ld vectors...\n", i);
-        }
-        const void *query_vector = data + i * vector_dim;
-        usearch_search(idx, query_vector, usearch_scalar_f32_k, k, labels, distances, &error);
-        assert(!error);
+    // Find the vectors
+    for (size_t i = 0; i < vectors_count; i++) {
+        const void *query_vector = data + i * vector_dimension;
+        size_t found_count = usearch_search(idx, query_vector, usearch_scalar_f32_k, results_count, labels, distances, &error);
+        ASSERT(!error, error);
+        ASSERT(found_count = results_count, "Vector is missing");
     }
 
-    fprintf(stderr, "saving the index...\n");
-    start = clock();
-    usearch_save(idx, "usearch_index.bin", &error);
-    assert(!error);
-    fprintf(stderr, "saving took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
+    usearch_free(idx, &error);
+    printf("Test: Find Vector - PASSED\n");
+}
+
+void test_remove_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
+    printf("Test: Remove Vector...\n");
+
+    usearch_index_t idx = NULL;
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(vector_dimension);
+    idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, vectors_count, &error);
+
+    // Add vectors
+    for (size_t i = 0; i < vectors_count; ++i) {
+        usearch_label_t label = i;
+        usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        ASSERT(!error, error);
+    }
+
+    // Remove the vectors
+    for (size_t i = 0; i < vectors_count; i++) {
+        usearch_label_t label = i;
+        usearch_remove(idx, label, &error);
+        ASSERT(error, "Currently, Remove is not supported");
+    }
+
+    usearch_free(idx, &error);
+    printf("Test: Remove Vector - PASSED\n");
+}
 
-    fprintf(stderr, "destroying the index...\n");
-    start = clock();
+void test_save_load(size_t vectors_count, size_t vector_dimension, float const* data) {
+    printf("Test: Save/Load...\n");
+
+    usearch_index_t idx = NULL;
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(vector_dimension);
+    idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, vectors_count, &error);
+
+    // Add vectors
+    for (size_t i = 0; i < vectors_count; ++i) {
+        usearch_label_t label = i;
+        usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        ASSERT(!error, error);
+    }
+
+    // Save and free the index
+    usearch_save(idx, "usearch_index.bin", &error);
+    ASSERT(!error, error);
     usearch_free(idx, &error);
-    assert(!error);
-    fprintf(stderr, "destroying the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
+    ASSERT(!error, error);
 
-    fprintf(stderr, "loading the index...\n");
-    start = clock();
+    // Reinit
     idx = usearch_init(&opts, &error);
-    assert(!error);
-    assert(usearch_size(idx, &error) == 0);
-    assert(!error);
+    ASSERT(!error, error);
+    ASSERT(usearch_size(idx, &error) == 0, error);
+
+    // Load
     usearch_load(idx, "usearch_index.bin", &error);
-    assert(!error);
-    fprintf(stderr, "loading the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
-    assert(usearch_size(idx, &error) == num_vectors);
-    assert(!error);
+    ASSERT(!error, error);
+    ASSERT(usearch_size(idx, &error) == vectors_count, error);
+    ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
+    ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
+    ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
+
+    // Check vectors in the index
+    for (size_t i = 0; i < vectors_count; ++i) {
+        usearch_label_t label = i;
+        ASSERT(usearch_contains(idx, label, &error), error);
+    }
+
     usearch_free(idx, &error);
+    printf("Test: Save/Load - PASSED\n");
+}
+
+void test_view(size_t vectors_count, size_t vector_dimension, float const* data) {
+    printf("Test: View...\n");
 
-    fprintf(stderr, "viewing the index...\n");
-    start = clock();
+    usearch_index_t idx = NULL;
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(vector_dimension);
     idx = usearch_init(&opts, &error);
-    assert(!error);
-    assert(usearch_size(idx, &error) == 0);
-    assert(!error);
-    usearch_view(idx, "usearch_index.bin", &error);
-    assert(!error);
-    fprintf(stderr, "viewing the index took %.2f ms\n", ((double)(clock() - start)) / CLOCKS_PER_SEC * 1000);
-    assert(usearch_size(idx, &error) == num_vectors);
-    assert(!error);
+    usearch_reserve(idx, vectors_count, &error);
+
+    // Add vectors
+    for (size_t i = 0; i < vectors_count; ++i) {
+        usearch_label_t label = i;
+        usearch_add(idx, label, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        ASSERT(!error, error);
+    }
+
+    // Save and free the index
+    usearch_save(idx, "usearch_index.bin", &error);
+    ASSERT(!error, error);
     usearch_free(idx, &error);
+    ASSERT(!error, error);
+
+    // Reinit
+    idx = usearch_init(&opts, &error);
+    ASSERT(!error, error);
+
+    // View
+    usearch_view(idx, "usearch_index.bin", &error);
+    ASSERT(!error, error);
 
+    usearch_free(idx, &error);
+    printf("Test: View - PASSED\n");
 }
 
-int main() {
-    const char* filename = "sift/sift_base.fvecs";
-    size_t num_vectors = 0, vector_dim = 0;
-    float* data = read_fvecs_file(filename, &num_vectors, &vector_dim);
-    if (data != NULL) {
+int main(int argc, char* argv[]) {
+    // Resolve file path
+    char const* filename = "sample.fvecs";
+    if (argc == 2)
+        filename = argv[1];
 
-        test_index(data, num_vectors, vector_dim);
+    // Read data from the file
+    float* data = NULL;
+    size_t vectors_count = 0;
+    size_t vector_dimension = 0;
+    parse_vectors_from_file(filename, &vectors_count, &vector_dimension, &data);
+    vectors_count = vectors_count > 100000 ? 100000 : vectors_count; // Just limit vectors
+
+    // Test
+    test_init(vectors_count, vector_dimension);
+    test_add_vector(vectors_count, vector_dimension, data);
+    test_find_vector(vectors_count, vector_dimension, data);
+    test_remove_vector(vectors_count, vector_dimension, data);
+    test_save_load(vectors_count, vector_dimension, data);
+    test_view(vectors_count, vector_dimension, data);
+
+    // Free vectors' buffer
+    free(data);
 
-        free(data); // Remember to free the dynamically allocated memory
-    }
     return 0;
 }

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '0.19.2'
+    version = '0.20.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/cpp/bench.cpp

@@ -1,7 +1,7 @@
 /**
- * @brief A benchmark for the construction speed of the USearch index
- * and the resulting accuracy (recall) of the Approximate Nearest Neighbors
- * Search queries.
+ *  @brief A benchmark for the construction speed of the USearch index
+ *  and the resulting accuracy (recall) of the Approximate Nearest Neighbors
+ *  Search queries.
  */
 
 #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)

package/cpp/test.cpp

@@ -9,8 +9,6 @@
 using namespace unum::usearch;
 using namespace unum;
 
-using point_id_t = std::int64_t;
-
 void expect(bool must_be_true) {
     if (!must_be_true)
         throw std::runtime_error("Failed!");
@@ -22,6 +20,7 @@ template <typename scalar_at, typename index_at> void test3d(index_at&& index) {
     using view_t = span_gt<scalar_t const>;
     using index_t = typename std::remove_reference<index_at>::type;
     using distance_t = typename index_t::distance_t;
+    using label_t = typename index_t::label_t;
 
     scalar_t vec[3] = {10, 20, 15};
     scalar_t vec_a[3] = {15, 16, 17};
@@ -31,7 +30,7 @@ template <typename scalar_at, typename index_at> void test3d(index_at&& index) {
     index.add(42, view_t{&vec[0], 3ul});
 
     // Default approximate search
-    point_id_t matched_labels[10] = {0};
+    label_t matched_labels[10] = {0};
     distance_t matched_distances[10] = {0};
     std::size_t matched_count = index.search(view_t{&vec[0], 3ul}, 5).dump_to(matched_labels, matched_distances);
 
@@ -52,7 +51,7 @@ template <typename scalar_at, typename index_at> void test3d(index_at&& index) {
     // Validate scans
     std::size_t count = 0;
     for (auto member : index) {
-        point_id_t id = member.label;
+        label_t id = member.label;
         expect(id >= 42 && id <= 44);
         count++;
     }
@@ -146,11 +145,12 @@ int main(int, char**) {
     // static_assert(!std::is_same<index_gt<pearson_correlation_gt<>>::value_type, std::true_type>());
     // static_assert(!std::is_same<index_gt<haversine_gt<>>::value_type, std::true_type>());
 
-    test3d<float>(index_gt<cos_gt<float>, point_id_t, std::uint32_t>{});
-    test3d<float>(index_gt<l2sq_gt<float>, point_id_t, std::uint32_t>{});
+    using big_point_id_t = std::int64_t;
+    test3d<float>(index_gt<cos_gt<float>, big_point_id_t, std::uint32_t>{});
+    test3d<float>(index_gt<l2sq_gt<float>, big_point_id_t, std::uint32_t>{});
 
-    test3d<double>(index_gt<cos_gt<double>, point_id_t, std::uint32_t>{});
-    test3d<double>(index_gt<l2sq_gt<double>, point_id_t, std::uint32_t>{});
+    test3d<double>(index_gt<cos_gt<double>, big_point_id_t, std::uint32_t>{});
+    test3d<double>(index_gt<l2sq_gt<double>, big_point_id_t, std::uint32_t>{});
 
     test3d<float>(punned_small_t::make(3, metric_kind_t::cos_k));
     test3d<float>(punned_small_t::make(3, metric_kind_t::l2sq_k));
@@ -161,11 +161,11 @@ int main(int, char**) {
     test3d_punned<float>(punned_small_t::make(3, metric_kind_t::cos_k));
     test3d_punned<float>(punned_small_t::make(3, metric_kind_t::l2sq_k));
 
-    test_sets(index_gt<jaccard_gt<std::int32_t, float>, point_id_t, std::uint32_t>{});
-    test_sets(index_gt<jaccard_gt<std::int64_t, float>, point_id_t, std::uint32_t>{});
+    test_sets(index_gt<jaccard_gt<std::int32_t, float>, big_point_id_t, std::uint32_t>{});
+    test_sets(index_gt<jaccard_gt<std::int64_t, float>, big_point_id_t, std::uint32_t>{});
 
-    test_sets_moved<index_gt<jaccard_gt<std::int32_t, float>, point_id_t, std::uint32_t>>();
-    test_sets_moved<index_gt<jaccard_gt<std::int64_t, float>, point_id_t, std::uint32_t>>();
+    test_sets_moved<index_gt<jaccard_gt<std::int32_t, float>, big_point_id_t, std::uint32_t>>();
+    test_sets_moved<index_gt<jaccard_gt<std::int64_t, float>, big_point_id_t, std::uint32_t>>();
 
     return 0;
 }