usearch 1.2.2 → 2.0.0

package/c/test.c

@@ -13,46 +13,18 @@
     }
 
 
-void parse_vectors_from_file(char const* file_path, size_t* count, size_t* dimension, float** data) {
-
-    // Open the file
-    struct stat st = {};
-    ASSERT(stat(file_path, &st) == 0, "Failed to get metadata of the file");
-    FILE* file = fopen(file_path, "rb");
-    ASSERT(file, "Failed to open the file");
-
-    // Read vectors dimension and calculate count
-    int dim = 0;
-    size_t read = fread(&dim, sizeof(int), 1, file);
-    ASSERT(read == 1, "Failed to read from the file");
-    ASSERT(st.st_size % ((dim + 1) * sizeof(float)) == 0, "File does not contain a whole number of vectors");
-    *dimension = (size_t)dim;
-    *count = st.st_size / ((dim + 1) * sizeof(float));
-
-    // Allocate memory for the vectors' data
-    *data = (float*)malloc(*count * *dimension * sizeof(float));
-    ASSERT(*data, "Failed to allocate memory");
-
-    // Read the data
-    for (size_t i = 0; i < *count; ++i) {
-        read = fread(*data + i * *dimension, sizeof(float), *dimension, file);
-        ASSERT(read == *dimension, "Failed to read from the file");
-         if (i == *count - 1)
-            break;
-
-        // Skip
-        float skip;
-        read = fread(&skip, sizeof(float), 1, file);
-        ASSERT(read == 1, "Failed to read from the file");
-    }
-
-    fclose(file);
+float* create_vectors(size_t count, size_t dimensions) {
+    float* data = (float*)malloc(count * dimensions * sizeof(float));
+    ASSERT(data, "Failed to allocate memory");
+    for (size_t idx = 0; idx < count * dimensions; ++idx)
+        data[idx] = (float)rand() / (float)RAND_MAX;
+    return data;
 }
 
-usearch_init_options_t create_options(size_t vector_dimension) {
+usearch_init_options_t create_options(size_t dimensions) {
     usearch_init_options_t opts;
     opts.connectivity = 2; // 32 in faiss
-    opts.dimensions = vector_dimension;
+    opts.dimensions = dimensions;
     opts.expansion_add = 40;    // 40 in faiss
     opts.expansion_search = 16; // 10 in faiss
     opts.metric_kind = usearch_metric_ip_k;
@@ -61,14 +33,13 @@ usearch_init_options_t create_options(size_t vector_dimension) {
     return opts;
 }
 
-void test_init(size_t vectors_count, size_t vector_dimension) {
+void test_init(size_t collection_size, size_t dimensions) {
     printf("Test: Index Initialization...\n");
 
     // Init index
-    usearch_index_t idx = NULL;
     usearch_error_t error = NULL;
-    usearch_init_options_t opts = create_options(vector_dimension);
-    idx = usearch_init(&opts, &error);
+    usearch_init_options_t opts = create_options(dimensions);
+    usearch_index_t idx = usearch_init(&opts, &error);
     ASSERT(!error, error);
     usearch_free(idx, &error);
     ASSERT(!error, error);
@@ -79,15 +50,15 @@ void test_init(size_t vectors_count, size_t vector_dimension) {
 
     ASSERT(usearch_size(idx, &error) == 0, error);
     ASSERT(usearch_capacity(idx, &error) == 0, error);
-    ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
+    ASSERT(usearch_dimensions(idx, &error) == dimensions, error);
     ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
 
     // Reserve
-    usearch_reserve(idx, vectors_count, &error);
+    usearch_reserve(idx, collection_size, &error);
     ASSERT(!error, error);
     ASSERT(usearch_size(idx, &error) == 0, error);
-    ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
-    ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
+    ASSERT(usearch_capacity(idx, &error) == collection_size, error);
+    ASSERT(usearch_dimensions(idx, &error) == dimensions, error);
     ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
 
     usearch_free(idx, &error);
@@ -96,110 +67,148 @@ void test_init(size_t vectors_count, size_t vector_dimension) {
     printf("Test: Index Initialization - PASSED\n");
 }
 
-void test_add_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
+void test_add_vector(size_t collection_size, size_t dimensions) {
     printf("Test: Add Vector...\n");
 
-    usearch_index_t idx = NULL;
     usearch_error_t error = NULL;
-    usearch_init_options_t opts = create_options(vector_dimension);
-    idx = usearch_init(&opts, &error);
-    usearch_reserve(idx, vectors_count, &error);
+    usearch_init_options_t opts = create_options(dimensions);
+    usearch_index_t idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, collection_size, &error);
 
     // Add vectors
-    for (size_t i = 0; i < vectors_count; ++i) {
+    float* data = create_vectors(collection_size, dimensions);
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
-        usearch_add(idx, key, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        usearch_add(idx, key, data + i * dimensions, usearch_scalar_f32_k, &error);
         ASSERT(!error, error);
     }
 
-    ASSERT(usearch_size(idx, &error) == vectors_count, error);
-    ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
+    ASSERT(usearch_size(idx, &error) == collection_size, error);
+    ASSERT(usearch_capacity(idx, &error) == collection_size, error);
 
     // Check vectors in the index
-    for (size_t i = 0; i < vectors_count; ++i) {
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
         ASSERT(usearch_contains(idx, key, &error), error);
     }
     ASSERT(!usearch_contains(idx, -1, &error), error); // Non existing key
 
+    free(data);
     usearch_free(idx, &error);
     printf("Test: Add Vector - PASSED\n");
 }
 
-void test_find_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
+void test_find_vector(size_t collection_size, size_t dimensions) {
     printf("Test: Find Vector...\n");
 
-    usearch_index_t idx = NULL;
     usearch_error_t error = NULL;
-    usearch_init_options_t opts = create_options(vector_dimension);
-    idx = usearch_init(&opts, &error);
-    usearch_reserve(idx, vectors_count, &error);
+    usearch_init_options_t opts = create_options(dimensions);
+    usearch_index_t idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, collection_size, &error);
 
     // Create result buffers
-    int results_count = 10;
+    int results_count = collection_size;
     usearch_key_t* keys = (usearch_key_t*)malloc(results_count * sizeof(usearch_key_t));
     float* distances = (float*)malloc(results_count * sizeof(float));
     ASSERT(keys && distances, "Failed to allocate memory");
 
     // Add vectors
-    for (size_t i = 0; i < vectors_count; ++i) {
+    float* data = create_vectors(collection_size, dimensions);
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
-        usearch_add(idx, key, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        usearch_add(idx, key, data + i * dimensions, usearch_scalar_f32_k, &error);
         ASSERT(!error, error);
     }
 
     // Find the vectors
-    for (size_t i = 0; i < vectors_count; i++) {
-        const void *query_vector = data + i * vector_dimension;
+    for (size_t i = 0; i < collection_size; i++) {
+        const void *query_vector = data + i * dimensions;
         size_t found_count = usearch_search(idx, query_vector, usearch_scalar_f32_k, results_count, keys, distances, &error);
         ASSERT(!error, error);
         ASSERT(found_count = results_count, "Vector is missing");
     }
 
+    free(data);
+    free(keys);
+    free(distances);
     usearch_free(idx, &error);
     printf("Test: Find Vector - PASSED\n");
 }
 
-void test_remove_vector(size_t vectors_count, size_t vector_dimension, float const* data) {
+void test_get_vector(size_t collection_size, size_t dimensions) {
+    printf("Test: Get Vector...\n");
+
+    usearch_error_t error = NULL;
+    usearch_init_options_t opts = create_options(dimensions);
+    opts.multi = true;
+    usearch_index_t idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, collection_size, &error);
+
+    // Create result buffers
+    int results_count = collection_size;
+    float* vectors = (float*)malloc(results_count * dimensions * sizeof(float));
+    ASSERT(vectors, "Failed to allocate memory");
+
+    // Add multiple vectors with SAME key
+    usearch_key_t const key = 1;
+    float* data = create_vectors(collection_size, dimensions);
+    for (size_t i = 0; i < collection_size; i++) {
+        usearch_add(idx, key, data + i * dimensions, usearch_scalar_f32_k, &error);
+        ASSERT(!error, error);
+    }
+
+    // Retrieve vectors from index
+    size_t count = usearch_get(idx, key, collection_size, vectors, usearch_scalar_f32_k, &error);
+    ASSERT(count == results_count, "Vector is missing");
+
+    free(vectors);
+    free(data);
+    usearch_free(idx, &error);
+
+    printf("Test: Get Vector - PASSED\n");
+}
+
+void test_remove_vector(size_t collection_size, size_t dimensions) {
     printf("Test: Remove Vector...\n");
 
-    usearch_index_t idx = NULL;
     usearch_error_t error = NULL;
-    usearch_init_options_t opts = create_options(vector_dimension);
-    idx = usearch_init(&opts, &error);
-    usearch_reserve(idx, vectors_count, &error);
+    usearch_init_options_t opts = create_options(dimensions);
+    usearch_index_t idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, collection_size, &error);
 
     // Add vectors
-    for (size_t i = 0; i < vectors_count; ++i) {
+    float* data = create_vectors(collection_size, dimensions);
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
-        usearch_add(idx, key, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        usearch_add(idx, key, data + i * dimensions, usearch_scalar_f32_k, &error);
         ASSERT(!error, error);
     }
 
     // Remove the vectors
-    for (size_t i = 0; i < vectors_count; i++) {
+    for (size_t i = 0; i < collection_size; i++) {
         usearch_key_t key = i;
         usearch_remove(idx, key, &error);
-        ASSERT(error, "Currently, Remove is not supported");
+        ASSERT(!error, "Currently, Remove is not supported");
     }
 
+    free(data);
     usearch_free(idx, &error);
     printf("Test: Remove Vector - PASSED\n");
 }
 
-void test_save_load(size_t vectors_count, size_t vector_dimension, float const* data) {
+void test_save_load(size_t collection_size, size_t dimensions) {
     printf("Test: Save/Load...\n");
 
-    usearch_index_t idx = NULL;
     usearch_error_t error = NULL;
-    usearch_init_options_t opts = create_options(vector_dimension);
-    idx = usearch_init(&opts, &error);
-    usearch_reserve(idx, vectors_count, &error);
+    usearch_init_options_t opts = create_options(dimensions);
+    usearch_index_t idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, collection_size, &error);
 
     // Add vectors
-    for (size_t i = 0; i < vectors_count; ++i) {
+    float* data = create_vectors(collection_size, dimensions);
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
-        usearch_add(idx, key, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        usearch_add(idx, key, data + i * dimensions, usearch_scalar_f32_k, &error);
         ASSERT(!error, error);
     }
 
@@ -217,34 +226,35 @@ void test_save_load(size_t vectors_count, size_t vector_dimension, float const*
     // Load
     usearch_load(idx, "usearch_index.bin", &error);
     ASSERT(!error, error);
-    ASSERT(usearch_size(idx, &error) == vectors_count, error);
-    ASSERT(usearch_capacity(idx, &error) == vectors_count, error);
-    ASSERT(usearch_dimensions(idx, &error) == vector_dimension, error);
+    ASSERT(usearch_size(idx, &error) == collection_size, error);
+    ASSERT(usearch_capacity(idx, &error) == collection_size, error);
+    ASSERT(usearch_dimensions(idx, &error) == dimensions, error);
     ASSERT(usearch_connectivity(idx, &error) == opts.connectivity, error);
 
     // Check vectors in the index
-    for (size_t i = 0; i < vectors_count; ++i) {
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
         ASSERT(usearch_contains(idx, key, &error), error);
     }
 
+    free(data);
     usearch_free(idx, &error);
     printf("Test: Save/Load - PASSED\n");
 }
 
-void test_view(size_t vectors_count, size_t vector_dimension, float const* data) {
+void test_view(size_t collection_size, size_t dimensions) {
     printf("Test: View...\n");
 
-    usearch_index_t idx = NULL;
     usearch_error_t error = NULL;
-    usearch_init_options_t opts = create_options(vector_dimension);
-    idx = usearch_init(&opts, &error);
-    usearch_reserve(idx, vectors_count, &error);
+    usearch_init_options_t opts = create_options(dimensions);
+    usearch_index_t idx = usearch_init(&opts, &error);
+    usearch_reserve(idx, collection_size, &error);
 
     // Add vectors
-    for (size_t i = 0; i < vectors_count; ++i) {
+    float* data = create_vectors(collection_size, dimensions);
+    for (size_t i = 0; i < collection_size; ++i) {
         usearch_key_t key = i;
-        usearch_add(idx, key, data + i * vector_dimension, usearch_scalar_f32_k, &error);
+        usearch_add(idx, key, data + i * dimensions, usearch_scalar_f32_k, &error);
         ASSERT(!error, error);
     }
 
@@ -262,33 +272,26 @@ void test_view(size_t vectors_count, size_t vector_dimension, float const* data)
     usearch_view(idx, "usearch_index.bin", &error);
     ASSERT(!error, error);
 
+    free(data);
     usearch_free(idx, &error);
     printf("Test: View - PASSED\n");
 }
 
-int main(int argc, char* argv[]) {
-    // Resolve file path
-    char const* filename = "sample.fvecs";
-    if (argc == 2)
-        filename = argv[1];
-
-    // Read data from the file
-    float* data = NULL;
-    size_t vectors_count = 0;
-    size_t vector_dimension = 0;
-    parse_vectors_from_file(filename, &vectors_count, &vector_dimension, &data);
-    vectors_count = vectors_count > 100000 ? 100000 : vectors_count; // Just limit vectors
-
-    // Test
-    test_init(vectors_count, vector_dimension);
-    test_add_vector(vectors_count, vector_dimension, data);
-    test_find_vector(vectors_count, vector_dimension, data);
-    test_remove_vector(vectors_count, vector_dimension, data);
-    test_save_load(vectors_count, vector_dimension, data);
-    test_view(vectors_count, vector_dimension, data);
-
-    // Free vectors' buffer
-    free(data);
+int main() {
+
+    size_t collection_sizes[] = {11, 512};
+    size_t dimensions[] = {83, 1};
+    for (size_t idx = 0; idx < sizeof(collection_sizes) / sizeof(collection_sizes[0]); ++idx) {
+        for (size_t jdx = 0; jdx < sizeof(dimensions) / sizeof(dimensions[0]); ++jdx) {
+            test_init(collection_sizes[idx], dimensions[jdx]);
+            test_add_vector(collection_sizes[idx], dimensions[jdx]);
+            test_find_vector(collection_sizes[idx], dimensions[jdx]);
+            test_get_vector(collection_sizes[idx], dimensions[jdx]);
+            test_remove_vector(collection_sizes[idx], dimensions[jdx]);
+            test_save_load(collection_sizes[idx], dimensions[jdx]);
+            test_view(collection_sizes[idx], dimensions[jdx]);
+        }
+    }
 
     return 0;
 }

package/c/usearch.h

@@ -109,6 +109,12 @@ USEARCH_EXPORT usearch_index_t usearch_init(usearch_init_options_t* options, use
  */
 USEARCH_EXPORT void usearch_free(usearch_index_t, usearch_error_t* error);
 
+/**
+ *  @brief Reports expected file size after serialization.
+ *  @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
+ */
+USEARCH_EXPORT size_t usearch_serialized_length(usearch_index_t, usearch_error_t* error);
+
 /**
  *  @brief Saves the index to a file.
  *  @param[in] path The file path where the index will be saved.
@@ -124,12 +130,33 @@ USEARCH_EXPORT void usearch_save(usearch_index_t, char const* path, usearch_erro
 USEARCH_EXPORT void usearch_load(usearch_index_t, char const* path, usearch_error_t* error);
 
 /**
- *  @brief Creates a view of the index from a file without loading it into memory.
+ *  @brief Creates a view of the index from a file without copying it into memory.
  *  @param[in] path The file path from where the view will be created.
  *  @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
  */
 USEARCH_EXPORT void usearch_view(usearch_index_t, char const* path, usearch_error_t* error);
 
+/**
+ *  @brief Saves the index to an in-memory buffer.
+ *  @param[in] buffer The in-memory continuous buffer where the index will be saved.
+ *  @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
+ */
+USEARCH_EXPORT void usearch_save_buffer(usearch_index_t, void* buffer, size_t length, usearch_error_t* error);
+
+/**
+ *  @brief Loads the index from an in-memory buffer.
+ *  @param[in] buffer The in-memory continuous buffer from where the index will be loaded.
+ *  @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
+ */
+USEARCH_EXPORT void usearch_load_buffer(usearch_index_t, void const* buffer, size_t length, usearch_error_t* error);
+
+/**
+ *  @brief Creates a view of the index from an in-memory buffer without copying it into memory.
+ *  @param[in] buffer The in-memory continuous buffer from where the view will be created.
+ *  @param[out] error Pointer to a string where the error message will be stored, if an error occurs.
+ */
+USEARCH_EXPORT void usearch_view_buffer(usearch_index_t, void const* buffer, size_t length, usearch_error_t* error);
+
 USEARCH_EXPORT size_t usearch_size(usearch_index_t, usearch_error_t* error);
 USEARCH_EXPORT size_t usearch_capacity(usearch_index_t, usearch_error_t* error);
 USEARCH_EXPORT size_t usearch_dimensions(usearch_index_t, usearch_error_t* error);

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '1.2.2'
+    version = '2.0.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/cpp/test.cpp

@@ -118,7 +118,7 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
     }
 
     // Search again over mapped index
-    // file_head_result_t head = index_dense_metadata("tmp.usearch");
+    // file_head_result_t head = index_dense_metadata_from_path("tmp.usearch");
     // expect(head.size == 3);
     index.view("tmp.usearch");
     matched_count = index.search(vector_first, 5, args...).dump_to(matched_keys, matched_distances);
@@ -140,7 +140,7 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
 
     // Check metadata
     if constexpr (punned_ak) {
-        index_dense_metadata_result_t meta = index_dense_metadata("tmp.usearch");
+        index_dense_metadata_result_t meta = index_dense_metadata_from_path("tmp.usearch");
         expect(bool(meta));
     }
 }