usearch 2.6.0 → 2.7.0

package/rust/lib.rs

@@ -1,12 +1,13 @@
 #[cxx::bridge]
 pub mod ffi {
-
     // Shared structs with fields visible to both languages.
+    #[derive(Debug)]
     struct Matches {
         keys: Vec<u64>,
         distances: Vec<f32>,
     }
 
+    #[derive(Debug)]
     enum MetricKind {
         IP,
         L2sq,
@@ -18,6 +19,7 @@ pub mod ffi {
         Sorensen,
     }
 
+    #[derive(Debug)]
     enum ScalarKind {
         F64,
         F32,
@@ -26,6 +28,7 @@ pub mod ffi {
         B1,
     }
 
+    #[derive(Debug, PartialEq)]
     struct IndexOptions {
         dimensions: usize,
         metric: MetricKind,
@@ -40,7 +43,7 @@ pub mod ffi {
     unsafe extern "C++" {
         include!("lib.hpp");
 
-        /// Low-level C++ interface, that is further wrapped into the high-level `Index`
+        /// Low-level C++ interface that is further wrapped into the high-level `Index`
         type NativeIndex;
 
         pub fn new_native_index(options: &IndexOptions) -> Result<UniquePtr<NativeIndex>>;
@@ -74,6 +77,9 @@ pub mod ffi {
         pub fn save(self: &NativeIndex, path: &str) -> Result<()>;
         pub fn load(self: &NativeIndex, path: &str) -> Result<()>;
         pub fn view(self: &NativeIndex, path: &str) -> Result<()>;
+        pub fn reset(self: &NativeIndex) -> Result<()>;
+        pub fn memory_usage(self: &NativeIndex) -> usize;
+
         pub fn save_to_buffer(self: &NativeIndex, buffer: &mut [u8]) -> Result<()>;
         pub fn load_from_buffer(self: &NativeIndex, buffer: &[u8]) -> Result<()>;
         pub fn view_from_buffer(self: &NativeIndex, buffer: &[u8]) -> Result<()>;
@@ -100,6 +106,20 @@ impl Default for ffi::IndexOptions {
     }
 }
 
+impl Clone for ffi::IndexOptions {
+    fn clone(&self) -> Self {
+        ffi::IndexOptions {
+            dimensions: (self.dimensions),
+            metric: (self.metric),
+            quantization: (self.quantization),
+            connectivity: (self.connectivity),
+            expansion_add: (self.expansion_add),
+            expansion_search: (self.expansion_search),
+            multi: (self.multi),
+        }
+    }
+}
+
 pub trait VectorType {
     fn add(index: &Index, key: u64, vector: &[Self]) -> Result<(), cxx::Exception>
     where
@@ -185,6 +205,11 @@ impl Index {
     }
 
     /// Extracts one or more vectors matching specified key.
+    /// The `vector` slice must be a multiple number of dimensions in the index.
+    /// After the execution return the number `X` of vectors found.
+    /// The first `X * dimensions` elements of the vector slice will be filled.
+    ///
+    /// If you are a novice user, consider `export`.
     ///
     /// # Arguments
     ///
@@ -198,11 +223,34 @@ impl Index {
         T::get(self, key, vector)
     }
 
+    /// Extracts one or more vectors matching specified key into supplied resizable vector.
+    /// The `vector` is resized to a multiple number of dimensions in the index.
+    ///
+    /// # Arguments
+    ///
+    /// * `key` - The key associated with the vector.
+    /// * `vector` - A mutable vector containing the vector data.
+    pub fn export<T: VectorType + Default + Clone>(
+        self: &Index,
+        key: u64,
+        vector: &mut Vec<T>,
+    ) -> Result<usize, cxx::Exception> {
+        let dim = self.dimensions();
+        let max_matches = self.count(key);
+        vector.resize(dim * max_matches, T::default());
+        let matches = T::get(self, key, &mut vector[..]);
+        if matches.is_err() {
+            return matches;
+        }
+        vector.resize(dim * matches.as_ref().unwrap(), T::default());
+        return matches;
+    }
+
     /// Reserves memory for a specified number of incoming vectors.
     ///
     /// # Arguments
     ///
-    /// * `capacity` - The desired total capacity including the current size.
+    /// * `capacity` - The desired total capacity, including the current size.
     pub fn reserve(self: &Index, capacity: usize) -> Result<(), cxx::Exception> {
         self.inner.reserve(capacity)
     }
@@ -245,7 +293,7 @@ impl Index {
         self.inner.remove(key)
     }
 
-    /// Renames the vector under a certain key.
+    /// Renames the vector under a specific key.
     ///
     /// # Arguments
     ///
@@ -254,7 +302,7 @@ impl Index {
     ///
     /// # Returns
     ///
-    /// `true` if the vector is successfully renamed, `false` otherwise.
+    /// `true` if the vector is renamed, `false` otherwise.
     pub fn rename(self: &Index, from: u64, to: u64) -> Result<usize, cxx::Exception> {
         self.inner.rename(from, to)
     }
@@ -272,7 +320,7 @@ impl Index {
         self.inner.contains(key)
     }
 
-    /// Count the count of vector with the same specified key.
+    /// Count the count of vectors with the same specified key.
     ///
     /// # Arguments
     ///
@@ -312,6 +360,17 @@ impl Index {
         self.inner.view(path)
     }
 
+    /// Erases all members from the index, closes files, and returns RAM to OS.
+    pub fn reset(self: &Index) -> Result<(), cxx::Exception> {
+        self.inner.reset()
+    }
+
+    /// A relatively accurate lower bound on the amount of memory consumed by the system.
+    /// In practice, its error will be below 10%.
+    pub fn memory_usage(self: &Index) -> usize {
+        self.inner.memory_usage()
+    }
+
     /// Saves the index to a specified file.
     ///
     /// # Arguments
@@ -352,6 +411,36 @@ mod tests {
     use crate::new_index;
     use crate::Index;
 
+    #[test]
+    fn test_add_get_vector() {
+        let mut options = IndexOptions::default();
+        options.dimensions = 5;
+        let index = Index::new(&options).unwrap();
+        assert!(index.reserve(10).is_ok());
+
+        let first: [f32; 5] = [0.2, 0.1, 0.2, 0.1, 0.3];
+        let second: [f32; 5] = [0.3, 0.2, 0.4, 0.0, 0.1];
+        assert!(index.add(1, &first).is_ok());
+        assert!(index.add(2, &second).is_ok());
+        assert_eq!(index.size(), 2);
+
+        // Test using Vec<T>
+        let mut found_vec: Vec<f32> = Vec::new();
+        assert_eq!(index.export(1, &mut found_vec).unwrap(), 1);
+        assert_eq!(found_vec.len(), 5);
+        assert_eq!(found_vec, first.to_vec());
+
+        // Test using slice
+        let mut found_slice = [0.0 as f32; 5];
+        assert_eq!(index.get(1, &mut found_slice).unwrap(), 1);
+        assert_eq!(found_slice, first);
+
+        // Create a slice with incorrect size
+        let mut found = [0.0 as f32; 6]; // This isn't a multiple of the index's dimensions.
+        let result = index.get(1, &mut found);
+        assert!(result.is_err());
+    }
+
     #[test]
     fn integration() {
         let mut options = IndexOptions::default();
@@ -366,14 +455,29 @@ mod tests {
         assert_eq!(index.size(), 0);
 
         let first: [f32; 5] = [0.2, 0.1, 0.2, 0.1, 0.3];
-        let second: [f32; 5] = [0.2, 0.1, 0.2, 0.1, 0.3];
-
+        let second: [f32; 5] = [0.3, 0.2, 0.4, 0.0, 0.1];
+
+        println!(
+            "before add, memory_usage: {} \
+            cap: {} \
+            ",
+            index.memory_usage(),
+            index.capacity(),
+        );
         assert!(index.add(42, &first).is_ok());
         assert!(index.add(43, &second).is_ok());
         assert_eq!(index.size(), 2);
+        println!(
+            "after add, memory_usage: {} \
+            cap: {} \
+            ",
+            index.memory_usage(),
+            index.capacity(),
+        );
 
         // Read back the tags
         let results = index.search(&first, 10).unwrap();
+        println!("{:?}", results);
         assert_eq!(results.keys.len(), 2);
 
         // Validate serialization
@@ -399,5 +503,21 @@ mod tests {
             .load_from_buffer(&serialization_buffer)
             .is_ok());
         assert_eq!(index.size(), deserialized_index.size());
+
+        // reset
+        assert_ne!(index.memory_usage(), 0);
+        assert!(index.reset().is_ok());
+        assert_eq!(index.size(), 0);
+        assert_eq!(index.memory_usage(), 0);
+
+        // clone
+        options.metric = MetricKind::Haversine;
+        let mut opts = options.clone();
+        assert_eq!(opts.metric, options.metric);
+        assert_eq!(opts.quantization, options.quantization);
+        assert_eq!(opts, options);
+        opts.metric = MetricKind::Cos;
+        assert_ne!(opts.metric, options.metric);
+        assert!(new_index(&opts).is_ok());
     }
 }

package/setup.py

@@ -12,6 +12,7 @@ macros_args = []
 if sys.platform == "linux":
     compile_args.append("-std=c++17")
     compile_args.append("-O3")  # Maximize performance
+    compile_args.append("-ffast-math")  # Maximize floating-point performance
     compile_args.append("-Wno-unknown-pragmas")
     compile_args.append("-fdiagnostics-color=always")
 
@@ -30,20 +31,29 @@ if sys.platform == "darwin":
     compile_args.append("-mmacosx-version-min=10.15")
     compile_args.append("-std=c++17")
     compile_args.append("-O3")  # Maximize performance
+    compile_args.append("-ffast-math")  # Maximize floating-point performance
     compile_args.append("-fcolor-diagnostics")
     compile_args.append("-Wno-unknown-pragmas")
 
     # Simplify debugging, but the normal `-g` may make builds much longer!
     compile_args.append("-g1")
 
-    # Linking OpenMP requires additional preparation in CIBuildWheel
-    # macros_args.append(("USEARCH_USE_OPENMP", "1"))
-    # compile_args.append("-Xpreprocessor -fopenmp")
-    # link_args.append("-Xpreprocessor -lomp")
+    # Linking OpenMP requires additional preparation in CIBuildWheel.
+    # We must install `brew install llvm` ahead of time.
+    # import subprocess as cli
+    # llvm_base = cli.check_output(["brew", "--prefix", "llvm"]).strip().decode("utf-8")
+    # if len(llvm_base):
+    #     compile_args.append(f"-I{llvm_base}/include")
+    #     compile_args.append("-Xpreprocessor -fopenmp")
+    #     link_args.append(f"-L{llvm_base}/lib")
+    #     link_args.append("-lomp")
+    #     macros_args.append(("USEARCH_USE_OPENMP", "1"))
 
 if sys.platform == "win32":
     compile_args.append("/std:c++17")
     compile_args.append("/O2")
+    compile_args.append("/fp:fast")  # Enable fast math for MSVC
+    compile_args.append("/W1")  # Reduce warnings verbosity
 
 ext_modules = [
     Pybind11Extension(

package/wasm/CMakeLists.txt

@@ -1,15 +1,7 @@
 include(emscripten)
 
-set(CMAKE_C_COMPILER emcc)
-set(CMAKE_CXX_COMPILER em++)
+# Setting the compiler globally is never a good idea! set(CMAKE_C_COMPILER emcc) set(CMAKE_CXX_COMPILER em++)
 
-set(USEARCH_PUNNED_INCLUDE_DIRS
-    "${CMAKE_CURRENT_SOURCE_DIR}/../c"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../include"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../fp16/include"
-)
-
-add_executable(index lib.cpp)
-target_include_directories(index PRIVATE ${USEARCH_PUNNED_INCLUDE_DIRS})
-set_target_properties(index PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/wasm)
-set_target_properties(index PROPERTIES CXX_STANDARD 11)
+add_executable(usearch_wasm lib.cpp)
+setup_target(usearch_wasm)
+set_target_properties(usearch_wasm PROPERTIES CXX_STANDARD 11)

package/wasm/lib.cpp

@@ -2,4 +2,4 @@
 
 #include <emscripten/emscripten.h>
 
-#include <lib.cpp>
+#include <../c/lib.cpp>

package/wasmer.toml

@@ -1,6 +1,6 @@
 [package]
 name="unum/usearch"
-version="2.6.0"
+version="2.7.0"
 description="Smaller & Faster Single-File Vector Search Engine from Unum"
 license="Apache-2.0"
 readme="README.md"

package/javascript/docs.js

@@ -1,111 +0,0 @@
-/** Search result object. */
-class Matches {
-    /**
-     * @param {BigUint64Array} keys - The keys of the nearest neighbors found.
-     * @param {Float32Array} distances - The distances of the nearest neighbors found.
-     * @param {bigint} count - The count of nearest neighbors found.
-     */
-    constructor(keys, distances, count) {
-        this.keys = keys;
-        this.distances = distances;
-        this.count = count;
-    }
-}
-
-/** K-Approximate Nearest Neighbors search index. */
-class Index {
-    /**
-     * Constructs a new index.
-     * 
-     * @param {bigint} dimensions
-     * @param {string} metric
-     * @param {string} quantization
-     * @param {bigint} capacity
-     * @param {bigint} connectivity
-     * @param {bigint} expansion_add
-     * @param {bigint} expansion_search
-     */
-    constructor(
-        dimensions,
-        metric,
-        quantization,
-        capacity,
-        connectivity,
-        expansion_add,
-        expansion_search
-    ) {}
-
-    /**
-     * Returns the dimensionality of vectors.
-     * @return {bigint} The dimensionality of vectors.
-     */
-    dimensions() {}
-
-    /**
-     * Returns the bigint of vectors currently indexed.
-     * @return {bigint} The bigint of vectors currently indexed.
-     */
-    size() {}
-
-    /**
-     * Returns index capacity.
-     * @return {bigint} The capacity of index.
-     */
-    capacity() {}
-
-    /**
-     * Returns connectivity.
-     * @return {bigint} The connectivity of index.
-     */
-    connectivity() {}
-
-    /** 
-     * Write index to a file.
-     * @param {string} path File path to write.
-     */
-    save(path) {}
-
-    /** 
-     * Load index from a file.
-     * @param {string} path File path to read.
-     */
-    load(path) {}
-
-    /** 
-     * View index from a file, without loading into RAM.
-     * @param {string} path File path to read.
-     */
-    view(path) {}
-
-    /** 
-     * Add n vectors of dimension d to the index.
-     * 
-     * @param {bigint | bigint[]} keys Input identifiers for every vector.
-     * @param {Float32Array | Float32Array[]} mat Input matrix, matrix of size n * d.
-     */
-    add(keys, mat) {}
-
-    /** 
-     * Query n vectors of dimension d to the index. Return at most k vectors for each. 
-     * If there are not enough results for a query, the result array is padded with -1s.
-     *
-     * @param {Float32Array} mat Input vectors to search, matrix of size n * d.
-     * @param {bigint} k The bigint of nearest neighbors to search for.
-     * @return {Matches} Output of the search result.
-     */
-    search(mat, k) {}
-
-    /** 
-     * Check if an entry is contained in the index.
-     * 
-     * @param {bigint} key Identifier to look up.
-     */
-    contains(key) {}
-
-    /** 
-     * Remove a vector from the index.
-     * 
-     * @param {bigint} key Input identifier for every vector to be removed.
-     */
-    remove(key) {}
-}

package/javascript/test.js

@@ -1,32 +0,0 @@
-var assert = require('assert');
-var usearch = require('bindings')('usearch');
-
-// Single-entry operations
-
-var index = new usearch.Index({ metric: 'l2sq', connectivity: 16n, dimensions: 2n });
-assert.equal(index.connectivity(), 16n, 'connectivity should be 16');
-assert.equal(index.dimensions(), 2n, 'dimensions should be 2');
-assert.equal(index.size(), 0n, 'initial size should be 0');
-
-index.add(15n, new Float32Array([10, 20]));
-index.add(16n, new Float32Array([10, 25]));
-assert.equal(index.size(), 2n, 'size after adding elements should be 2');
-assert.equal(index.contains(15n), true, 'entry must be present after insertion');
-
-var results = index.search(new Float32Array([13, 14]), 2n);
-assert.deepEqual(results.keys, new BigUint64Array([15n, 16n]), 'keys should be 15 and 16');
-assert.deepEqual(results.distances, new Float32Array([45, 130]), 'distances should be 45 and 130');
-
-// Batch operations
-
-var indexBatch = new usearch.Index({ metric: 'l2sq', connectivity: 16n, dimensions: 2n });
-const keys = [15n, 16n];
-const vectors = [new Float32Array([10, 20]), new Float32Array([10, 25])];
-indexBatch.add(keys, vectors);
-assert.equal(indexBatch.size(), 2, 'size after adding batch should be 2');
-
-results = indexBatch.search(new Float32Array([13, 14]), 2n);
-assert.deepEqual(results.keys, new BigUint64Array([15n, 16n]), 'keys should be 15 and 16');
-assert.deepEqual(results.distances, new Float32Array([45, 130]), 'distances should be 45 and 130');
-
-console.log('JavaScript tests passed!');

package/javascript/usearch.d.ts

@@ -1,110 +0,0 @@
-
-/** Search result object. */
-export interface Matches {
-    /** The keys of the nearest neighbors found, size n*k. */
-    keys: BigUint64Array,
-    /** The distances of the nearest neighbors found, size n*k. */
-    distances: Float32Array,
-    /** The distances of the nearest neighbors found, size n*k. */
-    count: bigint
-}
-
-/** K-Approximate Nearest Neighbors search index. */
-export class Index {
-
-    /**
-     * Constructs a new index.
-     * 
-     * @param {bigint} dimensions
-     * @param {string} metric
-     * @param {string} quantization
-     * @param {bigint} capacity
-     * @param {bigint} connectivity
-     * @param {bigint} expansion_add
-     * @param {bigint} expansion_search
-     */
-    constructor(
-        dimensions: bigint,
-        metric: string,
-        quantization: string,
-        capacity: bigint,
-        connectivity: bigint,
-        expansion_add: bigint,
-        expansion_search: bigint
-    );
-
-    /**
-     * Returns the dimensionality of vectors.
-     * @return {bigint} The dimensionality of vectors.
-     */
-    dimensions(): bigint;
-
-    /**
-     * Returns the bigint of vectors currently indexed.
-     * @return {bigint} The bigint of vectors currently indexed.
-     */
-    size(): bigint;
-
-    /**
-     * Returns index capacity.
-     * @return {bigints} The capacity of index.
-     */
-    capacity(): bigint;
-
-    /**
-     * Returns connectivity.
-     * @return {bigint} The connectivity of index.
-     */
-    connectivity(): bigint;
-
-    /** 
-     * Write index to a file.
-     * @param {string} path File path to write.
-     */
-    save(path: string): void;
-
-    /** 
-     * Load index from a file.
-     * @param {string} path File path to read.
-     */
-    load(path: string): void;
-
-    /** 
-     * View index from a file, without loading into RAM.
-     * @param {string} path File path to read.
-     */
-    load(path: string): void;
-
-    /** 
-     * Add n vectors of dimension d to the index.
-     * 
-     * @param {bigint | bigint[]} keys Input identifiers for every vector.
-     * @param {Float32Array | Float32Array[]} mat Input matrix, matrix of size n * d.
-     */
-    add(keys: bigint | bigint[], mat: Float32Array | Float32Array[]): void;
-
-    /** 
-     * Query n vectors of dimension d to the index. Return at most k vectors for each. 
-     * If there are not enough results for a query, the result array is padded with -1s.
-     *
-     * @param {Float32Array} mat Input vectors to search, matrix of size n * d.
-     * @param {bigint} k The bigint of nearest neighbors to search for.
-     * @return {Matches} Output of the search result.
-     */
-    search(mat: Float32Array, k: bigint): Matches;
-
-    /** 
-     * Check if an entry is contained in the index.
-     * 
-     * @param {bigint} key Identifier to look up.
-     */
-    contains(key: bigint): boolean;
-
-    /** 
-     * Remove a vector from the index.
-     * 
-     * @param {bigint} key Input identifier for every vector to be removed.
-     */
-    remove(key: bigint): boolean;
-
-}

package/tsconfig.json

@@ -1,8 +0,0 @@
-{
-    "typedocOptions": {
-        "entryPoints": [
-            "javascript/usearch.d.ts"
-        ],
-        "out": "docs"
-    }
-}