usearch 1.2.2 → 2.0.0

package/python/usearch/index.py

@@ -21,14 +21,21 @@ from typing import (
 import numpy as np
 from tqdm import tqdm
 
-from usearch.compiled import Index as _CompiledIndex
-from usearch.compiled import Indexes as _CompiledIndexes
-from usearch.compiled import IndexStats as _CompiledIndexStats
+# Precompiled symbols that won't be exposed directly:
+from usearch.compiled import (
+    Index as _CompiledIndex,
+    Indexes as _CompiledIndexes,
+    IndexStats as _CompiledIndexStats,
+    index_dense_metadata_from_path as _index_dense_metadata_from_path,
+    index_dense_metadata_from_buffer as _index_dense_metadata_from_buffer,
+    exact_search as _exact_search,
+)
 
-from usearch.compiled import index_dense_metadata as _index_dense_metadata
-from usearch.compiled import exact_search as _exact_search
-from usearch.compiled import MetricKind, ScalarKind, MetricSignature
+# Precompiled symbols that will be exposed
 from usearch.compiled import (
+    MetricKind,
+    ScalarKind,
+    MetricSignature,
     DEFAULT_CONNECTIVITY,
     DEFAULT_EXPANSION_ADD,
     DEFAULT_EXPANSION_SEARCH,
@@ -60,6 +67,10 @@ DTypeLike = Union[str, ScalarKind]
 
 MetricLike = Union[str, MetricKind, CompiledMetric]
 
+NoneType = type(None)
+
+PathOrBuffer = Union[str, os.PathLike, bytes]
+
 
 def _normalize_dtype(dtype, metric: MetricKind = MetricKind.Cos) -> ScalarKind:
     if dtype is None or dtype == "":
@@ -559,29 +570,38 @@ class Index:
                 self._compiled.load(path)
 
     @staticmethod
-    def metadata(path: os.PathLike) -> Optional[dict]:
-        path = os.fspath(path)
-        if not os.path.exists(path):
-            return None
+    def metadata(path_or_buffer: PathOrBuffer) -> Optional[dict]:
         try:
-            return _index_dense_metadata(path)
-        except Exception:
-            return None
+            if isinstance(path_or_buffer, bytearray):
+                path_or_buffer = bytes(path_or_buffer)
+            if isinstance(path_or_buffer, bytes):
+                return _index_dense_metadata_from_buffer(path_or_buffer)
+            else:
+                path_or_buffer = os.fspath(path_or_buffer)
+                if not os.path.exists(path_or_buffer):
+                    return None
+                return _index_dense_metadata_from_path(path_or_buffer)
+        except Exception as e:
+            raise e
 
     @staticmethod
-    def restore(path: os.PathLike, view: bool = False) -> Optional[Index]:
-        path = os.fspath(path)
-        meta = Index.metadata(path)
+    def restore(path_or_buffer: PathOrBuffer, view: bool = False) -> Optional[Index]:
+        meta = Index.metadata(path_or_buffer)
         if not meta:
             return None
-        return Index(
+
+        index = Index(
             ndim=meta["dimensions"],
             dtype=meta["kind_scalar"],
             metric=meta["kind_metric"],
-            path=path,
-            view=view,
         )
 
+        if view:
+            index.view(path_or_buffer)
+        else:
+            index.load(path_or_buffer)
+        return index
+
     def __len__(self) -> int:
         return self._compiled.__len__()
 
@@ -837,6 +857,10 @@ class Index:
     def ndim(self) -> int:
         return self._compiled.ndim
 
+    @property
+    def serialized_length(self) -> int:
+        return self._compiled.serialized_length
+
     @property
     def metric(self) -> Union[MetricKind, CompiledMetric]:
         return self._metric_jit if self._metric_jit else self._metric_kind
@@ -895,23 +919,34 @@ class Index:
     def expansion_search(self, v: int):
         self._compiled.expansion_search = v
 
-    def save(self, path: Optional[os.PathLike] = None):
-        path = path if path else self.path
-        if path is None:
-            raise Exception("Define `path` argument")
-        self._compiled.save(os.fspath(path))
-
-    def load(self, path: Optional[os.PathLike] = None):
-        path = path if path else self.path
-        if path is None:
-            raise Exception("Define `path` argument")
-        self._compiled.load(os.fspath(path))
-
-    def view(self, path: Optional[os.PathLike] = None):
-        path = path if path else self.path
-        if path is None:
-            raise Exception("Define `path` argument")
-        self._compiled.view(os.fspath(path))
+    def save(self, path_or_buffer: Union[str, os.PathLike, NoneType] = None) -> Optional[bytes]:
+        path_or_buffer = path_or_buffer if path_or_buffer else self.path
+        if path_or_buffer is None:
+            return self._compiled.save_index_to_buffer()
+        else:
+            self._compiled.save_index_to_path(os.fspath(path_or_buffer))
+
+    def load(self, path_or_buffer: Union[str, os.PathLike, bytes, NoneType] = None):
+        path_or_buffer = path_or_buffer if path_or_buffer else self.path
+        if path_or_buffer is None:
+            raise Exception("Define the source")
+        if isinstance(path_or_buffer, bytearray):
+            path_or_buffer = bytes(path_or_buffer)
+        if isinstance(path_or_buffer, bytes):
+            self._compiled.load_index_from_buffer(path_or_buffer)
+        else:
+            self._compiled.load_index_from_path(os.fspath(path_or_buffer))
+
+    def view(self, path_or_buffer: Union[str, os.PathLike, bytes, bytearray, NoneType] = None):
+        path_or_buffer = path_or_buffer if path_or_buffer else self.path
+        if path_or_buffer is None:
+            raise Exception("Define the source")
+        if isinstance(path_or_buffer, bytearray):
+            path_or_buffer = bytes(path_or_buffer)
+        if isinstance(path_or_buffer, bytes):
+            self._compiled.view_index_from_buffer(path_or_buffer)
+        else:
+            self._compiled.view_index_from_path(os.fspath(path_or_buffer))
 
     def clear(self):
         """Erases all the vectors from the index, preserving the space for future insertions."""
@@ -1049,12 +1084,27 @@ class Index:
         return self._compiled.max_level + 1
 
     @property
-    def levels_stats(self) -> _CompiledIndexStats:
+    def stats(self) -> _CompiledIndexStats:
         """Get the accumulated statistics for the entire multi-level graph.
 
         :return: Statistics for the entire multi-level graph.
         :rtype: _CompiledIndexStats
 
+        Statistics:
+            - ``nodes`` (int): The number of nodes in that level.
+            - ``edges`` (int): The number of edges in that level.
+            - ``max_edges`` (int): The maximum possible number of edges in that level.
+            - ``allocated_bytes`` (int): The amount of allocated memory for that level.
+        """
+        return self._compiled.stats
+
+    @property
+    def levels_stats(self) -> List[_CompiledIndexStats]:
+        """Get the accumulated statistics for every level graph.
+
+        :return: Statistics for every level graph.
+        :rtype: List[_CompiledIndexStats]
+
         Statistics:
             - ``nodes`` (int): The number of nodes in that level.
             - ``edges`` (int): The number of edges in that level.

package/rust/README.md

@@ -11,7 +11,7 @@ cargo add usearch
 ```rust
 
 let options = IndexOptions {
-            dimensions: 5,
+            dimensions: 3,
             metric: MetricKind::IP,
             quantization: ScalarKind::F16,
             connectivity: 0,
@@ -36,7 +36,7 @@ assert_eq!(index.size(), 2);
 
 // Read back the tags
 let results = index.search(&first, 10).unwrap();
-assert_eq!(results.count, 2);
+assert_eq!(results.keys.len(), 2);
 ```
 
 ## Multi-Threading

package/rust/lib.cpp

@@ -9,9 +9,35 @@ using add_result_t = typename index_t::add_result_t;
 using search_result_t = typename index_t::search_result_t;
 using labeling_result_t = typename index_t::labeling_result_t;
 
+template <typename scalar_at> Matches search_(index_dense_t& index, scalar_at const* vec, size_t count) {
+    Matches matches;
+    matches.keys.reserve(count);
+    matches.distances.reserve(count);
+    for (size_t i = 0; i != count; ++i)
+        matches.keys.push_back(0), matches.distances.push_back(0);
+    search_result_t result = index.search(vec, count);
+    result.error.raise();
+    count = result.dump_to(matches.keys.data(), matches.distances.data());
+    matches.keys.truncate(count);
+    matches.distances.truncate(count);
+    return matches;
+}
+
 Index::Index(std::unique_ptr<index_t> index) : index_(std::move(index)) {}
 
-void Index::add(key_t key, rust::Slice<float const> vector) const { index_->add(key, vector.data()).error.raise(); }
+// clang-format off
+void Index::add(key_t key, rust::Slice<float const> vec) const { index_->add(key, vec.data()).error.raise(); }
+void Index::add_i8(key_t key, rust::Slice<int8_t const> vec) const { index_->add(key, vec.data()).error.raise(); }
+void Index::add_f16(key_t key, rust::Slice<uint16_t const> vec) const { index_->add(key, (f16_t const*)vec.data()).error.raise(); }
+void Index::add_f32(key_t key, rust::Slice<float const> vec) const { index_->add(key, vec.data()).error.raise(); }
+void Index::add_f64(key_t key, rust::Slice<double const> vec) const { index_->add(key, vec.data()).error.raise(); }
+
+Matches Index::search(rust::Slice<float const> vec, size_t count) const { return search_(*index_, vec.data(), count); }
+Matches Index::search_i8(rust::Slice<int8_t const> vec, size_t count) const { return search_(*index_, vec.data(), count); }
+Matches Index::search_f16(rust::Slice<uint16_t const> vec, size_t count) const { return search_(*index_, (f16_t const*)vec.data(), count); }
+Matches Index::search_f32(rust::Slice<float const> vec, size_t count) const { return search_(*index_, vec.data(), count); }
+Matches Index::search_f64(rust::Slice<double const> vec, size_t count) const { return search_(*index_, vec.data(), count); }
+// clang-format on
 
 bool Index::remove(key_t key) const {
     labeling_result_t result = index_->remove(key);
@@ -27,32 +53,29 @@ bool Index::rename(key_t from, key_t to) const {
 
 bool Index::contains(key_t key) const { return index_->contains(key); }
 
-Matches Index::search(rust::Slice<float const> vector, size_t count) const {
-    Matches matches;
-    matches.keys.reserve(count);
-    matches.distances.reserve(count);
-    for (size_t i = 0; i != count; ++i)
-        matches.keys.push_back(0), matches.distances.push_back(0);
-    search_result_t result = index_->search(vector.data(), count);
-    result.error.raise();
-    count = result.dump_to(matches.keys.data(), matches.distances.data());
-    matches.keys.truncate(count);
-    matches.distances.truncate(count);
-    return matches;
-}
-
 void Index::reserve(size_t capacity) const { index_->reserve(capacity); }
 
 size_t Index::dimensions() const { return index_->dimensions(); }
 size_t Index::connectivity() const { return index_->connectivity(); }
 size_t Index::size() const { return index_->size(); }
 size_t Index::capacity() const { return index_->capacity(); }
+size_t Index::serialized_length() const { return index_->serialized_length(); }
 
-void Index::save(rust::Str path) const { index_->save(std::string(path).c_str()); }
-void Index::load(rust::Str path) const { index_->load(std::string(path).c_str()); }
-void Index::view(rust::Str path) const { index_->view(std::string(path).c_str()); }
+void Index::save(rust::Str path) const { index_->save(output_file_t(std::string(path).c_str())).error.raise(); }
+void Index::load(rust::Str path) const { index_->load(input_file_t(std::string(path).c_str())).error.raise(); }
+void Index::view(rust::Str path) const { index_->view(memory_mapped_file_t(std::string(path).c_str())).error.raise(); }
+
+void Index::save_to_buffer(rust::Slice<uint8_t> buffer) const {
+    index_->save(memory_mapped_file_t((byte_t*)buffer.data(), buffer.size())).error.raise();
+}
 
-scalar_kind_t quantization(rust::Str quant) { return scalar_kind_from_name(quant.data(), quant.size()); }
+void Index::load_from_buffer(rust::Slice<uint8_t const> buffer) const {
+    index_->load(memory_mapped_file_t((byte_t*)buffer.data(), buffer.size())).error.raise();
+}
+
+void Index::view_from_buffer(rust::Slice<uint8_t const> buffer) const {
+    index_->view(memory_mapped_file_t((byte_t*)buffer.data(), buffer.size())).error.raise();
+}
 
 std::unique_ptr<Index> wrap(index_t&& index) {
     std::unique_ptr<index_t> punned_ptr;

package/rust/lib.hpp

@@ -22,7 +22,16 @@ class Index {
     void reserve(size_t) const;
 
     void add(key_t key, rust::Slice<float const> vector) const;
+    void add_i8(key_t key, rust::Slice<int8_t const> vector) const;
+    void add_f16(key_t key, rust::Slice<uint16_t const> vector) const;
+    void add_f32(key_t key, rust::Slice<float const> vector) const;
+    void add_f64(key_t key, rust::Slice<double const> vector) const;
+
     Matches search(rust::Slice<float const> vector, size_t count) const;
+    Matches search_i8(rust::Slice<int8_t const> vector, size_t count) const;
+    Matches search_f16(rust::Slice<uint16_t const> vector, size_t count) const;
+    Matches search_f32(rust::Slice<float const> vector, size_t count) const;
+    Matches search_f64(rust::Slice<double const> vector, size_t count) const;
 
     bool remove(key_t key) const;
     bool contains(key_t key) const;
@@ -32,11 +41,16 @@ class Index {
     size_t connectivity() const;
     size_t size() const;
     size_t capacity() const;
+    size_t serialized_length() const;
 
     void save(rust::Str path) const;
     void load(rust::Str path) const;
     void view(rust::Str path) const;
 
+    void save_to_buffer(rust::Slice<uint8_t> buffer) const;
+    void load_from_buffer(rust::Slice<uint8_t const> buffer) const;
+    void view_from_buffer(rust::Slice<uint8_t const> buffer) const;
+
   private:
     std::unique_ptr<index_dense_t> index_;
 };

package/rust/lib.rs

@@ -1,6 +1,6 @@
 #[cxx::bridge]
 pub mod ffi {
-    
+
     // Shared structs with fields visible to both languages.
     struct Matches {
         keys: Vec<u64>,
@@ -43,20 +43,20 @@ pub mod ffi {
         type Index;
 
         /// Initializes a new instance of the index with the provided options.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `options` - A reference to the `IndexOptions` structure containing initialization options.
-        /// 
+        ///
         /// # Returns
-        /// 
+        ///
         /// A `Result` which is `Ok` if the index is successfully initialized, or `Err` if an error occurs.
         pub fn new_index(options: &IndexOptions) -> Result<UniquePtr<Index>>;
 
         /// Reserves memory for a specified number of incoming vectors.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `capacity` - The desired total capacity including the current size.
         pub fn reserve(self: &Index, capacity: usize) -> Result<()>;
 
@@ -72,128 +72,160 @@ pub mod ffi {
         /// Retrieves the total capacity of the index, including reserved space.
         pub fn capacity(self: &Index) -> usize;
 
+        /// Reports expected file size after serialization.
+        pub fn serialized_length(self: &Index) -> usize;
+
         /// Adds a vector with a specified key to the index.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `key` - The key associated with the vector.
         /// * `vector` - A slice containing the vector data.
         pub fn add(self: &Index, key: u64, vector: &[f32]) -> Result<()>;
 
+        pub fn add_i8(self: &Index, key: u64, vector: &[i8]) -> Result<()>;
+        pub fn add_f16(self: &Index, key: u64, vector: &[u16]) -> Result<()>;
+        pub fn add_f32(self: &Index, key: u64, vector: &[f32]) -> Result<()>;
+        pub fn add_f64(self: &Index, key: u64, vector: &[f64]) -> Result<()>;
+
         /// Performs k-Approximate Nearest Neighbors (kANN) Search for closest vectors to the provided query.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `query` - A slice containing the query vector data.
         /// * `count` - The maximum number of neighbors to search for.
-        /// 
+        ///
         /// # Returns
-        /// 
+        ///
         /// A `Result` containing the matches found.
         pub fn search(self: &Index, query: &[f32], count: usize) -> Result<Matches>;
 
+        pub fn search_i8(self: &Index, query: &[i8], count: usize) -> Result<Matches>;
+        pub fn search_f16(self: &Index, query: &[u16], count: usize) -> Result<Matches>;
+        pub fn search_f32(self: &Index, query: &[f32], count: usize) -> Result<Matches>;
+        pub fn search_f64(self: &Index, query: &[f64], count: usize) -> Result<Matches>;
+
         /// Removes the vector associated with the given key from the index.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `key` - The key of the vector to be removed.
-        /// 
+        ///
         /// # Returns
-        /// 
+        ///
         /// `true` if the vector is successfully removed, `false` otherwise.
         pub fn remove(self: &Index, key: u64) -> Result<bool>;
 
         /// Renames the vector under a certain key.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `from` - The key of the vector to be renamed.
         /// * `to` - The new name.
-        /// 
+        ///
         /// # Returns
-        /// 
+        ///
         /// `true` if the vector is successfully renamed, `false` otherwise.
         pub fn rename(self: &Index, from: u64, to: u64) -> Result<bool>;
 
         /// Checks if the index contains a vector with a specified key.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `key` - The key to be checked.
-        /// 
+        ///
         /// # Returns
-        /// 
+        ///
         /// `true` if the index contains the vector with the given key, `false` otherwise.
         pub fn contains(self: &Index, key: u64) -> bool;
 
         /// Saves the index to a specified file.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `path` - The file path where the index will be saved.
         pub fn save(self: &Index, path: &str) -> Result<()>;
 
         /// Loads the index from a specified file.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `path` - The file path from where the index will be loaded.
         pub fn load(self: &Index, path: &str) -> Result<()>;
 
         /// Creates a view of the index from a file without loading it into memory.
-        /// 
+        ///
         /// # Arguments
-        /// 
+        ///
         /// * `path` - The file path from where the view will be created.
         pub fn view(self: &Index, path: &str) -> Result<()>;
 
+        /// Saves the index to a specified file.
+        ///
+        /// # Arguments
+        ///
+        /// * `path` - The file path where the index will be saved.
+        pub fn save_to_buffer(self: &Index, buffer: &mut [u8]) -> Result<()>;
+
+        /// Loads the index from a specified file.
+        ///
+        /// # Arguments
+        ///
+        /// * `path` - The file path from where the index will be loaded.
+        pub fn load_from_buffer(self: &Index, buffer: &[u8]) -> Result<()>;
+
+        /// Creates a view of the index from a file without loading it into memory.
+        ///
+        /// # Arguments
+        ///
+        /// * `path` - The file path from where the view will be created.
+        pub fn view_from_buffer(self: &Index, buffer: &[u8]) -> Result<()>;
+
     }
 }
 
-
 #[cfg(test)]
 mod tests {
+    use crate::ffi::new_index;
+    use crate::ffi::IndexOptions;
     use crate::ffi::MetricKind;
     use crate::ffi::ScalarKind;
-    use crate::ffi::IndexOptions;
-    use crate::ffi::new_index;
 
     #[test]
     fn integration() {
-
         let mut options = IndexOptions {
             dimensions: 5,
             metric: MetricKind::IP,
             quantization: ScalarKind::F16,
             connectivity: 0,
             expansion_add: 0,
-            expansion_search: 0
+            expansion_search: 0,
         };
 
         let index = new_index(&options).unwrap();
-    
+
         assert!(index.reserve(10).is_ok());
         assert!(index.capacity() >= 10);
         assert!(index.connectivity() != 0);
         assert_eq!(index.dimensions(), 5);
         assert_eq!(index.size(), 0);
-    
+
         let first: [f32; 5] = [0.2, 0.1, 0.2, 0.1, 0.3];
         let second: [f32; 5] = [0.2, 0.1, 0.2, 0.1, 0.3];
-    
+
         assert!(index.add(42, &first).is_ok());
         assert!(index.add(43, &second).is_ok());
         assert_eq!(index.size(), 2);
-    
+
         // Read back the tags
         let results = index.search(&first, 10).unwrap();
         assert_eq!(results.keys.len(), 2);
-    
+
         // Validate serialization
         assert!(index.save("index.rust.usearch").is_ok());
         assert!(index.load("index.rust.usearch").is_ok());
         assert!(index.view("index.rust.usearch").is_ok());
-    
+
         // Make sure every function is called at least once
         assert!(new_index(&options).is_ok());
         options.metric = MetricKind::L2sq;
@@ -202,5 +234,15 @@ mod tests {
         assert!(new_index(&options).is_ok());
         options.metric = MetricKind::Haversine;
         assert!(new_index(&options).is_ok());
+
+        let mut serialization_buffer = Vec::new();
+        serialization_buffer.resize(index.serialized_length(), 0);
+        assert!(index.save_to_buffer(&mut serialization_buffer).is_ok());
+
+        let deserialized_index = new_index(&options).unwrap();
+        assert!(deserialized_index
+            .load_from_buffer(&serialization_buffer)
+            .is_ok());
+        assert_eq!(index.size(), deserialized_index.size());
     }
-}
+}

package/wasmer.toml

@@ -1,6 +1,6 @@
 [package]
 name="unum/usearch"
-version="1.2.2"
+version="2.0.0"
 description="Smaller & Faster Single-File Vector Search Engine from Unum"
 license="Apache-2.0"
 readme="README.md"