usearch 2.1.1 → 2.1.2

package/.github/workflows/release.yml

@@ -30,7 +30,7 @@ jobs:
 
 
   rebase:
-    name: Rebase Development Branch
+    name: Rebase Dev. Branch
     needs: versioning
     runs-on: ubuntu-latest
     steps:
@@ -41,13 +41,14 @@ jobs:
 
       - name: Perform rebase
         run: |
+          git fetch origin main
           git checkout main-dev
-          git rebase main
+          git rebase origin/main
           
       - name: Push changes
         uses: CasperWA/push-protected@v2
         with:
-          token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
+          token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
           branch: main-dev
           unprotect_reviews: True
           force: True
@@ -239,12 +240,14 @@ jobs:
     - name: Calculate source hashes
       id: hashes
       run: |
-        source_zip_path=$(find $GITHUB_WORKSPACE/downloads -name "*.zip")
-        zip_hash=$(sha256sum "$source_zip_path" | awk '{print $1}')
-        source_tar_path=$(find $GITHUB_WORKSPACE/downloads -name "*.tar.gz" ! -name "docs.tar.gz")
-        tar_hash=$(sha256sum "$source_tar_path" | awk '{print $1}')
-        echo "zip_hash=$zip_hash" >> $GITHUB_OUTPUT
-        echo "tar_hash=$tar_hash" >> $GITHUB_OUTPUT
+        i=1
+        cd $GITHUB_WORKSPACE/downloads
+        for asset_name in $(ls); do
+          asset_hash=$(sha256sum $asset_name | awk '{print $1}')
+          echo "asset_name_$i=$asset_name" >> $GITHUB_OUTPUT
+          echo "asset_hash_$i=$asset_hash" >> $GITHUB_OUTPUT
+          i=$((i+1))
+        done
           
     - name: Upload hashes to release
       uses: irongut/EditRelease@v1.2.0
@@ -255,8 +258,14 @@ jobs:
         replacebody: false
         body: |
             ## Hashes
-            * Source code (zip) : `${{ steps.hashes.outputs.zip_hash }}` 
-            * Source code (tar.gz) : `${{ steps.hashes.outputs.tar_hash }}`
+            * ${{ steps.hashes.outputs.asset_name_1 }} : `${{ steps.hashes.outputs.asset_hash_1 }}`
+            * ${{ steps.hashes.outputs.asset_name_2 }} : `${{ steps.hashes.outputs.asset_hash_2 }}`
+            * ${{ steps.hashes.outputs.asset_name_3 }} : `${{ steps.hashes.outputs.asset_hash_3 }}`
+            * ${{ steps.hashes.outputs.asset_name_4 }} : `${{ steps.hashes.outputs.asset_hash_4 }}`
+            * ${{ steps.hashes.outputs.asset_name_5 }} : `${{ steps.hashes.outputs.asset_hash_5 }}`
+            * ${{ steps.hashes.outputs.asset_name_6 }} : `${{ steps.hashes.outputs.asset_hash_6 }}`
+            * ${{ steps.hashes.outputs.asset_name_7 }} : `${{ steps.hashes.outputs.asset_hash_7 }}`
+            * ${{ steps.hashes.outputs.asset_name_8 }} : `${{ steps.hashes.outputs.asset_hash_8 }}`
 
 
   build_wheels:

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "2.1.1"
+version = "2.1.2"
 authors = ["Ash Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/README.md

@@ -83,7 +83,7 @@ USearch is compact and broadly compatible without sacrificing performance, with
 Base functionality is identical to FAISS, and the interface must be familiar if you have ever investigated Approximate Nearest Neighbors search:
 
 ```py
-$ pip install usearch numpy
+$ pip install usearch
 
 import numpy as np
 from usearch.index import Index

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '2.1.1'
+    version = '2.1.2'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/docs/format.md

@@ -0,0 +1,63 @@
+# USearch Format Specification
+
+> ⚠️ **Note**: The format is expected to change in the 3.0 release to support sparse indexes and offer even faster startup times when `view`-ing large indexes from disk.
+
+## Table of Contents
+
+- [Dense Indexes](#dense-indexes)
+- [Core](#core)
+- [Fragmented Storage](#fragmented-storage)
+
+## Dense Indexes
+
+Dense index files consist of two main parts: [a binary matrix of all vectors](#matrix-blob) and the [index](#index-blob) itself.
+
+### Matrix BLOB
+
+The matrix BLOB is optionally prepended to the index file. It starts with the number of `rows` and `columns`, which can be either 32-bit or 64-bit unsigned integers.
+
+- The number of `rows` corresponds to the number of vectors in the index.
+- The number of `columns` corresponds to the number of bytes in a single vector.
+
+### Index BLOB
+
+The index BLOB is composed of three main sections:
+
+1. **Metadata**: Occupies 64 bytes and contains the following fields:
+    1. 7-byte magic string `usearch` to infer the MIME type of the file.
+    2. 3-byte version number, containing the `major`, `minor`, and `patch` release numbers.
+    3. 1-byte enums for `metric`, `scalar`, `key`, and `compressed_slot` types.
+    4. 8-byte integers for the number of present vectors, number of deleted vectors, and number of dimensions.
+    5. 1-byte flags for multi-vector support.
+
+2. **Levels**: Contains 1-byte integers representing the level each node occupies. This information is used to estimate memory requirements and to enable memory-mapping of the entire file.
+
+3. **Core**: Detailed in the [Core section](#core). This format is consistent across all types of indexes.
+
+## Fragmented Storage
+
+The matrix BLOB can be stored separately and is not required to successfully open the file.
+
+## Core
+
+The core section of the index is serialized as a sequence of nodes, each representing a point in the index. This section immediately follows the `levels` block in the serialized format.
+
+### Header
+
+The core section begins with a header containing the following fields:
+
+- `uint64_t size`: Number of nodes in the index.
+- `uint64_t connectivity`: Maximum number of edges for a node.
+- `uint64_t connectivity_base`: Base connectivity value.
+- `uint64_t max_level`: Maximum level in the multi-level graph.
+- `uint64_t entry_slot`: Entry slot for the graph.
+
+These fields are serialized in a contiguous block of bytes, in the order listed.
+
+### Levels
+
+Following the header, the `levels` block contains a sequence of 1-byte integers. Each integer represents the level of a node in the index. The number of levels is equal to the `size` field in the header.
+
+### Nodes
+
+After the `levels` block, the nodes are serialized. Each node is represented as a contiguous block of bytes. The size and format of these blocks are implementation-specific. Nodes are serialized in the same order as their corresponding levels in the `levels` block.

package/docs/format.rst

@@ -0,0 +1,5 @@
+========================
+Format
+========================
+
+.. mdinclude:: format.md

package/docs/index.rst

@@ -24,6 +24,7 @@ Overview
 
    compilation
    benchmarks
+   format
 
 .. toctree:: 
    :hidden:

package/include/usearch/index.hpp

@@ -1272,7 +1272,7 @@ struct dummy_executor_t {
  *  @brief  An example of what a USearch-compatible key-to-key mapping should look like.
  *
  *  This is particularly helpful for "Semantic Joins", where we map entries of one collection
- *  to entries of another. In assymetric setups, where A -> B is needed, but B -> A is not,
+ *  to entries of another. In asymmetric setups, where A -> B is needed, but B -> A is not,
  *  this can be passed to minimize memory usage.
  */
 struct dummy_key_to_key_mapping_t {
@@ -1284,16 +1284,16 @@ struct dummy_key_to_key_mapping_t {
 
 /**
  *  @brief  Checks if the provided object has a dummy type, emulating an interface,
- *          but performaing no real computation.
+ *          but performing no real computation.
  */
 template <typename object_at> static constexpr bool is_dummy() {
     using object_t = typename std::remove_all_extents<object_at>::type;
-    return std::is_same<object_t, dummy_predicate_t>::value || //
-           std::is_same<object_t, dummy_callback_t>::value ||  //
-           std::is_same<object_t, dummy_progress_t>::value ||  //
-           std::is_same<object_t, dummy_prefetch_t>::value ||  //
-           std::is_same<object_t, dummy_executor_t>::value ||  //
-           std::is_same<object_t, dummy_key_to_key_mapping_t>::value;
+    return std::is_same<typename std::decay<object_t>::type, dummy_predicate_t>::value || //
+           std::is_same<typename std::decay<object_t>::type, dummy_callback_t>::value ||  //
+           std::is_same<typename std::decay<object_t>::type, dummy_progress_t>::value ||  //
+           std::is_same<typename std::decay<object_t>::type, dummy_prefetch_t>::value ||  //
+           std::is_same<typename std::decay<object_t>::type, dummy_executor_t>::value ||  //
+           std::is_same<typename std::decay<object_t>::type, dummy_key_to_key_mapping_t>::value;
 }
 
 template <typename, typename at> struct has_reset_gt {
@@ -1513,6 +1513,8 @@ class memory_mapped_file_t {
 #else
         int descriptor = open(path_, O_RDONLY);
 #endif
+        if (descriptor < 0)
+            return result.failed(std::strerror(errno));
 
         // Estimate the file size
         struct stat file_stat;
@@ -2502,7 +2504,7 @@ class index_gt {
     }
 
     /**
-     *  @brief Identifies the closest cluster to the gived ::query. Thread-safe.
+     *  @brief Identifies the closest cluster to the given ::query. Thread-safe.
      *
      *  @param[in] query Content that will be compared against other entries in the index.
      *  @param[in] level The index level to target. Higher means lower resolution.
@@ -3325,7 +3327,7 @@ class index_gt {
         visits.clear();
 
         // Optional prefetching
-        if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
+        if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value)
             prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
 
         distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
@@ -3337,7 +3339,7 @@ class index_gt {
                 neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
 
                 // Optional prefetching
-                if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
+                if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value) {
                     candidates_range_t missing_candidates{*this, closest_neighbors, visits};
                     prefetch(missing_candidates.begin(), missing_candidates.end());
                 }
@@ -3379,7 +3381,7 @@ class index_gt {
             return false;
 
         // Optional prefetching
-        if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
+        if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value)
             prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
 
         distance_t radius = context.measure(query, citerator_at(start_slot), metric);
@@ -3404,7 +3406,7 @@ class index_gt {
             neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
 
             // Optional prefetching
-            if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
+            if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value) {
                 candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
                 prefetch(missing_candidates.begin(), missing_candidates.end());
             }
@@ -3453,7 +3455,7 @@ class index_gt {
             return false;
 
         // Optional prefetching
-        if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
+        if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value)
             prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
 
         distance_t radius = context.measure(query, citerator_at(start_slot), metric);
@@ -3473,7 +3475,7 @@ class index_gt {
             neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
 
             // Optional prefetching
-            if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
+            if (!std::is_same<typename std::decay<prefetch_at>::type, dummy_prefetch_t>::value) {
                 candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
                 prefetch(missing_candidates.begin(), missing_candidates.end());
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "2.1.1",
+    "version": "2.1.2",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ash Vardanian",
     "license": "Apache 2.0",

package/python/usearch/index.py

@@ -562,12 +562,11 @@ class Index:
         )
 
         self.path = path
-        if path and os.path.exists(path):
-            path = os.fspath(path)
+        if path is not None:
             if view:
-                self._compiled.view(path)
+                self.view(path)
             else:
-                self._compiled.load(path)
+                self.load(path)
 
     @staticmethod
     def metadata(path_or_buffer: PathOrBuffer) -> Optional[dict]:
@@ -919,7 +918,9 @@ class Index:
     def expansion_search(self, v: int):
         self._compiled.expansion_search = v
 
-    def save(self, path_or_buffer: Union[str, os.PathLike, NoneType] = None) -> Optional[bytes]:
+    def save(
+        self, path_or_buffer: Union[str, os.PathLike, NoneType] = None
+    ) -> Optional[bytes]:
         path_or_buffer = path_or_buffer if path_or_buffer else self.path
         if path_or_buffer is None:
             return self._compiled.save_index_to_buffer()
@@ -937,7 +938,9 @@ class Index:
         else:
             self._compiled.load_index_from_path(os.fspath(path_or_buffer))
 
-    def view(self, path_or_buffer: Union[str, os.PathLike, bytes, bytearray, NoneType] = None):
+    def view(
+        self, path_or_buffer: Union[str, os.PathLike, bytes, bytearray, NoneType] = None
+    ):
         path_or_buffer = path_or_buffer if path_or_buffer else self.path
         if path_or_buffer is None:
             raise Exception("Define the source")

package/setup.py

@@ -66,6 +66,7 @@ setup(
     package_dir={"usearch": "python/usearch"},
     description="Smaller & Faster Single-File Vector Search Engine from Unum",
     author="Ash Vardanian",
+    author_email="info@unum.cloud",
     long_description=long_description,
     long_description_content_type="text/markdown",
     license="Apache-2.0",

package/wasmer.toml

@@ -1,6 +1,6 @@
 [package]
 name="unum/usearch"
-version="2.1.1"
+version="2.1.2"
 description="Smaller & Faster Single-File Vector Search Engine from Unum"
 license="Apache-2.0"
 readme="README.md"