usearch 0.12.1 → 0.13.0

package/.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
     - run: npm install --save-dev @semantic-release/exec @semantic-release/git conventional-changelog-eslint semantic-release && npx semantic-release
 
   rebase:
-    name: Rebase from main into main-dev
+    name: Rebase Developmtn Branch
     needs: versioning
     runs-on: ubuntu-latest
     steps:
@@ -37,14 +37,14 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Perform Rebase
+      - name: Perform rebase
         run: |
           git checkout main-dev
           git rebase main
       - name: Push changes
         uses: ad-m/github-push-action@master
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
           branch: main-dev
 
   build_wheels:

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "0.12.1"
+version = "0.13.0"
 authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/README.md

@@ -30,22 +30,25 @@ Euclidean • Angular • Jaccard • Hamming • Haversine • User-Defined Met
 <a href="#golang">GoLang</a> •
 <a href="#wolfram">Wolfram</a>
 <br/>
-Linux • MacOS • Windows
+Linux • MacOS • Windows • Docker • WebAssembly 🔜
 </p>
 
 ---
 
 - [x] Industry-leading [performance](#performance).
 - [x] Easily-extendible [single C++11 header][usearch-header] implementation.
-- [x] [User-defined](#define-custom-metrics) and pre-packaged SIMD-accelerated metrics.
+- [x] SIMD-accelerated and [User-defined](#define-custom-metrics) metrics with JIT-compilation.
+- [x] Variable dimensionality vectors - for [obscure use-cases, like GIS and Chess][obscure-use-cases].
+- [x] Bitwise Tanimoto and Sorensen coefficients for Genomics and Chemistry.
 - [x] [Half-precision `f16` and Quarter-precision `f8`](#quantize-on-the-fly) support on any hardware.
 - [x] [View from disk](#view-larger-indexes-from-disk), without loading into RAM.
 - [x] [4B+](#go-beyond-4b-entries) sized space efficient point-clouds with `uint40_t`.
-- [x] Variable dimensionality vectors - for [obscure use-cases][obscure-use-cases].
-- [x] [Bring your threads](#bring-your-threads), like OpenMP.
+- [x] [Bring your threads](#bring-your-threads), like OpenMP or C++23 executors.
 - [x] Multiple vectors per label.
 - [ ] Thread-safe `reserve`.
-- [x] AI + Vector Search = [Semantic Search](#ai--vector-search--semantic-search).
+- [ ] On-the-fly deletions.
+- [x] USearch + UForm Transformers = [Semantic Search](#ai--vector-search--semantic-search).
+- [x] USearch + RDKit = [Molecular Search](#ai--vector-search--semantic-search).
 
 [usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/usearch.hpp
 [obscure-use-cases]: https://ashvardanian.com/posts/abusing-vector-search
@@ -500,25 +503,27 @@ func main() {
 - Rust: Allow passing a custom thread ID.
 - C# .NET bindings.
 
-## AI + Vector Search = Semantic Search
+## Application Examples
+
+### USearch + AI = Multi-Modal Semantic Search
 
 AI has a growing number of applications, but one of the coolest classic ideas is to use it for Semantic Search.
-One can take an encoder model, like the multi-modal UForm, and a web-programming framework, like UCall, and build an image search platform in just 20 lines of Python.
+One can take an encoder model, like the multi-modal UForm, and a web-programming framework, like UCall, and build a text-to-image search platform in just 20 lines of Python.
 
 ```python
-import ucall.rich_posix as ucall
+import ucall
 import uform
-from usearch.index import Index
+import usearch
 
 import numpy as np
-from PIL import Image
+import PIL as pil
 
 server = ucall.Server()
 model = uform.get_model('unum-cloud/uform-vl-multilingual')
-index = Index(ndim=256)
+index = usearch.index.Index(ndim=256)
 
 @server
-def add(label: int, photo: Image.Image):
+def add(label: int, photo: pil.Image.Image):
     image = model.preprocess_image(photo)
     vector = model.encode_image(image).detach().numpy()
     index.add(label, vector.flatten(), copy=True)
@@ -527,10 +532,43 @@ def add(label: int, photo: Image.Image):
 def search(query: str) -> np.ndarray:
     tokens = model.preprocess_text(query)
     vector = model.encode_text(tokens).detach().numpy()
-    neighbors, _, _ = index.search(vector.flatten(), 3)
-    return neighbors
+    matches = index.search(vector.flatten(), 3)
+    return matches.labels
 
 server.run()
 ```
 
+### USearch + RDKit = Molecular Search
+
+Research in natural sciences is expensive.
+To search for properties of different molecules, large databases are used.
+Comparing the molecules, however, is hard.
+Instead, binary hash-like fingerprints are produced from the graph structure, using packages like RDKit.
+
+```python
+from usearch.index import Index, MetricKind
+from rdkit import Chem
+from rdkit.Chem import AllChem
+
+import numpy as np
+
+molecules = [Chem.MolFromSmiles('CCOC'), Chem.MolFromSmiles('CCO')]
+encoder = AllChem.GetRDKitFPGenerator()
+
+fingerprints = np.vstack([encoder.GetFingerprint(x) for x in molecules])
+fingerprints = np.packbits(fingerprints, axis=1)
+
+index = Index(ndim=2048, metric=MetricKind.BitwiseTanimoto)
+labels = np.array(len(molecules), dtype=np.longlong)
+
+index.add(labels, fingerprints)
+matches = index.search(fingerprints, 10)
+```
+
+RDKit [provides][rdkit-fingerprints] Atom-Pair, Topological Torsion, Morgan, and Layered Fingerprints, among other options.
+
+[rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
+
+---
+
 Check [that](https://github.com/ashvardanian/image-search) and [other](https://github.com/unum-cloud/examples) examples on our corporate GitHub 🤗

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '0.12.1'
+    version = '0.13.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/docs/compilation.md

@@ -151,6 +151,13 @@ java -cp . -Djava.library.path=/Users/av/github/usearch/java cloud.unum.usearch.
 brew install --cask wolfram-engine
 ```
 
+## GoLang
+
+```sh
+cd golang
+go test -v
+```
+
 ## C
 
 ### Linux

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "0.12.1",
+    "version": "0.13.0",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ashot Vardanian",
     "license": "Apache 2.0",

package/pyproject.toml

@@ -46,4 +46,4 @@ before-build = [
     "git submodule update --init --recursive"
 ]
 archs = ["x86", "AMD64"]
-skip = ["*win32*"]
+skip = ["*win32*", "pp*"]

package/python/lib.cpp

@@ -717,7 +717,8 @@ PYBIND11_MODULE(compiled, m) {
     i.def_property_readonly("capacity", &punned_index_py_t::capacity);
     i.def_property_readonly( //
         "dtype", [](punned_index_py_t const& index) -> std::string { return accuracy_name(index.accuracy()); });
-    i.def_property_readonly("memory_usage", &punned_index_py_t::memory_usage);
+    i.def_property_readonly("memory_usage",
+                            [](punned_index_py_t const& index) -> std::size_t { return index.memory_usage(); });
 
     i.def_property("expansion_add", &get_expansion_add<punned_index_py_t>, &punned_index_py_t::change_expansion_add);
     i.def_property("expansion_search", &get_expansion_search<punned_index_py_t>,

package/python/scripts/bench.py

@@ -1,6 +1,7 @@
 import time
 import numpy as np
 import math
+from typing import Tuple, Any, Optional
 
 import fire
 
@@ -8,7 +9,7 @@ from usearch.index import Index
 from usearch.io import load_matrix
 
 
-def measure(f) -> float:
+def measure(f) -> Tuple[float, Any]:
     a = time.time_ns()
     result = f()
     b = time.time_ns()
@@ -18,7 +19,10 @@ def measure(f) -> float:
     return secs, result
 
 
-def bench_faiss(index, vectors: np.array, queries: np.array, neighbors: np.array):
+def bench_faiss(
+        index,
+        vectors: np.ndarray, queries: np.ndarray, neighbors: np.ndarray):
+
     dt, _ = measure(lambda: index.add(vectors))
     print(f'- Performance: {vectors.shape[0]/dt:.2f} insertions/s')
 
@@ -33,7 +37,10 @@ def bench_faiss(index, vectors: np.array, queries: np.array, neighbors: np.array
     print(f'- Recall@1: {recall_at_one * 100:.2f} %')
 
 
-def bench_usearch(index, vectors: np.array, queries: np.array, neighbors: np.array):
+def bench_usearch(
+        index,
+        vectors: np.ndarray, queries: np.ndarray, neighbors: np.ndarray):
+
     labels = np.arange(vectors.shape[0], dtype=np.longlong)
     assert len(index) == 0
     dt, _ = measure(lambda: index.add(labels, vectors))
@@ -58,12 +65,15 @@ def main(
     connectivity: int = 16,
     expansion_add: int = 128,
     expansion_search: int = 64,
+    k: Optional[int] = None,
 ):
 
     vectors_mat = load_matrix(vectors)
     queries_mat = load_matrix(queries)
     neighbors_mat = load_matrix(neighbors)
     dim = vectors_mat.shape[1]
+    if k:
+        neighbors_mat = neighbors_mat[:, :1]
 
     print('USearch: HNSW')
     index = Index(
@@ -142,7 +152,7 @@ def main(
             expansion_add=expansion_add,
             expansion_search=expansion_search,
             connectivity=connectivity,
-            metric=inner_product_f32.address,
+            metric=inner_product_f32,
         )
         bench_usearch(index, vectors_mat, queries_mat, neighbors_mat)
 
@@ -153,7 +163,7 @@ def main(
             expansion_add=expansion_add,
             expansion_search=expansion_search,
             connectivity=connectivity,
-            metric=cos_f8.address,
+            metric=cos_f8,
         )
         bench_usearch(index, vectors_mat, queries_mat, neighbors_mat)
     except ImportError:

package/python/scripts/bench_params.py

@@ -8,7 +8,6 @@ import pandas as pd
 
 
 from usearch.index import Index
-from usearch.io import load_matrix
 
 
 def measure(f) -> float:
@@ -40,6 +39,11 @@ class Main:
         expansion_add: int = 128,
         expansion_search: int = 64,
     ):
+        """Measures indexing speed for different dimensionality vectors.
+
+        :param count: Number of vectors, defaults to 1_000_000
+        :type count: int, optional
+        """
 
         dimensions = [
             2, 3, 4, 8, 16, 32, 96, 100,
@@ -78,12 +82,20 @@ class Main:
         expansion_add: int = 128,
         expansion_search: int = 64,
     ):
+        """Measures indexing speed and accuracy for different level of
+        connectivity in the levels of the hierarchical proximity graph.
+
+        :param count: Number of vectors, defaults to 1_000_000
+        :type count: int, optional
+        :param ndim: Number of dimensions per vector, defaults to 256
+        :type ndim: int, optional
+        """
 
-        connectivities = range(10, 20)
+        connectivity_options = range(10, 20)
         results = []
 
         try:
-            for connectivity in connectivities:
+            for connectivity in connectivity_options:
                 vectors_mat = np.random.rand(count, ndim).astype(np.float32)
 
                 print(f'USearch for {connectivity} connectivity')
@@ -107,6 +119,43 @@ class Main:
             __file__), 'bench_connectivity.png'))
         fig.show()
 
+    def robustness(
+        self,
+        experiments: int = 10,
+        count: int = 1_000_000,
+        ndim: int = 256,
+        connectivity: int = 16,
+        expansion_add: int = 128,
+        expansion_search: int = 64,
+    ):
+        """How much does accuracy fluctuate depending on the indexing order.
+
+        :param experiments: How many times to repeat, defaults to 10
+        :type experiments: int, optional
+        :param count: Number of vectors, defaults to 1_000_000
+        :type count: int, optional
+        :param ndim: Number of dimensions per vector, defaults to 256
+        :type ndim: int, optional
+        """
+
+        recall_levels = []
+        vectors_mat = np.random.rand(count, ndim).astype(np.float32)
+
+        for _ in range(experiments):
+
+            index = Index(
+                ndim=ndim,
+                expansion_add=expansion_add,
+                expansion_search=expansion_search,
+                connectivity=connectivity,
+            )
+            bench_usearch(index, vectors_mat)
+            del index
+
+        min_ = min(recall_levels)
+        max_ = max(recall_levels)
+        print(f'Recall@1 varies between {min_:.3f} and {max_:.3f}')
+
 
 if __name__ == '__main__':
     fire.Fire(Main)

package/python/usearch/index.py

@@ -31,18 +31,43 @@ SetsIndex = _CompiledSetsIndex
 class Matches(NamedTuple):
     labels: np.ndarray
     distances: np.ndarray
-    counts: np.ndarray
+    counts: Union[np.ndarray, int]
 
+    @property
+    def is_batch(self) -> bool:
+        return isinstance(self.counts, np.ndarray)
+
+    @property
+    def batch_size(self) -> int:
+        return len(self.counts) if isinstance(self.counts, np.ndarray) else 1
+
+    @property
+    def total_matches(self) -> int:
+        return np.sum(self.counts)
+
+    def to_list(self, row: Optional[int] = None) -> Union[List[dict], List[List[dict]]]:
 
-def list_matches(results: Matches, row: int) -> List[dict]:
+        if not self.is_batch:
+            assert row is None, 'Exporting a single sequence is only for batch requests'
+            labels = self.labels
+            distances = self.distances
 
-    count = results[2][row]
-    labels = results[0][row, :count]
-    distances = results[1][row, :count]
-    return [
-        {'label': int(label), 'distance': float(distance)}
-        for label, distance in zip(labels, distances)
-    ]
+        elif row is None:
+            return [self.to_list(i) for i in range(self.batch_size)]
+
+        else:
+            count = self.counts[row]
+            labels = self.labels[row, :count]
+            distances = self.distances[row, :count]
+
+        return [
+            {'label': int(label), 'distance': float(distance)}
+            for label, distance in zip(labels, distances)
+        ]
+
+    def __repr__(self) -> str:
+        return f'usearch.Matches({self.total_matches})' if self.is_batch else \
+            f'usearch.Matches({self.total_matches} across {self.batch_size} queries)'
 
 
 def jit_metric(ndim: int, metric: MetricKind, dtype: str = 'f32') -> Callable:
@@ -150,9 +175,9 @@ class Index:
         :param metric: Distance function, defaults to MetricKind.IP
         :type metric: Union[MetricKind, Callable], optional
             Kind of the distance function, or the Numba `cfunc` JIT-compiled object.
-            Possible `MetricKind` values: IP, Cosine, L2sq, Haversine, 
-            Hamming, Tanimoto, Sorensen.
-            Not every kind is JIT-able.
+            Possible `MetricKind` values: IP, Cos, L2sq, Haversine, Pearson,
+            BitwiseHamming, BitwiseTanimoto, BitwiseSorensen.
+            Not every kind is JIT-able. For Jaccard distance, use `SetsIndex`.
 
         :param dtype: Scalar type for internal vector storage, defaults to None
         :type dtype: str, optional
@@ -333,6 +358,10 @@ class Index:
     def ndim(self) -> int:
         return self._compiled.ndim
 
+    @property
+    def metric(self) -> MetricKind:
+        return self._metric_kind
+
     @property
     def dtype(self) -> str:
         return self._compiled.dtype

package/python/usearch/io.py

@@ -1,16 +1,21 @@
 import struct
+
 import numpy as np
 
 
-def load_matrix(filename: str, start_row: int = 0, count_rows: int = None, view: bool = False):
-    """
-    Read *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
-    Args:
-        :param filename (str): path to the matrix file
-        :param start_row (int): start reading vectors from this index
-        :param count_rows (int): number of vectors to read. If None, read all vectors
-    Returns:
-        Parsed matrix (numpy.ndarray)
+def load_matrix(
+        filename: str,
+        start_row: int = 0, count_rows: int = None,
+        view: bool = False) -> np.ndarray:
+    """Read *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
+
+    :param filename: path to the matrix file
+    :param start_row: start reading vectors from this index
+    :param count_rows: number of vectors to read. If None, read all vectors
+    :param view: set to `True` to memory-map the file instead of loading to RAM
+
+    :return: parsed matrix
+    :rtype: numpy.ndarray
     """
     dtype = np.float32
     scalar_size = 4
@@ -35,17 +40,21 @@ def load_matrix(filename: str, start_row: int = 0, count_rows: int = None, view:
         row_offset = start_row * scalar_size * cols
 
         if view:
-            return np.memmap(f, dtype=dtype, mode='r', offset=8+row_offset, shape=(rows, cols))
+            return np.memmap(
+                f, dtype=dtype, mode='r', offset=8+row_offset, shape=(rows, cols))
         else:
-            return np.fromfile(f, count=rows * cols, dtype=dtype, offset=row_offset).reshape(rows, cols)
+            return np.fromfile(
+                f, count=rows * cols,
+                dtype=dtype, offset=row_offset).reshape(rows, cols)
 
 
-def save_matrix(vecs: np.array, filename: str):
-    """
-    Write *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
-    Args:
-        :param vecs (numpy.array): the matrix to serialize
-        :param filename (str): path to the matrix file
+def save_matrix(vectors: np.ndarray, filename: str):
+    """Write *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
+
+    :param vectors: the matrix to serialize
+    :type vectors: numpy.ndarray
+    :param filename: path to the matrix file
+    :type filename: str
     """
     dtype = np.float32
     if filename.endswith('.fbin'):
@@ -59,9 +68,9 @@ def save_matrix(vecs: np.array, filename: str):
     else:
         raise Exception('Unknown file type')
 
-    assert len(vecs.shape) == 2, 'Input array must have 2 dimensions'
+    assert len(vectors.shape) == 2, 'Input array must have 2 dimensions'
     with open(filename, 'wb') as f:
-        nvecs, dim = vecs.shape
-        f.write(struct.pack('<i', nvecs))
+        count, dim = vectors.shape
+        f.write(struct.pack('<i', count))
         f.write(struct.pack('<i', dim))
-        vecs.astype(dtype).flatten().tofile(f)
+        vectors.astype(dtype).flatten().tofile(f)

package/python/usearch/synthetic.py

@@ -0,0 +1,32 @@
+import numpy as np
+
+from usearch.index import Index, Matches, MetricKind, BitwiseMetricKind
+
+
+def vectors(index: Index, count: int) -> np.ndarray:
+
+    if not isinstance(index, Index):
+        raise ValueError('Unsupported `index` type')
+
+    dimensions = index.ndim
+
+    if index.metric in BitwiseMetricKind:
+        bit_vectors = np.random.randint(2, size=(count, dimensions))
+        bit_vectors = np.packbits(bit_vectors, axis=1)
+        return bit_vectors
+
+    else:
+        x = np.random.rand(count, dimensions).astype(index.numpy_dtype)
+        if index.metric == MetricKind.IP:
+            return x / np.linalg.norm(x, axis=1, keepdims=True)
+        return x
+
+
+def recall_at_one(index: Index, vectors: np.ndarray) -> float:
+    empty_replica: Index = index.fork()
+
+    labels = np.arange(vectors.shape[0])
+    empty_replica.add(labels=labels, vectors=vectors)
+    matches: Matches = empty_replica.search(empty_replica, 1)
+
+    return np.sum(matches.labels.flatten() == labels) / len(labels)