usearch 0.12.2 → 0.14.0

package/.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
     - run: npm install --save-dev @semantic-release/exec @semantic-release/git conventional-changelog-eslint semantic-release && npx semantic-release
 
   rebase:
-    name: Rebase from main into main-dev
+    name: Rebase Development Branch
     needs: versioning
     runs-on: ubuntu-latest
     steps:
@@ -37,15 +37,18 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Perform Rebase
+      - name: Perform rebase
         run: |
           git checkout main-dev
           git rebase main
+          
       - name: Push changes
-        uses: ad-m/github-push-action@master
+        uses: CasperWA/push-protected@v2
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
           branch: main-dev
+          unprotect_reviews: True
+          force: True
 
   build_wheels:
     name: Build Wheels for ${{ matrix.os }}

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "0.12.2"
+version = "0.14.0"
 authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/README.md

@@ -30,22 +30,25 @@ Euclidean • Angular • Jaccard • Hamming • Haversine • User-Defined Met
 <a href="#golang">GoLang</a> •
 <a href="#wolfram">Wolfram</a>
 <br/>
-Linux • MacOS • Windows
+Linux • MacOS • Windows • Docker • WebAssembly 🔜
 </p>
 
 ---
 
 - [x] Industry-leading [performance](#performance).
 - [x] Easily-extendible [single C++11 header][usearch-header] implementation.
-- [x] [User-defined](#define-custom-metrics) and pre-packaged SIMD-accelerated metrics.
+- [x] SIMD-accelerated and [User-defined](#define-custom-metrics) metrics with JIT-compilation.
+- [x] Variable dimensionality vectors - for [obscure use-cases, like GIS and Chess][obscure-use-cases].
+- [x] Bitwise Tanimoto and Sorensen coefficients for Genomics and Chemistry.
 - [x] [Half-precision `f16` and Quarter-precision `f8`](#quantize-on-the-fly) support on any hardware.
 - [x] [View from disk](#view-larger-indexes-from-disk), without loading into RAM.
 - [x] [4B+](#go-beyond-4b-entries) sized space efficient point-clouds with `uint40_t`.
-- [x] Variable dimensionality vectors - for [obscure use-cases][obscure-use-cases].
-- [x] [Bring your threads](#bring-your-threads), like OpenMP.
+- [x] [Bring your threads](#bring-your-threads), like OpenMP or C++23 executors.
 - [x] Multiple vectors per label.
 - [ ] Thread-safe `reserve`.
-- [x] AI + Vector Search = [Semantic Search](#ai--vector-search--semantic-search).
+- [ ] On-the-fly deletions.
+- [x] USearch + UForm Transformers = [Semantic Search](#ai--vector-search--semantic-search).
+- [x] USearch + RDKit = [Molecular Search](#ai--vector-search--semantic-search).
 
 [usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/usearch.hpp
 [obscure-use-cases]: https://ashvardanian.com/posts/abusing-vector-search
@@ -500,25 +503,27 @@ func main() {
 - Rust: Allow passing a custom thread ID.
 - C# .NET bindings.
 
-## AI + Vector Search = Semantic Search
+## Application Examples
+
+### USearch + AI = Multi-Modal Semantic Search
 
 AI has a growing number of applications, but one of the coolest classic ideas is to use it for Semantic Search.
-One can take an encoder model, like the multi-modal UForm, and a web-programming framework, like UCall, and build an image search platform in just 20 lines of Python.
+One can take an encoder model, like the multi-modal UForm, and a web-programming framework, like UCall, and build a text-to-image search platform in just 20 lines of Python.
 
 ```python
-import ucall.rich_posix as ucall
+import ucall
 import uform
-from usearch.index import Index
+import usearch
 
 import numpy as np
-from PIL import Image
+import PIL as pil
 
 server = ucall.Server()
 model = uform.get_model('unum-cloud/uform-vl-multilingual')
-index = Index(ndim=256)
+index = usearch.index.Index(ndim=256)
 
 @server
-def add(label: int, photo: Image.Image):
+def add(label: int, photo: pil.Image.Image):
     image = model.preprocess_image(photo)
     vector = model.encode_image(image).detach().numpy()
     index.add(label, vector.flatten(), copy=True)
@@ -527,10 +532,43 @@ def add(label: int, photo: Image.Image):
 def search(query: str) -> np.ndarray:
     tokens = model.preprocess_text(query)
     vector = model.encode_text(tokens).detach().numpy()
-    neighbors, _, _ = index.search(vector.flatten(), 3)
-    return neighbors
+    matches = index.search(vector.flatten(), 3)
+    return matches.labels
 
 server.run()
 ```
 
+### USearch + RDKit = Molecular Search
+
+Research in natural sciences is expensive.
+To search for properties of different molecules, large databases are used.
+Comparing the molecules, however, is hard.
+Instead, binary hash-like fingerprints are produced from the graph structure, using packages like RDKit.
+
+```python
+from usearch.index import Index, MetricKind
+from rdkit import Chem
+from rdkit.Chem import AllChem
+
+import numpy as np
+
+molecules = [Chem.MolFromSmiles('CCOC'), Chem.MolFromSmiles('CCO')]
+encoder = AllChem.GetRDKitFPGenerator()
+
+fingerprints = np.vstack([encoder.GetFingerprint(x) for x in molecules])
+fingerprints = np.packbits(fingerprints, axis=1)
+
+index = Index(ndim=2048, metric=MetricKind.BitwiseTanimoto)
+labels = np.array(len(molecules), dtype=np.longlong)
+
+index.add(labels, fingerprints)
+matches = index.search(fingerprints, 10)
+```
+
+RDKit [provides][rdkit-fingerprints] Atom-Pair, Topological Torsion, Morgan, and Layered Fingerprints, among other options.
+
+[rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
+
+---
+
 Check [that](https://github.com/ashvardanian/image-search) and [other](https://github.com/unum-cloud/examples) examples on our corporate GitHub 🤗

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '0.12.2'
+    version = '0.14.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/docs/compilation.md

@@ -94,6 +94,7 @@ Checking the effect of different embedding dimensions on construction speed:
 ```sh
 python python/scripts/bench_params.py dimensions
 python python/scripts/bench_params.py connectivity
+python python/scripts/bench_params.py robustness
 ```
 
 ## JavaScript
@@ -151,6 +152,13 @@ java -cp . -Djava.library.path=/Users/av/github/usearch/java cloud.unum.usearch.
 brew install --cask wolfram-engine
 ```
 
+## GoLang
+
+```sh
+cd golang
+go test -v
+```
+
 ## C
 
 ### Linux

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "0.12.2",
+    "version": "0.14.0",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ashot Vardanian",
     "license": "Apache 2.0",

package/python/lib.cpp

@@ -347,8 +347,8 @@ static py::tuple search_one_in_index(punned_index_py_t& index, py::buffer vector
     py::array_t<label_t> labels_py(static_cast<Py_ssize_t>(wanted));
     py::array_t<distance_t> distances_py(static_cast<Py_ssize_t>(wanted));
     std::size_t count{};
-    auto labels_py1d = labels_py.mutable_unchecked<1>();
-    auto distances_py1d = distances_py.mutable_unchecked<1>();
+    auto labels_py1d = labels_py.template mutable_unchecked<1>();
+    auto distances_py1d = distances_py.template mutable_unchecked<1>();
 
     search_config_t config;
     config.exact = exact;
@@ -416,9 +416,9 @@ static py::tuple search_many_in_index( //
     py::array_t<label_t> labels_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<distance_t> distances_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<Py_ssize_t> counts_py(vectors_count);
-    auto labels_py2d = labels_py.mutable_unchecked<2>();
-    auto distances_py2d = distances_py.mutable_unchecked<2>();
-    auto counts_py1d = counts_py.mutable_unchecked<1>();
+    auto labels_py2d = labels_py.template mutable_unchecked<2>();
+    auto distances_py2d = distances_py.template mutable_unchecked<2>();
+    auto counts_py1d = counts_py.template mutable_unchecked<1>();
 
     // https://docs.python.org/3/library/struct.html#format-characters
     if (vectors_info.format == "c" || vectors_info.format == "b")
@@ -508,9 +508,9 @@ static py::tuple search_many_in_bits_index( //
     py::array_t<label_t> labels_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<distance_t> distances_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<Py_ssize_t> counts_py(vectors_count);
-    auto labels_py2d = labels_py.mutable_unchecked<2>();
-    auto distances_py2d = distances_py.mutable_unchecked<2>();
-    auto counts_py1d = counts_py.mutable_unchecked<1>();
+    auto labels_py2d = labels_py.template mutable_unchecked<2>();
+    auto distances_py2d = distances_py.template mutable_unchecked<2>();
+    auto counts_py1d = counts_py.template mutable_unchecked<1>();
 
     // https://docs.python.org/3/library/struct.html#format-characters
     if (vectors_info.format != "B")
@@ -548,6 +548,36 @@ template <typename index_at> std::size_t get_expansion_add(index_at const &index
 template <typename index_at> std::size_t get_expansion_search(index_at const &index) { return index.config().expansion_search; }
 // clang-format on
 
+template <typename scalar_at, typename index_at>
+py::array_t<scalar_at> get_typed_member(index_at const& index, label_t label) {
+    py::array_t<scalar_at> result_py(static_cast<Py_ssize_t>(index.dimensions()));
+    auto result_py1d = result_py.template mutable_unchecked<1>();
+    index.reconstruct(label, &result_py1d(0));
+    return result_py;
+}
+
+template <typename index_at>
+py::array get_member(index_at const& index, label_t label, common_scalar_kind_t scalar_kind) {
+    if (scalar_kind == common_scalar_kind_t::f32_k)
+        return get_typed_member<f32_t>(index, label);
+    else if (scalar_kind == common_scalar_kind_t::f64_k)
+        return get_typed_member<f64_t>(index, label);
+    else if (scalar_kind == common_scalar_kind_t::f16_k)
+        return get_typed_member<f16_bits_t>(index, label);
+    else if (scalar_kind == common_scalar_kind_t::f8_k)
+        return get_typed_member<f8_bits_t>(index, label);
+    else
+        throw std::invalid_argument("Incompatible scalars in the query matrix!");
+}
+
+template <typename index_at> py::array_t<label_t> get_labels(index_at const& index) {
+    std::size_t result_length = index.size();
+    py::array_t<label_t> result_py(static_cast<Py_ssize_t>(result_length));
+    auto result_py1d = result_py.template mutable_unchecked<1>();
+    index.export_labels(&result_py1d(0), result_length);
+    return result_py;
+}
+
 template <typename element_at> bool has_duplicates(element_at const* begin, element_at const* end) {
     if (begin == end)
         return false;
@@ -611,7 +641,7 @@ py::array_t<bits_numpy_word_t> hash_typed_buffer(py::buffer_info const& info, st
         Py_ssize_t rows_stride = info.strides[1];
         auto hashes_shape = py_shape_t{rows_count, words_per_row};
         auto hashes_py = py::array_t<bits_numpy_word_t>(hashes_shape);
-        auto hashes_proxy = hashes_py.mutable_unchecked<2>();
+        auto hashes_proxy = hashes_py.template mutable_unchecked<2>();
 
         for (Py_ssize_t i = 0; i != rows_count; ++i)
             hash_typed_row(                                                                                 //
@@ -622,7 +652,7 @@ py::array_t<bits_numpy_word_t> hash_typed_buffer(py::buffer_info const& info, st
     } else {
         auto hashes_shape = py_shape_t{words_per_row};
         auto hashes_py = py::array_t<bits_numpy_word_t>(hashes_shape);
-        auto hashes_proxy = hashes_py.mutable_unchecked<1>();
+        auto hashes_proxy = hashes_py.template mutable_unchecked<1>();
 
         hash_typed_row(                                                               //
             data, hashes_info.strides[0], static_cast<std::size_t>(elements_per_row), //
@@ -658,6 +688,18 @@ PYBIND11_MODULE(compiled, m) {
     m.attr("DEFAULT_EXPANSION_ADD") = py::int_(default_expansion_add());
     m.attr("DEFAULT_EXPANSION_SEARCH") = py::int_(default_expansion_search());
 
+#if defined(USEARCH_USE_OPENMP)
+    m.attr("USES_OPENMP") = py::int_(1);
+#else
+    m.attr("USES_OPENMP") = py::int_(0);
+#endif
+
+#if defined(USEARCH_USE_SIMSIMD)
+    m.attr("USES_SIMSIMD") = py::int_(1);
+#else
+    m.attr("USES_SIMSIMD") = py::int_(0);
+#endif
+
     py::enum_<common_metric_kind_t>(m, "MetricKind")
         .value("Unknown", common_metric_kind_t::unknown_k)
         .value("IP", common_metric_kind_t::ip_k)
@@ -670,6 +712,13 @@ PYBIND11_MODULE(compiled, m) {
         .value("BitwiseTanimoto", common_metric_kind_t::bitwise_tanimoto_k)
         .value("BitwiseSorensen", common_metric_kind_t::bitwise_sorensen_k);
 
+    py::enum_<common_scalar_kind_t>(m, "ScalarKind")
+        .value("F64", common_scalar_kind_t::f64_k)
+        .value("F32", common_scalar_kind_t::f32_k)
+        .value("F16", common_scalar_kind_t::f16_k)
+        .value("F8", common_scalar_kind_t::f8_k)
+        .value("B1", common_scalar_kind_t::b1_k);
+
     auto i = py::class_<punned_index_py_t>(m, "Index");
 
     i.def(py::init(&make_index),                                    //
@@ -717,12 +766,19 @@ PYBIND11_MODULE(compiled, m) {
     i.def_property_readonly("capacity", &punned_index_py_t::capacity);
     i.def_property_readonly( //
         "dtype", [](punned_index_py_t const& index) -> std::string { return accuracy_name(index.accuracy()); });
-    i.def_property_readonly("memory_usage", &punned_index_py_t::memory_usage);
+    i.def_property_readonly( //
+        "memory_usage", [](punned_index_py_t const& index) -> std::size_t { return index.memory_usage(); });
 
     i.def_property("expansion_add", &get_expansion_add<punned_index_py_t>, &punned_index_py_t::change_expansion_add);
     i.def_property("expansion_search", &get_expansion_search<punned_index_py_t>,
                    &punned_index_py_t::change_expansion_search);
 
+    i.def_property_readonly("labels", &get_labels<punned_index_py_t>);
+    i.def("__contains__", &punned_index_py_t::contains);
+    i.def( //
+        "__getitem__", &get_member<punned_index_py_t>, py::arg("label"),
+        py::arg("dtype") = common_scalar_kind_t::f32_k);
+
     i.def("save", &save_index<punned_index_py_t>, py::arg("path"));
     i.def("load", &load_index<punned_index_py_t>, py::arg("path"));
     i.def("view", &view_index<punned_index_py_t>, py::arg("path"));
@@ -804,7 +860,7 @@ PYBIND11_MODULE(compiled, m) {
             auto proxy = set.unchecked<1>();
             auto view = set_view_t{&proxy(0), static_cast<std::size_t>(proxy.shape(0))};
             auto labels_py = py::array_t<label_t>(py_shape_t{static_cast<Py_ssize_t>(count)});
-            auto labels_proxy = labels_py.mutable_unchecked<1>();
+            auto labels_proxy = labels_py.template mutable_unchecked<1>();
             auto result = index.search(view, count);
             result.error.raise();
             auto found = result.dump_to(&labels_proxy(0));

package/python/scripts/bench.py

@@ -1,14 +1,20 @@
 import time
 import numpy as np
 import math
+from typing import Tuple, Any, Optional
 
 import fire
 
 from usearch.index import Index
 from usearch.io import load_matrix
+from usearch.index import (
+    DEFAULT_CONNECTIVITY,
+    DEFAULT_EXPANSION_ADD,
+    DEFAULT_EXPANSION_SEARCH,
+)
 
 
-def measure(f) -> float:
+def measure(f) -> Tuple[float, Any]:
     a = time.time_ns()
     result = f()
     b = time.time_ns()
@@ -18,7 +24,10 @@ def measure(f) -> float:
     return secs, result
 
 
-def bench_faiss(index, vectors: np.array, queries: np.array, neighbors: np.array):
+def bench_faiss(
+        index,
+        vectors: np.ndarray, queries: np.ndarray, neighbors: np.ndarray):
+
     dt, _ = measure(lambda: index.add(vectors))
     print(f'- Performance: {vectors.shape[0]/dt:.2f} insertions/s')
 
@@ -33,7 +42,10 @@ def bench_faiss(index, vectors: np.array, queries: np.array, neighbors: np.array
     print(f'- Recall@1: {recall_at_one * 100:.2f} %')
 
 
-def bench_usearch(index, vectors: np.array, queries: np.array, neighbors: np.array):
+def bench_usearch(
+        index,
+        vectors: np.ndarray, queries: np.ndarray, neighbors: np.ndarray):
+
     labels = np.arange(vectors.shape[0], dtype=np.longlong)
     assert len(index) == 0
     dt, _ = measure(lambda: index.add(labels, vectors))
@@ -55,15 +67,18 @@ def main(
     vectors: str,
     queries: str,
     neighbors: str,
-    connectivity: int = 16,
-    expansion_add: int = 128,
-    expansion_search: int = 64,
+    connectivity: int = DEFAULT_CONNECTIVITY,
+    expansion_add: int = DEFAULT_EXPANSION_ADD,
+    expansion_search: int = DEFAULT_EXPANSION_SEARCH,
+    k: Optional[int] = None,
 ):
 
     vectors_mat = load_matrix(vectors)
     queries_mat = load_matrix(queries)
     neighbors_mat = load_matrix(neighbors)
     dim = vectors_mat.shape[1]
+    if k:
+        neighbors_mat = neighbors_mat[:, :1]
 
     print('USearch: HNSW')
     index = Index(
@@ -142,7 +157,7 @@ def main(
             expansion_add=expansion_add,
             expansion_search=expansion_search,
             connectivity=connectivity,
-            metric=inner_product_f32.address,
+            metric=inner_product_f32,
         )
         bench_usearch(index, vectors_mat, queries_mat, neighbors_mat)
 
@@ -153,7 +168,7 @@ def main(
             expansion_add=expansion_add,
             expansion_search=expansion_search,
             connectivity=connectivity,
-            metric=cos_f8.address,
+            metric=cos_f8,
         )
         bench_usearch(index, vectors_mat, queries_mat, neighbors_mat)
     except ImportError:

package/python/scripts/bench_params.py

@@ -6,9 +6,15 @@ import plotly.express as px
 import numpy as np
 import pandas as pd
 
-
 from usearch.index import Index
-from usearch.io import load_matrix
+from usearch.index import (
+    DEFAULT_CONNECTIVITY,
+    DEFAULT_EXPANSION_ADD,
+    DEFAULT_EXPANSION_SEARCH,
+
+    USES_OPENMP,
+    USES_SIMSIMD,
+)
 
 
 def measure(f) -> float:
@@ -31,15 +37,29 @@ def bench_usearch(index, vectors: np.ndarray) -> float:
     return insertions_per_second
 
 
+def recall_at_one(index: Index, vectors: np.ndarray) -> float:
+
+    labels = np.arange(vectors.shape[0])
+    index.add(labels=labels, vectors=vectors)
+    matches = index.search(vectors, 1)
+
+    return np.sum(matches.labels.flatten() == labels) / len(labels)
+
+
 class Main:
 
     def dimensions(
         self,
         count: int = 1_000_000,
-        connectivity: int = 16,
-        expansion_add: int = 128,
-        expansion_search: int = 64,
+        connectivity: int = DEFAULT_CONNECTIVITY,
+        expansion_add: int = DEFAULT_EXPANSION_ADD,
+        expansion_search: int = DEFAULT_EXPANSION_SEARCH,
     ):
+        """Measures indexing speed for different dimensionality vectors.
+
+        :param count: Number of vectors, defaults to 1_000_000
+        :type count: int, optional
+        """
 
         dimensions = [
             2, 3, 4, 8, 16, 32, 96, 100,
@@ -75,15 +95,23 @@ class Main:
         self,
         count: int = 1_000_000,
         ndim: int = 256,
-        expansion_add: int = 128,
-        expansion_search: int = 64,
+        expansion_add: int = DEFAULT_EXPANSION_ADD,
+        expansion_search: int = DEFAULT_EXPANSION_SEARCH,
     ):
+        """Measures indexing speed and accuracy for different level of
+        connectivity in the levels of the hierarchical proximity graph.
 
-        connectivities = range(10, 20)
+        :param count: Number of vectors, defaults to 1_000_000
+        :type count: int, optional
+        :param ndim: Number of dimensions per vector, defaults to 256
+        :type ndim: int, optional
+        """
+
+        connectivity_options = range(10, 20)
         results = []
 
         try:
-            for connectivity in connectivities:
+            for connectivity in connectivity_options:
                 vectors_mat = np.random.rand(count, ndim).astype(np.float32)
 
                 print(f'USearch for {connectivity} connectivity')
@@ -107,6 +135,67 @@ class Main:
             __file__), 'bench_connectivity.png'))
         fig.show()
 
+    def robustness(
+        self,
+        experiments: int = 1,
+        count: int = 10000,
+        ndim: int = 256,
+        connectivity: int = DEFAULT_CONNECTIVITY,
+        expansion_add: int = DEFAULT_EXPANSION_ADD,
+        expansion_search: int = DEFAULT_EXPANSION_SEARCH,
+    ):
+        """How much does accuracy and speed fluctuate depending on the indexing order.
+
+        :param experiments: How many times to repeat, defaults to 10
+        :type experiments: int, optional
+        :param count: Number of vectors, defaults to 1_000_000
+        :type count: int, optional
+        :param ndim: Number of dimensions per vector, defaults to 256
+        :type ndim: int, optional
+        """
+
+        print(f'Connectivity: {connectivity}')
+        print(f'Number of vectors: {count}')
+        print(f'Vector dimensions: {ndim}')
+        print(f'Expansion on Add: {expansion_add}')
+        print(f'Expansion on Search: {expansion_search}')
+        print(f'Uses OpenMP: {USES_OPENMP}')
+        print(f'Uses SimSIMD: {USES_SIMSIMD}')
+
+        recall_levels = []
+        vectors_mat = np.random.rand(count, ndim).astype(np.float32)
+
+        for _ in range(experiments):
+            np.random.shuffle(vectors_mat)
+            index = Index(
+                ndim=ndim,
+                expansion_add=expansion_add,
+                expansion_search=expansion_search,
+                connectivity=connectivity,
+            )
+            bench_usearch(index, vectors_mat)
+            recall = recall_at_one(index, vectors_mat) * 100
+            print(f'- Recall @ 1 {recall:.2f} %')
+            recall_levels.append(recall)
+            del index
+
+        min_ = min(recall_levels) * 100
+        max_ = max(recall_levels) * 100
+        print(f'Recall @ 1 varies between {min_:.3f} and {max_:.3f}')
+
+        # Now let's try clustering and inserting in clusters
+        try:
+            from sklearn.cluster import KMeans
+        except ImportError:
+            return
+
+        count_clusters = count / expansion_add
+        clustering = KMeans(
+            n_clusters=count_clusters,
+            random_state=0, n_init='auto').fit(vectors_mat)
+        NamedTuple()
+
 
 if __name__ == '__main__':
+    Main().robustness()
     fire.Fire(Main)

package/python/scripts/test.py

@@ -3,14 +3,21 @@ import pytest
 import numpy as np
 
 from usearch.io import load_matrix, save_matrix
-from usearch.index import Index, SetsIndex, MetricKind
+from usearch.synthetic import recall_members
+
+from usearch.index import Index, SetsIndex, MetricKind, Matches
+from usearch.index import (
+    DEFAULT_CONNECTIVITY,
+    DEFAULT_EXPANSION_ADD,
+    DEFAULT_EXPANSION_SEARCH,
+)
 
 
 dimensions = [13, 99, 100, 256]
 batch_sizes = [1, 2, 100]
 index_types = ['f32', 'f64', 'f16', 'f8']
 numpy_types = [np.float32, np.float64, np.float16, np.byte]
-connectivity_options = [3, 13, 50]
+connectivity_options = [3, 13, 50, DEFAULT_CONNECTIVITY]
 jit_options = [False]
 continuous_metrics = [
     MetricKind.Cos,
@@ -26,6 +33,7 @@ hash_metrics = [
 @pytest.mark.parametrize('rows', batch_sizes)
 @pytest.mark.parametrize('cols', dimensions)
 def test_serializing_fbin_matrix(rows: int, cols: int):
+
     original = np.random.rand(rows, cols).astype(np.float32)
     save_matrix(original, 'tmp.fbin')
     reconstructed = load_matrix('tmp.fbin')
@@ -36,6 +44,7 @@ def test_serializing_fbin_matrix(rows: int, cols: int):
 @pytest.mark.parametrize('rows', batch_sizes)
 @pytest.mark.parametrize('cols', dimensions)
 def test_serializing_ibin_matrix(rows: int, cols: int):
+
     original = np.random.randint(0, rows+1, size=(rows, cols)).astype(np.int32)
     save_matrix(original, 'tmp.ibin')
     reconstructed = load_matrix('tmp.ibin')
@@ -59,6 +68,8 @@ def test_index(
         ndim=ndim,
         dtype=index_type,
         connectivity=connectivity,
+        expansion_add=DEFAULT_EXPANSION_ADD,
+        expansion_search=DEFAULT_EXPANSION_SEARCH,
         jit=jit,
     )
     assert index.ndim == ndim
@@ -76,6 +87,11 @@ def test_index(
     assert matches[0] == 42
     assert distances[0] == pytest.approx(0, abs=1e-3)
 
+    assert 42 in index
+    assert 42 in index.labels
+    if numpy_type != np.byte:
+        assert np.allclose(index[42], vector, atol=0.1)
+
     index.save('tmp.usearch')
     index.clear()
     assert len(index) == 0
@@ -101,10 +117,13 @@ def test_index_batch(
     labels = np.array(range(batch_size), dtype=np.longlong)
 
     index.add(labels, vectors)
-    matches, distances, count = index.search(vectors, 10)
+    matches: Matches = index.search(vectors, 10)
 
-    assert matches.shape[0] == distances.shape[0]
-    assert count.shape[0] == batch_size
+    assert matches.labels.shape[0] == matches.distances.shape[0]
+    assert matches.counts.shape[0] == batch_size
+    assert np.all(np.sort(index.labels) == np.sort(labels))
+
+    assert recall_members(index, exact=True) == 1
 
 
 @pytest.mark.parametrize('ndim', dimensions)

package/python/usearch/index.py

@@ -17,6 +17,9 @@ from usearch.compiled import (
     DEFAULT_CONNECTIVITY,
     DEFAULT_EXPANSION_ADD,
     DEFAULT_EXPANSION_SEARCH,
+
+    USES_OPENMP,
+    USES_SIMSIMD,
 )
 
 BitwiseMetricKind = (
@@ -31,18 +34,47 @@ SetsIndex = _CompiledSetsIndex
 class Matches(NamedTuple):
     labels: np.ndarray
     distances: np.ndarray
-    counts: np.ndarray
+    counts: Union[np.ndarray, int]
+
+    @property
+    def is_batch(self) -> bool:
+        return isinstance(self.counts, np.ndarray)
+
+    @property
+    def batch_size(self) -> int:
+        return len(self.counts) if isinstance(self.counts, np.ndarray) else 1
+
+    @property
+    def total_matches(self) -> int:
+        return np.sum(self.counts)
+
+    def to_list(self, row: Optional[int] = None) -> Union[List[dict], List[List[dict]]]:
+
+        if not self.is_batch:
+            assert row is None, 'Exporting a single sequence is only for batch requests'
+            labels = self.labels
+            distances = self.distances
 
+        elif row is None:
+            return [self.to_list(i) for i in range(self.batch_size)]
 
-def list_matches(results: Matches, row: int) -> List[dict]:
+        else:
+            count = self.counts[row]
+            labels = self.labels[row, :count]
+            distances = self.distances[row, :count]
+
+        return [
+            {'label': int(label), 'distance': float(distance)}
+            for label, distance in zip(labels, distances)
+        ]
+
+    def recall_first(self, expected: np.ndarray) -> float:
+        best_matches = self.labels if self.is_batch else self.labels[:, 0]
+        return np.sum(best_matches == expected) / len(expected)
 
-    count = results[2][row]
-    labels = results[0][row, :count]
-    distances = results[1][row, :count]
-    return [
-        {'label': int(label), 'distance': float(distance)}
-        for label, distance in zip(labels, distances)
-    ]
+    def __repr__(self) -> str:
+        return f'usearch.Matches({self.total_matches})' if self.is_batch else \
+            f'usearch.Matches({self.total_matches} across {self.batch_size} queries)'
 
 
 def jit_metric(ndim: int, metric: MetricKind, dtype: str = 'f32') -> Callable:
@@ -150,9 +182,9 @@ class Index:
         :param metric: Distance function, defaults to MetricKind.IP
         :type metric: Union[MetricKind, Callable], optional
             Kind of the distance function, or the Numba `cfunc` JIT-compiled object.
-            Possible `MetricKind` values: IP, Cosine, L2sq, Haversine, 
-            Hamming, Tanimoto, Sorensen.
-            Not every kind is JIT-able.
+            Possible `MetricKind` values: IP, Cos, L2sq, Haversine, Pearson,
+            BitwiseHamming, BitwiseTanimoto, BitwiseSorensen.
+            Not every kind is JIT-able. For Jaccard distance, use `SetsIndex`.
 
         :param dtype: Scalar type for internal vector storage, defaults to None
         :type dtype: str, optional
@@ -319,7 +351,7 @@ class Index:
         return Matches(*tuple_)
 
     def __len__(self) -> int:
-        return len(self._compiled)
+        return self._compiled.__len__()
 
     @property
     def jit(self) -> bool:
@@ -333,6 +365,10 @@ class Index:
     def ndim(self) -> int:
         return self._compiled.ndim
 
+    @property
+    def metric(self) -> MetricKind:
+        return self._metric_kind
+
     @property
     def dtype(self) -> str:
         return self._compiled.dtype
@@ -358,11 +394,11 @@ class Index:
         return self._compiled.expansion_search
 
     @expansion_add.setter
-    def change_expansion_add(self, v: int):
+    def expansion_add(self, v: int):
         self._compiled.expansion_add = v
 
     @expansion_search.setter
-    def change_expansion_search(self, v: int):
+    def expansion_search(self, v: int):
         self._compiled.expansion_search = v
 
     def save(self, path: os.PathLike):
@@ -377,5 +413,15 @@ class Index:
     def clear(self):
         self._compiled.clear()
 
-    def remove(self, label: int):
-        pass
+    @property
+    def labels(self) -> np.ndarray:
+        return self._compiled.labels
+
+    def __delitem__(self, label: int):
+        raise NotImplementedError()
+
+    def __contains__(self, label: int) -> bool:
+        return self._compiled.__contains__(label)
+
+    def __getitem__(self, label: int) -> np.ndarray:
+        return self._compiled.__getitem__(label)

package/python/usearch/io.py

@@ -1,16 +1,21 @@
 import struct
+
 import numpy as np
 
 
-def load_matrix(filename: str, start_row: int = 0, count_rows: int = None, view: bool = False):
-    """
-    Read *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
-    Args:
-        :param filename (str): path to the matrix file
-        :param start_row (int): start reading vectors from this index
-        :param count_rows (int): number of vectors to read. If None, read all vectors
-    Returns:
-        Parsed matrix (numpy.ndarray)
+def load_matrix(
+        filename: str,
+        start_row: int = 0, count_rows: int = None,
+        view: bool = False) -> np.ndarray:
+    """Read *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
+
+    :param filename: path to the matrix file
+    :param start_row: start reading vectors from this index
+    :param count_rows: number of vectors to read. If None, read all vectors
+    :param view: set to `True` to memory-map the file instead of loading to RAM
+
+    :return: parsed matrix
+    :rtype: numpy.ndarray
     """
     dtype = np.float32
     scalar_size = 4
@@ -35,17 +40,21 @@ def load_matrix(filename: str, start_row: int = 0, count_rows: int = None, view:
         row_offset = start_row * scalar_size * cols
 
         if view:
-            return np.memmap(f, dtype=dtype, mode='r', offset=8+row_offset, shape=(rows, cols))
+            return np.memmap(
+                f, dtype=dtype, mode='r', offset=8+row_offset, shape=(rows, cols))
         else:
-            return np.fromfile(f, count=rows * cols, dtype=dtype, offset=row_offset).reshape(rows, cols)
+            return np.fromfile(
+                f, count=rows * cols,
+                dtype=dtype, offset=row_offset).reshape(rows, cols)
 
 
-def save_matrix(vecs: np.array, filename: str):
-    """
-    Write *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
-    Args:
-        :param vecs (numpy.array): the matrix to serialize
-        :param filename (str): path to the matrix file
+def save_matrix(vectors: np.ndarray, filename: str):
+    """Write *.ibin, *.hbin, *.fbin, *.dbin files with matrixes.
+
+    :param vectors: the matrix to serialize
+    :type vectors: numpy.ndarray
+    :param filename: path to the matrix file
+    :type filename: str
     """
     dtype = np.float32
     if filename.endswith('.fbin'):
@@ -59,9 +68,9 @@ def save_matrix(vecs: np.array, filename: str):
     else:
         raise Exception('Unknown file type')
 
-    assert len(vecs.shape) == 2, 'Input array must have 2 dimensions'
+    assert len(vectors.shape) == 2, 'Input array must have 2 dimensions'
     with open(filename, 'wb') as f:
-        nvecs, dim = vecs.shape
-        f.write(struct.pack('<i', nvecs))
+        count, dim = vectors.shape
+        f.write(struct.pack('<i', count))
         f.write(struct.pack('<i', dim))
-        vecs.astype(dtype).flatten().tofile(f)
+        vectors.astype(dtype).flatten().tofile(f)

package/python/usearch/synthetic.py

@@ -0,0 +1,31 @@
+import numpy as np
+
+from usearch.index import Index, Matches, MetricKind, BitwiseMetricKind
+
+
+def vectors(index: Index, count: int) -> np.ndarray:
+
+    if not isinstance(index, Index):
+        raise ValueError('Unsupported `index` type')
+
+    dimensions = index.ndim
+
+    if index.metric in BitwiseMetricKind:
+        bit_vectors = np.random.randint(2, size=(count, dimensions))
+        bit_vectors = np.packbits(bit_vectors, axis=1)
+        return bit_vectors
+
+    else:
+        x = np.random.rand(count, dimensions).astype(index.numpy_dtype)
+        if index.metric == MetricKind.IP:
+            return x / np.linalg.norm(x, axis=1, keepdims=True)
+        return x
+
+
+def recall_members(index: Index, *args, **kwargs) -> float:
+    vectors: np.ndarray = np.vstack([
+        index[i] for i in range(len(index))
+    ])
+    labels: np.ndarray = np.arange(vectors.shape[0])
+    matches: Matches = index.search(vectors, 1, *args, **kwargs)
+    return matches.recall_first(labels)

package/src/punned.hpp

@@ -33,6 +33,14 @@
 namespace unum {
 namespace usearch {
 
+enum class common_scalar_kind_t {
+    f64_k,
+    f32_k,
+    f16_k,
+    f8_k,
+    b1_k,
+};
+
 using byte_t = char;
 using punned_distance_t = float;
 using punned_metric_t = punned_distance_t (*)(byte_t const*, byte_t const*, std::size_t, std::size_t);
@@ -527,6 +535,17 @@ class punned_gt {
         return typed_->memory_usage(allocator_entry_bytes);
     }
 
+    bool contains(label_t label) const {
+        shared_lock_t lock(lookup_table_mutex_);
+        return lookup_table_.contains(label);
+    }
+
+    void export_labels(label_t* labels, std::size_t limit) const {
+        shared_lock_t lock(lookup_table_mutex_);
+        for (auto it = lookup_table_.begin(); it != lookup_table_.end() && limit; ++it, ++labels, --limit)
+            *labels = it->first;
+    }
+
     // clang-format off
     add_result_t add(label_t label, f8_bits_t const* vector) { return add_(label, vector, casts_.from_f8); }
     add_result_t add(label_t label, f16_bits_t const* vector) { return add_(label, vector, casts_.from_f16); }