usearch 0.13.0 → 0.14.0

package/.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
     - run: npm install --save-dev @semantic-release/exec @semantic-release/git conventional-changelog-eslint semantic-release && npx semantic-release
 
   rebase:
-    name: Rebase Developmtn Branch
+    name: Rebase Development Branch
     needs: versioning
     runs-on: ubuntu-latest
     steps:
@@ -41,11 +41,14 @@ jobs:
         run: |
           git checkout main-dev
           git rebase main
+          
       - name: Push changes
-        uses: ad-m/github-push-action@master
+        uses: CasperWA/push-protected@v2
         with:
-          github_token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
+          token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
           branch: main-dev
+          unprotect_reviews: True
+          force: True
 
   build_wheels:
     name: Build Wheels for ${{ matrix.os }}

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "0.13.0"
+version = "0.14.0"
 authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '0.13.0'
+    version = '0.14.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/docs/compilation.md

@@ -94,6 +94,7 @@ Checking the effect of different embedding dimensions on construction speed:
 ```sh
 python python/scripts/bench_params.py dimensions
 python python/scripts/bench_params.py connectivity
+python python/scripts/bench_params.py robustness
 ```
 
 ## JavaScript

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "0.13.0",
+    "version": "0.14.0",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ashot Vardanian",
     "license": "Apache 2.0",

package/python/lib.cpp

@@ -347,8 +347,8 @@ static py::tuple search_one_in_index(punned_index_py_t& index, py::buffer vector
     py::array_t<label_t> labels_py(static_cast<Py_ssize_t>(wanted));
     py::array_t<distance_t> distances_py(static_cast<Py_ssize_t>(wanted));
     std::size_t count{};
-    auto labels_py1d = labels_py.mutable_unchecked<1>();
-    auto distances_py1d = distances_py.mutable_unchecked<1>();
+    auto labels_py1d = labels_py.template mutable_unchecked<1>();
+    auto distances_py1d = distances_py.template mutable_unchecked<1>();
 
     search_config_t config;
     config.exact = exact;
@@ -416,9 +416,9 @@ static py::tuple search_many_in_index( //
     py::array_t<label_t> labels_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<distance_t> distances_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<Py_ssize_t> counts_py(vectors_count);
-    auto labels_py2d = labels_py.mutable_unchecked<2>();
-    auto distances_py2d = distances_py.mutable_unchecked<2>();
-    auto counts_py1d = counts_py.mutable_unchecked<1>();
+    auto labels_py2d = labels_py.template mutable_unchecked<2>();
+    auto distances_py2d = distances_py.template mutable_unchecked<2>();
+    auto counts_py1d = counts_py.template mutable_unchecked<1>();
 
     // https://docs.python.org/3/library/struct.html#format-characters
     if (vectors_info.format == "c" || vectors_info.format == "b")
@@ -508,9 +508,9 @@ static py::tuple search_many_in_bits_index( //
     py::array_t<label_t> labels_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<distance_t> distances_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<Py_ssize_t> counts_py(vectors_count);
-    auto labels_py2d = labels_py.mutable_unchecked<2>();
-    auto distances_py2d = distances_py.mutable_unchecked<2>();
-    auto counts_py1d = counts_py.mutable_unchecked<1>();
+    auto labels_py2d = labels_py.template mutable_unchecked<2>();
+    auto distances_py2d = distances_py.template mutable_unchecked<2>();
+    auto counts_py1d = counts_py.template mutable_unchecked<1>();
 
     // https://docs.python.org/3/library/struct.html#format-characters
     if (vectors_info.format != "B")
@@ -548,6 +548,36 @@ template <typename index_at> std::size_t get_expansion_add(index_at const &index
 template <typename index_at> std::size_t get_expansion_search(index_at const &index) { return index.config().expansion_search; }
 // clang-format on
 
+template <typename scalar_at, typename index_at>
+py::array_t<scalar_at> get_typed_member(index_at const& index, label_t label) {
+    py::array_t<scalar_at> result_py(static_cast<Py_ssize_t>(index.dimensions()));
+    auto result_py1d = result_py.template mutable_unchecked<1>();
+    index.reconstruct(label, &result_py1d(0));
+    return result_py;
+}
+
+template <typename index_at>
+py::array get_member(index_at const& index, label_t label, common_scalar_kind_t scalar_kind) {
+    if (scalar_kind == common_scalar_kind_t::f32_k)
+        return get_typed_member<f32_t>(index, label);
+    else if (scalar_kind == common_scalar_kind_t::f64_k)
+        return get_typed_member<f64_t>(index, label);
+    else if (scalar_kind == common_scalar_kind_t::f16_k)
+        return get_typed_member<f16_bits_t>(index, label);
+    else if (scalar_kind == common_scalar_kind_t::f8_k)
+        return get_typed_member<f8_bits_t>(index, label);
+    else
+        throw std::invalid_argument("Incompatible scalars in the query matrix!");
+}
+
+template <typename index_at> py::array_t<label_t> get_labels(index_at const& index) {
+    std::size_t result_length = index.size();
+    py::array_t<label_t> result_py(static_cast<Py_ssize_t>(result_length));
+    auto result_py1d = result_py.template mutable_unchecked<1>();
+    index.export_labels(&result_py1d(0), result_length);
+    return result_py;
+}
+
 template <typename element_at> bool has_duplicates(element_at const* begin, element_at const* end) {
     if (begin == end)
         return false;
@@ -611,7 +641,7 @@ py::array_t<bits_numpy_word_t> hash_typed_buffer(py::buffer_info const& info, st
         Py_ssize_t rows_stride = info.strides[1];
         auto hashes_shape = py_shape_t{rows_count, words_per_row};
         auto hashes_py = py::array_t<bits_numpy_word_t>(hashes_shape);
-        auto hashes_proxy = hashes_py.mutable_unchecked<2>();
+        auto hashes_proxy = hashes_py.template mutable_unchecked<2>();
 
         for (Py_ssize_t i = 0; i != rows_count; ++i)
             hash_typed_row(                                                                                 //
@@ -622,7 +652,7 @@ py::array_t<bits_numpy_word_t> hash_typed_buffer(py::buffer_info const& info, st
     } else {
         auto hashes_shape = py_shape_t{words_per_row};
         auto hashes_py = py::array_t<bits_numpy_word_t>(hashes_shape);
-        auto hashes_proxy = hashes_py.mutable_unchecked<1>();
+        auto hashes_proxy = hashes_py.template mutable_unchecked<1>();
 
         hash_typed_row(                                                               //
             data, hashes_info.strides[0], static_cast<std::size_t>(elements_per_row), //
@@ -658,6 +688,18 @@ PYBIND11_MODULE(compiled, m) {
     m.attr("DEFAULT_EXPANSION_ADD") = py::int_(default_expansion_add());
     m.attr("DEFAULT_EXPANSION_SEARCH") = py::int_(default_expansion_search());
 
+#if defined(USEARCH_USE_OPENMP)
+    m.attr("USES_OPENMP") = py::int_(1);
+#else
+    m.attr("USES_OPENMP") = py::int_(0);
+#endif
+
+#if defined(USEARCH_USE_SIMSIMD)
+    m.attr("USES_SIMSIMD") = py::int_(1);
+#else
+    m.attr("USES_SIMSIMD") = py::int_(0);
+#endif
+
     py::enum_<common_metric_kind_t>(m, "MetricKind")
         .value("Unknown", common_metric_kind_t::unknown_k)
         .value("IP", common_metric_kind_t::ip_k)
@@ -670,6 +712,13 @@ PYBIND11_MODULE(compiled, m) {
         .value("BitwiseTanimoto", common_metric_kind_t::bitwise_tanimoto_k)
         .value("BitwiseSorensen", common_metric_kind_t::bitwise_sorensen_k);
 
+    py::enum_<common_scalar_kind_t>(m, "ScalarKind")
+        .value("F64", common_scalar_kind_t::f64_k)
+        .value("F32", common_scalar_kind_t::f32_k)
+        .value("F16", common_scalar_kind_t::f16_k)
+        .value("F8", common_scalar_kind_t::f8_k)
+        .value("B1", common_scalar_kind_t::b1_k);
+
     auto i = py::class_<punned_index_py_t>(m, "Index");
 
     i.def(py::init(&make_index),                                    //
@@ -717,13 +766,19 @@ PYBIND11_MODULE(compiled, m) {
     i.def_property_readonly("capacity", &punned_index_py_t::capacity);
     i.def_property_readonly( //
         "dtype", [](punned_index_py_t const& index) -> std::string { return accuracy_name(index.accuracy()); });
-    i.def_property_readonly("memory_usage",
-                            [](punned_index_py_t const& index) -> std::size_t { return index.memory_usage(); });
+    i.def_property_readonly( //
+        "memory_usage", [](punned_index_py_t const& index) -> std::size_t { return index.memory_usage(); });
 
     i.def_property("expansion_add", &get_expansion_add<punned_index_py_t>, &punned_index_py_t::change_expansion_add);
     i.def_property("expansion_search", &get_expansion_search<punned_index_py_t>,
                    &punned_index_py_t::change_expansion_search);
 
+    i.def_property_readonly("labels", &get_labels<punned_index_py_t>);
+    i.def("__contains__", &punned_index_py_t::contains);
+    i.def( //
+        "__getitem__", &get_member<punned_index_py_t>, py::arg("label"),
+        py::arg("dtype") = common_scalar_kind_t::f32_k);
+
     i.def("save", &save_index<punned_index_py_t>, py::arg("path"));
     i.def("load", &load_index<punned_index_py_t>, py::arg("path"));
     i.def("view", &view_index<punned_index_py_t>, py::arg("path"));
@@ -805,7 +860,7 @@ PYBIND11_MODULE(compiled, m) {
             auto proxy = set.unchecked<1>();
             auto view = set_view_t{&proxy(0), static_cast<std::size_t>(proxy.shape(0))};
             auto labels_py = py::array_t<label_t>(py_shape_t{static_cast<Py_ssize_t>(count)});
-            auto labels_proxy = labels_py.mutable_unchecked<1>();
+            auto labels_proxy = labels_py.template mutable_unchecked<1>();
             auto result = index.search(view, count);
             result.error.raise();
             auto found = result.dump_to(&labels_proxy(0));

package/python/scripts/bench.py

@@ -7,6 +7,11 @@ import fire
 
 from usearch.index import Index
 from usearch.io import load_matrix
+from usearch.index import (
+    DEFAULT_CONNECTIVITY,
+    DEFAULT_EXPANSION_ADD,
+    DEFAULT_EXPANSION_SEARCH,
+)
 
 
 def measure(f) -> Tuple[float, Any]:
@@ -62,9 +67,9 @@ def main(
     vectors: str,
     queries: str,
     neighbors: str,
-    connectivity: int = 16,
-    expansion_add: int = 128,
-    expansion_search: int = 64,
+    connectivity: int = DEFAULT_CONNECTIVITY,
+    expansion_add: int = DEFAULT_EXPANSION_ADD,
+    expansion_search: int = DEFAULT_EXPANSION_SEARCH,
     k: Optional[int] = None,
 ):
 

package/python/scripts/bench_params.py

@@ -6,8 +6,15 @@ import plotly.express as px
 import numpy as np
 import pandas as pd
 
-
 from usearch.index import Index
+from usearch.index import (
+    DEFAULT_CONNECTIVITY,
+    DEFAULT_EXPANSION_ADD,
+    DEFAULT_EXPANSION_SEARCH,
+
+    USES_OPENMP,
+    USES_SIMSIMD,
+)
 
 
 def measure(f) -> float:
@@ -30,14 +37,23 @@ def bench_usearch(index, vectors: np.ndarray) -> float:
     return insertions_per_second
 
 
+def recall_at_one(index: Index, vectors: np.ndarray) -> float:
+
+    labels = np.arange(vectors.shape[0])
+    index.add(labels=labels, vectors=vectors)
+    matches = index.search(vectors, 1)
+
+    return np.sum(matches.labels.flatten() == labels) / len(labels)
+
+
 class Main:
 
     def dimensions(
         self,
         count: int = 1_000_000,
-        connectivity: int = 16,
-        expansion_add: int = 128,
-        expansion_search: int = 64,
+        connectivity: int = DEFAULT_CONNECTIVITY,
+        expansion_add: int = DEFAULT_EXPANSION_ADD,
+        expansion_search: int = DEFAULT_EXPANSION_SEARCH,
     ):
         """Measures indexing speed for different dimensionality vectors.
 
@@ -79,8 +95,8 @@ class Main:
         self,
         count: int = 1_000_000,
         ndim: int = 256,
-        expansion_add: int = 128,
-        expansion_search: int = 64,
+        expansion_add: int = DEFAULT_EXPANSION_ADD,
+        expansion_search: int = DEFAULT_EXPANSION_SEARCH,
     ):
         """Measures indexing speed and accuracy for different level of
         connectivity in the levels of the hierarchical proximity graph.
@@ -121,14 +137,14 @@ class Main:
 
     def robustness(
         self,
-        experiments: int = 10,
-        count: int = 1_000_000,
+        experiments: int = 1,
+        count: int = 10000,
         ndim: int = 256,
-        connectivity: int = 16,
-        expansion_add: int = 128,
-        expansion_search: int = 64,
+        connectivity: int = DEFAULT_CONNECTIVITY,
+        expansion_add: int = DEFAULT_EXPANSION_ADD,
+        expansion_search: int = DEFAULT_EXPANSION_SEARCH,
     ):
-        """How much does accuracy fluctuate depending on the indexing order.
+        """How much does accuracy and speed fluctuate depending on the indexing order.
 
         :param experiments: How many times to repeat, defaults to 10
         :type experiments: int, optional
@@ -138,11 +154,19 @@ class Main:
         :type ndim: int, optional
         """
 
+        print(f'Connectivity: {connectivity}')
+        print(f'Number of vectors: {count}')
+        print(f'Vector dimensions: {ndim}')
+        print(f'Expansion on Add: {expansion_add}')
+        print(f'Expansion on Search: {expansion_search}')
+        print(f'Uses OpenMP: {USES_OPENMP}')
+        print(f'Uses SimSIMD: {USES_SIMSIMD}')
+
         recall_levels = []
         vectors_mat = np.random.rand(count, ndim).astype(np.float32)
 
         for _ in range(experiments):
-
+            np.random.shuffle(vectors_mat)
             index = Index(
                 ndim=ndim,
                 expansion_add=expansion_add,
@@ -150,12 +174,28 @@ class Main:
                 connectivity=connectivity,
             )
             bench_usearch(index, vectors_mat)
+            recall = recall_at_one(index, vectors_mat) * 100
+            print(f'- Recall @ 1 {recall:.2f} %')
+            recall_levels.append(recall)
             del index
 
-        min_ = min(recall_levels)
-        max_ = max(recall_levels)
-        print(f'Recall@1 varies between {min_:.3f} and {max_:.3f}')
+        min_ = min(recall_levels) * 100
+        max_ = max(recall_levels) * 100
+        print(f'Recall @ 1 varies between {min_:.3f} and {max_:.3f}')
+
+        # Now let's try clustering and inserting in clusters
+        try:
+            from sklearn.cluster import KMeans
+        except ImportError:
+            return
+
+        count_clusters = count / expansion_add
+        clustering = KMeans(
+            n_clusters=count_clusters,
+            random_state=0, n_init='auto').fit(vectors_mat)
+        NamedTuple()
 
 
 if __name__ == '__main__':
+    Main().robustness()
     fire.Fire(Main)

package/python/scripts/test.py

@@ -3,14 +3,21 @@ import pytest
 import numpy as np
 
 from usearch.io import load_matrix, save_matrix
-from usearch.index import Index, SetsIndex, MetricKind
+from usearch.synthetic import recall_members
+
+from usearch.index import Index, SetsIndex, MetricKind, Matches
+from usearch.index import (
+    DEFAULT_CONNECTIVITY,
+    DEFAULT_EXPANSION_ADD,
+    DEFAULT_EXPANSION_SEARCH,
+)
 
 
 dimensions = [13, 99, 100, 256]
 batch_sizes = [1, 2, 100]
 index_types = ['f32', 'f64', 'f16', 'f8']
 numpy_types = [np.float32, np.float64, np.float16, np.byte]
-connectivity_options = [3, 13, 50]
+connectivity_options = [3, 13, 50, DEFAULT_CONNECTIVITY]
 jit_options = [False]
 continuous_metrics = [
     MetricKind.Cos,
@@ -26,6 +33,7 @@ hash_metrics = [
 @pytest.mark.parametrize('rows', batch_sizes)
 @pytest.mark.parametrize('cols', dimensions)
 def test_serializing_fbin_matrix(rows: int, cols: int):
+
     original = np.random.rand(rows, cols).astype(np.float32)
     save_matrix(original, 'tmp.fbin')
     reconstructed = load_matrix('tmp.fbin')
@@ -36,6 +44,7 @@ def test_serializing_fbin_matrix(rows: int, cols: int):
 @pytest.mark.parametrize('rows', batch_sizes)
 @pytest.mark.parametrize('cols', dimensions)
 def test_serializing_ibin_matrix(rows: int, cols: int):
+
     original = np.random.randint(0, rows+1, size=(rows, cols)).astype(np.int32)
     save_matrix(original, 'tmp.ibin')
     reconstructed = load_matrix('tmp.ibin')
@@ -59,6 +68,8 @@ def test_index(
         ndim=ndim,
         dtype=index_type,
         connectivity=connectivity,
+        expansion_add=DEFAULT_EXPANSION_ADD,
+        expansion_search=DEFAULT_EXPANSION_SEARCH,
         jit=jit,
     )
     assert index.ndim == ndim
@@ -76,6 +87,11 @@ def test_index(
     assert matches[0] == 42
     assert distances[0] == pytest.approx(0, abs=1e-3)
 
+    assert 42 in index
+    assert 42 in index.labels
+    if numpy_type != np.byte:
+        assert np.allclose(index[42], vector, atol=0.1)
+
     index.save('tmp.usearch')
     index.clear()
     assert len(index) == 0
@@ -101,10 +117,13 @@ def test_index_batch(
     labels = np.array(range(batch_size), dtype=np.longlong)
 
     index.add(labels, vectors)
-    matches, distances, count = index.search(vectors, 10)
+    matches: Matches = index.search(vectors, 10)
 
-    assert matches.shape[0] == distances.shape[0]
-    assert count.shape[0] == batch_size
+    assert matches.labels.shape[0] == matches.distances.shape[0]
+    assert matches.counts.shape[0] == batch_size
+    assert np.all(np.sort(index.labels) == np.sort(labels))
+
+    assert recall_members(index, exact=True) == 1
 
 
 @pytest.mark.parametrize('ndim', dimensions)

package/python/usearch/index.py

@@ -17,6 +17,9 @@ from usearch.compiled import (
     DEFAULT_CONNECTIVITY,
     DEFAULT_EXPANSION_ADD,
     DEFAULT_EXPANSION_SEARCH,
+
+    USES_OPENMP,
+    USES_SIMSIMD,
 )
 
 BitwiseMetricKind = (
@@ -65,6 +68,10 @@ class Matches(NamedTuple):
             for label, distance in zip(labels, distances)
         ]
 
+    def recall_first(self, expected: np.ndarray) -> float:
+        best_matches = self.labels if self.is_batch else self.labels[:, 0]
+        return np.sum(best_matches == expected) / len(expected)
+
     def __repr__(self) -> str:
         return f'usearch.Matches({self.total_matches})' if self.is_batch else \
             f'usearch.Matches({self.total_matches} across {self.batch_size} queries)'
@@ -344,7 +351,7 @@ class Index:
         return Matches(*tuple_)
 
     def __len__(self) -> int:
-        return len(self._compiled)
+        return self._compiled.__len__()
 
     @property
     def jit(self) -> bool:
@@ -387,11 +394,11 @@ class Index:
         return self._compiled.expansion_search
 
     @expansion_add.setter
-    def change_expansion_add(self, v: int):
+    def expansion_add(self, v: int):
         self._compiled.expansion_add = v
 
     @expansion_search.setter
-    def change_expansion_search(self, v: int):
+    def expansion_search(self, v: int):
         self._compiled.expansion_search = v
 
     def save(self, path: os.PathLike):
@@ -406,5 +413,15 @@ class Index:
     def clear(self):
         self._compiled.clear()
 
-    def remove(self, label: int):
-        pass
+    @property
+    def labels(self) -> np.ndarray:
+        return self._compiled.labels
+
+    def __delitem__(self, label: int):
+        raise NotImplementedError()
+
+    def __contains__(self, label: int) -> bool:
+        return self._compiled.__contains__(label)
+
+    def __getitem__(self, label: int) -> np.ndarray:
+        return self._compiled.__getitem__(label)

package/python/usearch/synthetic.py

@@ -22,11 +22,10 @@ def vectors(index: Index, count: int) -> np.ndarray:
         return x
 
 
-def recall_at_one(index: Index, vectors: np.ndarray) -> float:
-    empty_replica: Index = index.fork()
-
-    labels = np.arange(vectors.shape[0])
-    empty_replica.add(labels=labels, vectors=vectors)
-    matches: Matches = empty_replica.search(empty_replica, 1)
-
-    return np.sum(matches.labels.flatten() == labels) / len(labels)
+def recall_members(index: Index, *args, **kwargs) -> float:
+    vectors: np.ndarray = np.vstack([
+        index[i] for i in range(len(index))
+    ])
+    labels: np.ndarray = np.arange(vectors.shape[0])
+    matches: Matches = index.search(vectors, 1, *args, **kwargs)
+    return matches.recall_first(labels)

package/src/punned.hpp

@@ -33,6 +33,14 @@
 namespace unum {
 namespace usearch {
 
+enum class common_scalar_kind_t {
+    f64_k,
+    f32_k,
+    f16_k,
+    f8_k,
+    b1_k,
+};
+
 using byte_t = char;
 using punned_distance_t = float;
 using punned_metric_t = punned_distance_t (*)(byte_t const*, byte_t const*, std::size_t, std::size_t);
@@ -527,6 +535,17 @@ class punned_gt {
         return typed_->memory_usage(allocator_entry_bytes);
     }
 
+    bool contains(label_t label) const {
+        shared_lock_t lock(lookup_table_mutex_);
+        return lookup_table_.contains(label);
+    }
+
+    void export_labels(label_t* labels, std::size_t limit) const {
+        shared_lock_t lock(lookup_table_mutex_);
+        for (auto it = lookup_table_.begin(); it != lookup_table_.end() && limit; ++it, ++labels, --limit)
+            *labels = it->first;
+    }
+
     // clang-format off
     add_result_t add(label_t label, f8_bits_t const* vector) { return add_(label, vector, casts_.from_f8); }
     add_result_t add(label_t label, f16_bits_t const* vector) { return add_(label, vector, casts_.from_f16); }