usearch 0.20.0 → 0.22.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "0.20.0",
+    "version": "0.22.0",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ashot Vardanian",
     "license": "Apache 2.0",

package/python/README.md

@@ -0,0 +1,283 @@
+# USearch for Python
+
+## Installation
+
+```sh
+pip install usearch
+```
+
+## Quickstart
+
+```python
+import numpy as np
+from usearch.index import Index
+
+index = Index(
+    ndim=3, # Define the number of dimensions in input vectors
+    metric='cos', # Choose 'l2sq', 'haversine' or other metric, default = 'ip'
+    dtype='f32', # Quantize to 'f16' or 'f8' if needed, default = 'f32'
+    connectivity=16, # How frequent should the connections in the graph be, optional
+    expansion_add=128, # Control the recall of indexing, optional
+    expansion_search=64, # Control the quality of search, optional
+)
+
+vector = np.array([0.2, 0.6, 0.4])
+index.add(42, vector)
+matches, distances, count = index.search(vector, 10)
+
+assert len(index) == 1
+assert count == 1
+assert matches[0] == 42
+assert distances[0] <= 0.001
+assert np.allclose(index[42], vector)
+```
+
+Python bindings are implemented with [`pybind/pybind11`](https://github.com/pybind/pybind11).
+Assuming the presence of Global Interpreter Lock in Python, we spawn threads in the C++ layer on large insertions.
+
+## Serialization
+
+```py
+index.save('index.usearch')
+index.load('index.usearch') # Copy the whole index into memory
+index.view('index.usearch') # View from disk without loading in memory
+```
+
+If you don't know anything about the index except its path, there are two more endpoints to know:
+
+```py
+Index.metadata('index.usearch') -> IndexMetadata
+Index.restore('index.usearch', view=False) -> Index
+```
+
+## Batch Operations
+
+Adding or querying a batch of entries is identical to adding a single vector.
+The difference would be in the shape of the tensors.
+
+```py
+n = 100
+labels = np.arange(n)
+vectors = np.random.uniform(0, 0.3, (n, index.ndim)).astype(np.float32)
+
+index.add(labels, vectors, threads=..., copy=...)
+matches, distances, counts = index.search(vectors, 10, threads=...)
+
+assert matches.shape[0] == vectors.shape[0]
+assert counts[0] <= 10
+```
+
+You can also override the default `threads` and `copy` arguments in bulk workloads.
+The first controls the number of threads spawned for the task.
+The second controls whether the vector itself will be persisted inside the index.
+If you can preserve the lifetime of the vector somewhere else, you can avoid the copy.
+
+## User-Defined Metrics and JIT in Python
+
+### [Numba][numba]
+
+Assuming the language boundary exists between Python user code and C++ implementation, there are more efficient solutions than passing a Python callable to the engine.
+Luckily, with the help of [Numba][numba], we can JIT compile a function with a matching signature and pass it down to the engine.
+
+```py
+from numba import cfunc, types, carray
+
+ndim = 256
+signature = types.float32(
+    types.CPointer(types.float32),
+    types.CPointer(types.float32))
+
+@cfunc(signature)
+def inner_product(a, b):
+    a_array = carray(a, ndim)
+    b_array = carray(b, ndim)
+    c = 0.0
+    for i in range(ndim):
+        c += a_array[i] * b_array[i]
+    return 1 - c
+
+index = Index(ndim=ndim, metric=CompiledMetric(
+    pointer=inner_product.address,
+    kind=MetricKind.IP,
+    signature=MetricSignature.ArrayArray,
+))
+```
+
+Alternatively, you can avoid pre-defining the number of dimensions, and pass it separately:
+
+```py
+signature = types.float32(
+    types.CPointer(types.float32),
+    types.CPointer(types.float32),
+    types.uint64)
+
+@cfunc(signature)
+def inner_product(a, b, ndim):
+    a_array = carray(a, ndim)
+    b_array = carray(b, ndim)
+    c = 0.0
+    for i in range(ndim):
+        c += a_array[i] * b_array[i]
+    return 1 - c
+
+index = Index(ndim=ndim, metric=CompiledMetric(
+    pointer=inner_product.address,
+    kind=MetricKind.IP,
+    signature=MetricSignature.ArrayArraySize,
+))
+```
+
+```sh
+pip install numba
+```
+
+### [Cppyy][cppyy]
+
+Similarly, you can use Cppyy with Cling to JIT-compile native C or C++ code and pass it to USearch, which may be a good idea, if you want to explicitly request loop-unrolling or other low-level optimizations!
+
+```py
+import cppyy
+import cppyy.ll
+
+cppyy.cppdef("""
+float inner_product(float *a, float *b) {
+    float result = 0;
+#pragma unroll
+    for (size_t i = 0; i != ndim; ++i)
+        result += a[i] * b[i];
+    return 1 - result;
+}
+""".replace("ndim", str(ndim)))
+
+function = cppyy.gbl.inner_product
+index = Index(ndim=ndim, metric=CompiledMetric(
+    pointer=cppyy.ll.addressof(function),
+    kind=MetricKind.IP,
+    signature=MetricSignature.ArrayArraySize,
+))
+```
+
+```sh
+conda install -c conda-forge cppyy
+```
+
+### [PeachPy][peachpy]
+
+We have covered JIT-ing Python with Numba and C++ with Cppyy and Cling.
+How about writing Assembly directly?
+That is also possible.
+Below is an example of constructing the "Inner Product" distance for 8-dimensional `f32` vectors for x86 using [PeachPy][peachpy].
+
+```py
+from peachpy import (
+    Argument,
+    ptr,
+    float_,
+    const_float_,
+)
+from peachpy.x86_64 import (
+    abi,
+    Function,
+    uarch,
+    isa,
+    GeneralPurposeRegister64,
+    LOAD,
+    YMMRegister,
+    VSUBPS,
+    VADDPS,
+    VHADDPS,
+    VMOVUPS,
+    VFMADD231PS,
+    VPERM2F128,
+    VXORPS,
+    RETURN,
+)
+
+a = Argument(ptr(const_float_), name="a")
+b = Argument(ptr(const_float_), name="b")
+
+with Function(
+    "inner_product", (a, b), float_, target=uarch.default + isa.avx2
+) as asm_function:
+    
+    # Request two 64-bit general-purpose registers for addresses
+    reg_a, reg_b = GeneralPurposeRegister64(), GeneralPurposeRegister64()
+    LOAD.ARGUMENT(reg_a, a)
+    LOAD.ARGUMENT(reg_b, b)
+
+    # Load the vectors
+    ymm_a = YMMRegister()
+    ymm_b = YMMRegister()
+    VMOVUPS(ymm_a, [reg_a])
+    VMOVUPS(ymm_b, [reg_b])
+
+    # Prepare the accumulator
+    ymm_c = YMMRegister()
+    ymm_one = YMMRegister()
+    VXORPS(ymm_c, ymm_c, ymm_c)
+    VXORPS(ymm_one, ymm_one, ymm_one)
+
+    # Accumulate A and B product into C
+    VFMADD231PS(ymm_c, ymm_a, ymm_b)
+
+    # Reduce the contents of a YMM register
+    ymm_c_permuted = YMMRegister()
+    VPERM2F128(ymm_c_permuted, ymm_c, ymm_c, 1)
+    VADDPS(ymm_c, ymm_c, ymm_c_permuted)
+    VHADDPS(ymm_c, ymm_c, ymm_c)
+    VHADDPS(ymm_c, ymm_c, ymm_c)
+
+    # Negate the values, to go from "similarity" to "distance"
+    VSUBPS(ymm_c, ymm_one, ymm_c)
+
+    # A common convention is to return floats in XMM registers
+    RETURN(ymm_c.as_xmm)
+
+python_function = asm_function.finalize(abi.detect()).encode().load()
+metric = CompiledMetric(
+    pointer=python_function.loader.code_address,
+    kind=MetricKind.IP,
+    signature=MetricSignature.ArrayArray,
+)
+index = Index(ndim=ndim, metric=metric)
+```
+
+[numba]: https://numba.readthedocs.io/en/stable/reference/jit-compilation.html#c-callbacks
+[cppyy]: https://cppyy.readthedocs.io/en/latest/
+[peachpy]: https://github.com/Maratyszcza/PeachPy
+
+## Tooling
+
+To work with `bbin`, `fbin`, `ibin`, `hbin` matrix files USearch provides `load_matrix` and `save_matrix`.
+Such files are standard in k-ANN tasks and represent a binary object with all the scalars, prepended by two 32-bit integers - the number of rows and columns in the matrix.
+
+```py
+from usearch.index import Index
+from usearch.io import load_matrix, save_matrix
+
+vectors = load_matrix('deep1B.fbin')
+index = Index(ndim=vectors.shape[1])
+index.add(labels, vectors)
+```
+
+One may often want to evaluate the quality of the constructed index before running in production.
+The trivial way is to measure `recall@1` on the entries already present in the index.
+
+```py
+from usearch.eval import recall_members
+
+assert recall_members(index, exact=True) == 1
+print(recall_members(index, exact=False))
+```
+
+In case you have some ground-truth data for more than one entry, you compare search results against expected values:
+
+```py
+from usearch.eval import relevance, dcg, ndcg, random_vectors
+
+vectors = random_vectors(index=index)
+matches_approximate = index.search(vectors)
+matches_exact = index.search(vectors, exact=True)
+relevance_scores = relevance(matches_exact, matches_approximate)
+print(dcg(relevance_scores), ndcg(relevance_scores))
+```

package/python/lib.cpp

@@ -15,9 +15,11 @@
 #include <limits> // `std::numeric_limits`
 #include <thread> // `std::thread`
 
+#define _CRT_SECURE_NO_WARNINGS
 #define PY_SSIZE_T_CLEAN
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
 
 #include <usearch/index_punned_dense.hpp>
 
@@ -44,6 +46,7 @@ using distance_t = punned_distance_t;
 using dense_index_t = punned_small_t;
 using dense_add_result_t = typename dense_index_t::add_result_t;
 using dense_search_result_t = typename dense_index_t::search_result_t;
+using dense_labeling_result_t = typename dense_index_t::labeling_result_t;
 
 struct dense_index_py_t : public dense_index_t {
     using native_t = dense_index_t;
@@ -180,8 +183,8 @@ static void add_one_to_index(dense_index_py_t& index, label_t label, py::buffer
     if (vector_dimensions != static_cast<Py_ssize_t>(index.scalar_words()))
         throw std::invalid_argument("The number of vector dimensions doesn't match!");
 
-    if (index.size() + 1 >= index.capacity())
-        index.reserve(ceil2(index.size() + 1));
+    if (!index.reserve(ceil2(index.size() + 1)))
+        throw std::invalid_argument("Out of memory!");
 
     add_config_t config;
     config.store_vector = copy;
@@ -223,9 +226,6 @@ static void add_many_to_index(                                      //
     dense_index_py_t& index, py::buffer labels, py::buffer vectors, //
     bool copy, std::size_t threads) {
 
-    if (index.limits().threads_add < threads)
-        throw std::invalid_argument("Can't use that many threads!");
-
     py::buffer_info labels_info = labels.request();
     py::buffer_info vectors_info = vectors.request();
 
@@ -247,8 +247,10 @@ static void add_many_to_index(                                      //
     if (labels_count != vectors_count)
         throw std::invalid_argument("Number of labels and vectors must match!");
 
-    if (index.size() + vectors_count >= index.capacity())
-        index.reserve(ceil2(index.size() + vectors_count));
+    if (!threads)
+        threads = std::thread::hardware_concurrency();
+    if (!index.reserve(index_limits_t(ceil2(index.size() + vectors_count), threads)))
+        throw std::invalid_argument("Out of memory!");
 
     switch (numpy_string_to_kind(vectors_info.format)) {
     case scalar_kind_t::b1x8_k: add_typed_to_index<b1x8_t>(index, labels_info, vectors_info, copy, threads); break;
@@ -316,6 +318,11 @@ static void search_typed(                                   //
     Py_ssize_t vectors_count = vectors_info.shape[0];
     char const* vectors_data = reinterpret_cast<char const*>(vectors_info.ptr);
 
+    if (!threads)
+        threads = std::thread::hardware_concurrency();
+    if (!index.reserve(index_limits_t(index.size(), threads)))
+        throw std::invalid_argument("Out of memory!");
+
     executor_default_t{threads}.execute_bulk(vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
         search_config_t config;
         config.thread = thread_idx;
@@ -356,7 +363,6 @@ static py::tuple search_many_in_index( //
 
     Py_ssize_t vectors_count = vectors_info.shape[0];
     Py_ssize_t vectors_dimensions = vectors_info.shape[1];
-    char const* vectors_data = reinterpret_cast<char const*>(vectors_info.ptr);
     if (vectors_dimensions != static_cast<Py_ssize_t>(index.scalar_words()))
         throw std::invalid_argument("The number of vector dimensions doesn't match!");
 
@@ -381,10 +387,48 @@ static py::tuple search_many_in_index( //
     return results;
 }
 
+static std::unordered_map<label_t, label_t> join_index(   //
+    dense_index_py_t const& a, dense_index_py_t const& b, //
+    std::size_t max_proposals, bool exact) {
+
+    std::unordered_map<label_t, label_t> a_to_b;
+    a_to_b.reserve((std::min)(a.size(), b.size()));
+
+    using join_result_t = typename dense_index_py_t::join_result_t;
+    join_config_t config;
+
+    config.max_proposals = max_proposals;
+    config.exact = exact;
+    config.expansion = (std::max)(a.expansion_search(), b.expansion_search());
+    std::size_t threads = (std::min)(a.limits().threads(), b.limits().threads());
+    executor_default_t executor{threads};
+    join_result_t result = dense_index_py_t::join( //
+        a, b, config,                              //
+        a_to_b,                                    //
+        dummy_label_to_label_mapping_t{},          //
+        executor);
+    result.error.raise();
+    return a_to_b;
+}
+
+static dense_index_py_t copy_index(dense_index_py_t const& index) {
+
+    using copy_result_t = typename dense_index_py_t::copy_result_t;
+    copy_config_t config;
+
+    copy_result_t result = index.copy();
+    result.error.raise();
+    return std::move(result.index);
+}
+
 // clang-format off
 template <typename index_at> void save_index(index_at const& index, std::string const& path) { index.save(path.c_str()).error.raise(); }
 template <typename index_at> void load_index(index_at& index, std::string const& path) { index.load(path.c_str()).error.raise(); }
 template <typename index_at> void view_index(index_at& index, std::string const& path) { index.view(path.c_str()).error.raise(); }
+template <typename index_at> void clear_index(index_at& index) { index.clear(); }
+template <typename index_at> std::size_t max_level(index_at const &index) { return index.max_level(); }
+template <typename index_at> typename index_at::stats_t compute_stats(index_at const &index) { return index.stats(); }
+template <typename index_at> typename index_at::stats_t compute_level_stats(index_at const &index, std::size_t level) { return index.stats(level); }
 // clang-format on
 
 template <typename internal_at, typename external_at = internal_at, typename index_at = void>
@@ -412,16 +456,17 @@ template <typename index_at> py::object get_member(index_at const& index, label_
         throw std::invalid_argument("Incompatible scalars in the query matrix!");
 }
 
-template <typename index_at> py::array_t<label_t> get_labels(index_at const& index, std::size_t limit) {
+template <typename index_at>
+py::array_t<label_t> get_labels(index_at const& index, std::size_t offset, std::size_t limit) {
     limit = std::min(index.size(), limit);
     py::array_t<label_t> result_py(static_cast<Py_ssize_t>(limit));
     auto result_py1d = result_py.template mutable_unchecked<1>();
-    index.export_labels(&result_py1d(0), limit);
+    index.export_labels(&result_py1d(0), offset, limit);
     return result_py;
 }
 
 template <typename index_at> py::array_t<label_t> get_all_labels(index_at const& index) {
-    return get_labels(index, index.size());
+    return get_labels(index, 0, index.size());
 }
 
 template <typename element_at> bool has_duplicates(element_at const* begin, element_at const* end) {
@@ -628,6 +673,48 @@ PYBIND11_MODULE(compiled, m) {
         py::arg("threads") = 0           //
     );
 
+    i.def(
+        "rename",
+        [](dense_index_py_t& index, label_t from, label_t to) -> bool {
+            dense_labeling_result_t result = index.rename(from, to);
+            result.error.raise();
+            return result.completed;
+        },
+        py::arg("from"), py::arg("to"));
+
+    i.def(
+        "remove",
+        [](dense_index_py_t& index, label_t label, bool compact, std::size_t threads) -> bool {
+            dense_labeling_result_t result = index.remove(label);
+            result.error.raise();
+
+            if (!threads)
+                threads = std::thread::hardware_concurrency();
+            if (!index.reserve(index_limits_t(index.size(), threads)))
+                throw std::invalid_argument("Out of memory!");
+
+            index.compact(executor_default_t{threads});
+            return result.completed;
+        },
+        py::arg("label"), py::arg("compact"), py::arg("threads"));
+
+    i.def(
+        "remove",
+        [](dense_index_py_t& index, std::vector<label_t> const& labels, bool compact,
+           std::size_t threads) -> std::size_t {
+            dense_labeling_result_t result = index.remove(labels.begin(), labels.end());
+            result.error.raise();
+
+            if (!threads)
+                threads = std::thread::hardware_concurrency();
+            if (!index.reserve(index_limits_t(index.size(), threads)))
+                throw std::invalid_argument("Out of memory!");
+
+            index.compact(executor_default_t{threads});
+            return result.completed;
+        },
+        py::arg("label"), py::arg("compact"), py::arg("threads"));
+
     i.def("__len__", &dense_index_py_t::size);
     i.def_property_readonly("size", &dense_index_py_t::size);
     i.def_property_readonly("ndim", &dense_index_py_t::dimensions);
@@ -643,14 +730,29 @@ PYBIND11_MODULE(compiled, m) {
     i.def_property("expansion_search", &dense_index_py_t::expansion_search, &dense_index_py_t::change_expansion_search);
 
     i.def_property_readonly("labels", &get_all_labels<dense_index_py_t>);
-    i.def("get_labels", &get_labels<dense_index_py_t>, py::arg("limit") = std::numeric_limits<std::size_t>::max());
+    i.def("get_labels", &get_labels<dense_index_py_t>, py::arg("offset") = 0,
+          py::arg("limit") = std::numeric_limits<std::size_t>::max());
     i.def("__contains__", &dense_index_py_t::contains);
     i.def("__getitem__", &get_member<dense_index_py_t>, py::arg("label"), py::arg("dtype") = scalar_kind_t::f32_k);
 
     i.def("save", &save_index<dense_index_py_t>, py::arg("path"), py::call_guard<py::gil_scoped_release>());
     i.def("load", &load_index<dense_index_py_t>, py::arg("path"), py::call_guard<py::gil_scoped_release>());
     i.def("view", &view_index<dense_index_py_t>, py::arg("path"), py::call_guard<py::gil_scoped_release>());
-    i.def("clear", &dense_index_py_t::clear, py::call_guard<py::gil_scoped_release>());
+    i.def("clear", &clear_index<dense_index_py_t>, py::call_guard<py::gil_scoped_release>());
+    i.def("copy", &copy_index, py::call_guard<py::gil_scoped_release>());
+    i.def("join", &join_index, py::arg("other"), py::arg("max_proposals") = 0, py::arg("exact") = false,
+          py::call_guard<py::gil_scoped_release>());
+
+    using punned_index_stats_t = typename dense_index_py_t::stats_t;
+    auto i_stats = py::class_<punned_index_stats_t>(m, "IndexStats");
+    i_stats.def_readonly("nodes", &punned_index_stats_t::nodes);
+    i_stats.def_readonly("edges", &punned_index_stats_t::edges);
+    i_stats.def_readonly("max_edges", &punned_index_stats_t::max_edges);
+    i_stats.def_readonly("allocated_bytes", &punned_index_stats_t::allocated_bytes);
+
+    i.def_property_readonly("max_level", &max_level<dense_index_py_t>);
+    i.def_property_readonly("levels_stats", &compute_stats<dense_index_py_t>);
+    i.def("level_stats", &compute_level_stats<dense_index_py_t>, py::arg("level"));
 
     auto si = py::class_<sparse_index_py_t>(m, "SparseIndex");
 
@@ -704,8 +806,8 @@ PYBIND11_MODULE(compiled, m) {
     si.def_property_readonly("connectivity", &sparse_index_py_t::connectivity);
     si.def_property_readonly("capacity", &sparse_index_py_t::capacity);
 
-    si.def("save", &sparse_index_py_t::save, py::arg("path"));
-    si.def("load", &sparse_index_py_t::load, py::arg("path"));
-    si.def("view", &sparse_index_py_t::view, py::arg("path"));
+    si.def("save", &save_index<sparse_index_py_t>, py::arg("path"));
+    si.def("load", &load_index<sparse_index_py_t>, py::arg("path"));
+    si.def("view", &view_index<sparse_index_py_t>, py::arg("path"));
     si.def("clear", &sparse_index_py_t::clear);
 }

package/python/scripts/join.py

@@ -0,0 +1,155 @@
+"""
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/clip_images.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/clip_texts.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/images.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/texts.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/uform_english_images.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/uform_english_texts.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/clipbigg_images.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-cc-3m/resolve/main/clipbigg_texts.fbin -P datasets/cc_3M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-arxiv-2m/resolve/main/e5base_abstract.fbin -P datasets/arxiv_2M/
+wget -nc https://huggingface.co/datasets/unum-cloud/ann-arxiv-2m/resolve/main/e5base_title.fbin -P datasets/arxiv_2M/
+rm -rf datasets/cc_3M/*.usearch datasets/arxiv_2M/*.usearch
+python python/scripts/join.py
+"""
+from numpy import dot
+from numpy.linalg import norm
+from tqdm import tqdm
+from simsimd import cos_f32x4_neon, to_int
+
+from usearch.index import Index, MetricKind, CompiledMetric, MetricSignature
+from usearch.io import load_matrix
+from usearch.eval import measure_seconds
+
+k = 10
+exact = False
+batch_size = 1024 * 4
+max_elements = 1000000
+
+a_name = "cc_3M/texts"
+b_name = "cc_3M/images"
+
+a_mat = load_matrix(f"datasets/{a_name}.fbin", view=True)
+b_mat = load_matrix(f"datasets/{b_name}.fbin", view=True)
+
+a_mat = a_mat[:max_elements]
+b_mat = b_mat[:max_elements]
+
+print(f"Loaded two datasets of shape: {a_mat.shape}, {b_mat.shape}")
+print("--------------------------------------")
+print("---------------Indexing---------------")
+print("--------------------------------------")
+
+metric = CompiledMetric(
+    pointer=to_int(cos_f32x4_neon),
+    kind=MetricKind.Cos,
+    signature=MetricSignature.ArrayArraySize,
+)
+
+a = Index(
+    a_mat.shape[1],
+    metric=metric,
+    path=f"datasets/{a_name}-{max_elements}.f32.usearch",
+    dtype="f32",
+)
+b = Index(
+    b_mat.shape[1],
+    metric=metric,
+    path=f"datasets/{b_name}-{max_elements}.f32.usearch",
+    dtype="f32",
+)
+
+if len(a) != a_mat.shape[0]:
+    a.clear()
+    a.add(None, a_mat, log=True, batch_size=batch_size)
+    a.save()
+
+if len(b) != b_mat.shape[0]:
+    b.clear()
+    b.add(None, b_mat, log=True, batch_size=batch_size)
+    b.save()
+
+
+print(
+    f"Loaded two indexes of size: {len(a):,} for {a_name} and {len(b):,} for {b_name}"
+)
+min_elements = min(len(a), len(b))
+
+run_diagnostics = input("Would you like to run diagnostics? [Y/n]: ")
+if len(run_diagnostics) == 0 or run_diagnostics.lower() == "y":
+    print("--------------------------------------")
+    print("-------------Diagnostics--------------")
+    print("--------------------------------------")
+
+    mean_similarity = 0.0
+    mean_recovered_similarity = 0.0
+
+    for i in tqdm(range(min_elements), desc="Pairwise Similarity"):
+        a_vec = a_mat[i]
+        b_vec = b_mat[i]
+        cos_similarity = dot(a_vec, b_vec) / (norm(a_vec) * norm(b_vec))
+        mean_similarity += cos_similarity
+
+        a_vec = a[i]
+        b_vec = b[i]
+        cos_similarity = dot(a_vec, b_vec) / (norm(a_vec) * norm(b_vec))
+        mean_recovered_similarity += cos_similarity
+
+    mean_similarity /= min_elements
+    mean_recovered_similarity /= min_elements
+    print(
+        f"Average vector similarity is {mean_similarity:.4f} in original dataset, "
+        f"and {mean_recovered_similarity:.4f} in recovered state in index"
+    )
+
+    dt = measure_seconds
+    args = dict(
+        k=k,
+        batch_size=batch_size,
+        log=True,
+        exact=exact,
+    )
+
+    secs, a_self_recall = dt(lambda: a.search(a.vectors, **args).recall(a.labels))
+    print(
+        "Self-recall @{} of {} index: {:.2f}%, took {:.2f}s".format(
+            k, a_name, a_self_recall * 100, secs
+        )
+    )
+
+    secs, b_self_recall = dt(lambda: b.search(b.vectors, **args).recall(b.labels))
+    print(
+        "Self-recall @{} of {} index: {:.2f}%, took {:.2f}s".format(
+            k, b_name, b_self_recall * 100, secs
+        )
+    )
+
+    secs, ab_recall = dt(lambda: b.search(a.vectors, **args).recall(b.labels))
+    print(
+        "Cross-recall @{} of {} in {}: {:.2f}%, took {:.2f}s".format(
+            k, a_name, b_name, ab_recall * 100, secs
+        )
+    )
+
+    secs, ba_recall = dt(lambda: a.search(b.vectors, **args).recall(a.labels))
+    print(
+        "Cross-recall @{} of {} in {}: {:.2f}%, took {:.2f}s".format(
+            k, b_name, a_name, ba_recall * 100, secs
+        )
+    )
+
+
+print("--------------------------------------")
+print("-----------------Join-----------------")
+print("--------------------------------------")
+
+secs, bimapping = measure_seconds(lambda: a.join(b, max_proposals=100))
+mapping_size = len(bimapping)
+recall = 0
+for i, j in bimapping.items():
+    recall += i == j
+
+recall *= 100.0 / min_elements
+print(
+    f"Took {secs:.2f}s to find {mapping_size:,} pairings with {recall:.2f}% being exact"
+)