usearch 0.20.0 → 0.22.0

package/python/usearch/index.py

@@ -5,12 +5,13 @@ from __future__ import annotations
 # Python tooling, linters, and static analyzers. It also embeds JIT
 # into the primary `Index` class, connecting USearch with Numba.
 import os
+import math
 from typing import Optional, Union, NamedTuple, List, Iterable
 
 import numpy as np
 from tqdm import tqdm
 
-from usearch.compiled import Index as _CompiledIndex, IndexMetadata
+from usearch.compiled import Index as _CompiledIndex, IndexMetadata, IndexStats
 from usearch.compiled import SparseIndex as _CompiledSetsIndex
 
 from usearch.compiled import MetricKind, ScalarKind, MetricSignature
@@ -141,6 +142,13 @@ class Matches(NamedTuple):
         best_matches = self.labels if not self.is_multiple else self.labels[:, 0]
         return np.sum(best_matches == expected) / len(expected)
 
+    def recall(self, expected: np.ndarray) -> float:
+        assert len(expected) == self.batch_size
+        recall = 0
+        for i in range(self.batch_size):
+            recall += expected[i] in self.labels[i]
+        return recall / len(expected)
+
     def __repr__(self) -> str:
         return (
             f"usearch.Matches({self.total_matches})"
@@ -311,15 +319,18 @@ class Index:
         pass `copy=False`, if you can guarantee the lifetime of the
         primary vectors store during the process of construction.
 
-        :param labels: Unique identifier for passed vectors, optional
+        :param labels: Unique identifier(s) for passed vectors, optional
         :type labels: Buffer
-        :param vectors: Collection of vectors.
+        :param vectors: Vector or a row-major matrix
         :type vectors: Buffer
         :param copy: Should the index store a copy of vectors, defaults to True
         :type copy: bool, optional
         :param threads: Optimal number of cores to use, defaults to 0
         :type threads: int, optional
-
+        :param log: Whether to print the progress bar, default to False
+        :type log: Union[str, bool], optional
+        :param batch_size: Number of vectors to process at once, defaults to 0
+        :type batch_size: int, optional
         :return: Inserted label or labels
         :type: Union[int, np.ndarray]
         """
@@ -347,6 +358,10 @@ class Index:
         count_labels = len(labels) if isinstance(labels, Iterable) else 1
         assert count_labels == count_vectors
 
+        # If logging is requested, and batch size is undefined, set it to grow 1% at a time:
+        if log and batch_size == 0:
+            batch_size = int(math.ceil(count_vectors / 100))
+
         # Split into batches and log progress, if needed
         if batch_size:
             labels = [
@@ -387,27 +402,99 @@ class Index:
         log: Union[str, bool] = False,
         batch_size: int = 0,
     ) -> Matches:
-        """Performs approximate nearest neighbors search for one or more queries.
+        """
+        Performs approximate nearest neighbors search for one or more queries.
 
         :param vectors: Query vector or vectors.
         :type vectors: Buffer
         :param k: Upper limit on the number of matches to find, defaults to 10
         :type k: int, optional
-
         :param threads: Optimal number of cores to use, defaults to 0
         :type threads: int, optional
         :param exact: Perform exhaustive linear-time exact search, defaults to False
         :type exact: bool, optional
+        :param log: Whether to print the progress bar, default to False
+        :type log: Union[str, bool], optional
+        :param batch_size: Number of vectors to process at once, defaults to 0
+        :type batch_size: int, optional
         :return: Approximate matches for one or more queries
         :rtype: Matches
         """
-        tuple_ = self._compiled.search(
-            vectors,
-            k,
-            exact=exact,
-            threads=threads,
-        )
-        return Matches(*tuple_)
+
+        assert isinstance(vectors, np.ndarray), "Expects a NumPy array"
+        assert vectors.ndim == 1 or vectors.ndim == 2, "Expects a matrix or vector"
+        count_vectors = vectors.shape[0] if vectors.ndim == 2 else 1
+
+        if log and batch_size == 0:
+            batch_size = int(math.ceil(count_vectors / 100))
+
+        if batch_size:
+            tasks = [
+                vectors[start_row : start_row + batch_size, :]
+                for start_row in range(0, count_vectors, batch_size)
+            ]
+            tasks_matches = []
+            name = log if isinstance(log, str) else "Search"
+            pbar = tqdm(
+                tasks,
+                desc=name,
+                total=count_vectors,
+                unit="Vector",
+                disable=log is False,
+            )
+            for vectors in tasks:
+                tuple_ = self._compiled.search(
+                    vectors,
+                    k,
+                    exact=exact,
+                    threads=threads,
+                )
+                tasks_matches.append(Matches(*tuple_))
+                pbar.update(vectors.shape[0])
+
+            pbar.close()
+            return Matches(
+                labels=np.vstack([m.labels for m in tasks_matches]),
+                distances=np.vstack([m.distances for m in tasks_matches]),
+                counts=np.concatenate([m.counts for m in tasks_matches], axis=None),
+            )
+
+        else:
+            tuple_ = self._compiled.search(
+                vectors,
+                k,
+                exact=exact,
+                threads=threads,
+            )
+            return Matches(*tuple_)
+
+    def remove(
+        self,
+        labels: Union[int, Iterable[int]],
+        *,
+        compact: bool = False,
+        threads: int = 0,
+    ) -> Union[bool, int]:
+        """Removes one or move vectors from the index.
+
+        When working with extremely large indexes, you may want to
+        mark some entries deleted, instead of rebuilding a filtered index.
+        In other cases, rebuilding - is the recommended approach.
+
+        :param labels: Unique identifier for passed vectors, optional
+        :type labels: Buffer
+        :param compact: Removes links to removed nodes (expensive), defaults to False
+        :type compact: bool, optional
+        :param threads: Optimal number of cores to use, defaults to 0
+        :type threads: int, optional
+        :return: Number of removed entries
+        :type: Union[bool, int]
+        """
+        return self._compiled.remove(labels, compact=compact, threads=threads)
+
+    def rename(self, label_from: int, label_to: int) -> bool:
+        """Relabel existing entry"""
+        return self._compiled.rename(label_from, label_to)
 
     @property
     def specs(self) -> dict:
@@ -429,6 +516,19 @@ class Index:
     def __len__(self) -> int:
         return self._compiled.__len__()
 
+    def __delitem__(self, label: int) -> bool:
+        raise self.remove(label)
+
+    def __contains__(self, label: int) -> bool:
+        return self._compiled.__contains__(label)
+
+    def __getitem__(self, label: int) -> np.ndarray:
+        dtype = self.dtype
+        get_dtype = _to_numpy_compatible_dtype(dtype)
+        vector = self._compiled.__getitem__(label, get_dtype)
+        view_dtype = _to_numpy_dtype(dtype)
+        return None if vector is None else vector.view(view_dtype)
+
     @property
     def jit(self) -> bool:
         return self._metric_jit is not None
@@ -442,8 +542,8 @@ class Index:
         return self._compiled.ndim
 
     @property
-    def metric(self) -> MetricKind:
-        return self._metric_kind
+    def metric(self) -> Union[MetricKind, CompiledMetric]:
+        return self._metric_jit if self._metric_jit else self._metric_kind
 
     @property
     def dtype(self) -> ScalarKind:
@@ -501,13 +601,64 @@ class Index:
     def close(self):
         self._compiled.close()
 
+    def copy(self) -> Index:
+        result = Index(
+            ndim=self.ndim,
+            metric=self.metric,
+            dtype=self.dtype,
+            connectivity=self.connectivity,
+            expansion_add=self.expansion_add,
+            expansion_search=self.expansion_search,
+            path=self.path,
+        )
+        result._compiled = self._compiled.copy()
+        return result
+
+    def join(self, other: Index, max_proposals: int = 0, exact: bool = False) -> dict:
+        """Performs "Semantic Join" or pairwise matching between `self` & `other` index.
+        Is different from `search`, as no collisions are allowed in resulting pairs.
+        Uses the concept of "Stable Marriages" from Combinatorics, famous for the 2012
+        Nobel Prize in Economics.
+
+        :param other: Another index.
+        :type other: Index
+        :param max_proposals: Limit on candidates evaluated per vector, defaults to 0
+        :type max_proposals: int, optional
+        :param exact: Controls if underlying `search` should be exact, defaults to False
+        :type exact: bool, optional
+        :return: Mapping from labels of `self` to labels of `other`
+        :rtype: dict
+        """
+        return self._compiled.join(
+            other=other._compiled,
+            max_proposals=max_proposals,
+            exact=exact,
+        )
+
+    def get_labels(self, offset: int = 0, limit: int = 0) -> np.ndarray:
+        if limit == 0:
+            limit = 2**63 - 1
+        return self._compiled.get_labels(offset, limit)
+
     @property
     def labels(self) -> np.ndarray:
+        """Retrieves the labels of all vectors present in `self`
+
+        :return: Array of labels
+        :rtype: np.ndarray
+        """
         return self._compiled.labels
 
     def get_vectors(
-        self, labels: np.ndarray, dtype: ScalarKind = ScalarKind.F32
+        self,
+        labels: np.ndarray,
+        dtype: ScalarKind = ScalarKind.F32,
     ) -> np.ndarray:
+        """Retrieves vectors associated with given `labels`
+
+        :return: Matrix of vectors (row-major)
+        :rtype: np.ndarray
+        """
         dtype = _normalize_dtype(dtype, self._metric_kind)
         get_dtype = _to_numpy_compatible_dtype(dtype)
         vectors = np.vstack([self._compiled.__getitem__(l, get_dtype) for l in labels])
@@ -518,15 +669,45 @@ class Index:
     def vectors(self) -> np.ndarray:
         return self.get_vectors(self.labels, self.dtype)
 
-    def __delitem__(self, label: int):
-        raise NotImplementedError()
+    @property
+    def max_level(self) -> int:
+        return self._compiled.max_level
 
-    def __contains__(self, label: int) -> bool:
-        return self._compiled.__contains__(label)
+    @property
+    def levels_stats(self) -> IndexStats:
+        return self._compiled.levels_stats
 
-    def __getitem__(self, label: int) -> np.ndarray:
-        dtype = self.dtype
-        get_dtype = _to_numpy_compatible_dtype(dtype)
-        vector = self._compiled.__getitem__(label, get_dtype)
-        view_dtype = _to_numpy_dtype(dtype)
-        return None if vector is None else vector.view(view_dtype)
+    def level_stats(self, level: int) -> IndexStats:
+        return self._compiled.level_stats(level)
+
+    def __repr__(self) -> str:
+        f = "usearch.Index({} x {}, {}, expansion: {} & {}, {} vectors in {} levels)"
+        return f.format(
+            self.dtype,
+            self.ndim,
+            self.metric,
+            self.expansion_add,
+            self.expansion_search,
+            len(self),
+            self.max_level + 1,
+        )
+
+    def _repr_pretty_(self) -> str:
+        level_stats = [
+            f"--- {i}. {self.level_stats(i).nodes} nodes" for i in range(self.max_level)
+        ]
+        return "\n".join(
+            [
+                "usearch.Index",
+                "- config" f"-- data type: {self.dtype}",
+                f"-- dimensions: {self.ndim}",
+                f"-- metric: {self.metric}",
+                f"-- expansion on addition:{self.expansion_add} candidates",
+                f"-- expansion on search: {self.expansion_search} candidates",
+                "- state",
+                f"-- size: {self.size} vectors",
+                f"-- memory usage: {self.memory_usage} bytes",
+                f"-- max level: {self.max_level}",
+                *level_stats,
+            ]
+        )

package/rust/README.md

@@ -0,0 +1,67 @@
+# USearch for Rust
+
+## Installation
+
+```sh
+cargo add usearch
+```
+
+## Quickstart
+
+```rust
+
+let options = IndexOptions {
+            dimensions: 5,
+            metric: MetricKind::IP,
+            quantization: ScalarKind::F16,
+            connectivity: 0,
+            expansion_add: 0,
+            expansion_search: 0
+        };
+
+let index = new_index(&options).unwrap();
+
+assert!(index.reserve(10).is_ok());
+assert!(index.capacity() >= 10);
+assert!(index.connectivity() != 0);
+assert_eq!(index.dimensions(), 3);
+assert_eq!(index.size(), 0);
+
+let first: [f32; 3] = [0.2, 0.1, 0.2];
+let second: [f32; 3] = [0.2, 0.1, 0.2];
+
+assert!(index.add(42, &first).is_ok());
+assert!(index.add(43, &second).is_ok());
+assert_eq!(index.size(), 2);
+
+// Read back the tags
+let results = index.search(&first, 10).unwrap();
+assert_eq!(results.count, 2);
+```
+
+## Multi-Threading
+
+```rust
+assert!(index.add_in_thread(42, &first, 0).is_ok());
+assert!(index.add_in_thread(43, &second, 0).is_ok());
+let results = index.search_in_thread(&first, 10, 0).unwrap();
+```
+
+Being a systems-programming language, Rust has better control over memory management and concurrency but lacks function overloading.
+Aside from the `add` and `search`, USearch Rust binding also provides `add_in_thread` and `search_in_thread`, which let users identify the calling thread to use underlying temporary memory more efficiently.
+
+## Serialization
+
+```rust
+assert!(index.save("index.usearch").is_ok());
+assert!(index.load("index.usearch").is_ok());
+assert!(index.view("index.usearch").is_ok());
+```
+
+## Metrics
+
+```rust
+assert!(new_l2sq(3, &quant, 0, 0, 0).is_ok());
+assert!(new_cos(3, &quant, 0, 0, 0).is_ok());
+assert!(new_haversine(&quant, 0, 0, 0).is_ok());
+```

package/swift/README.md

@@ -0,0 +1,20 @@
+# USearch for Swift
+
+## Installation
+
+```txt
+https://github.com/unum-cloud/usearch
+```
+
+## Quickstart
+
+```swift
+let index = Index.l2sq(dimensions: 3, connectivity: 8)
+let vectorA: [Float32] = [0.3, 0.5, 1.2]
+let vectorB: [Float32] = [0.4, 0.2, 1.2]
+index.add(label: 42, vector: vectorA[...])
+index.add(label: 43, vector: vectorB[...])
+
+let results = index.search(vector: vectorA[...], count: 10)
+assert(results.0[0] == 42)
+```

package/wasm/CMakeLists.txt

@@ -0,0 +1,15 @@
+include(emscripten)
+
+set(CMAKE_C_COMPILER emcc)
+set(CMAKE_CXX_COMPILER em++)
+
+set(USEARCH_PUNNED_INCLUDE_DIRS
+    "${CMAKE_CURRENT_SOURCE_DIR}/../c"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../include"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../fp16/include"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../robin-map/include"
+)
+
+add_executable(index lib.cpp)
+target_include_directories(index PRIVATE ${USEARCH_PUNNED_INCLUDE_DIRS})
+set_target_properties(index PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/wasm)

package/wasm/lib.cpp

@@ -0,0 +1,5 @@
+#define USEARCH_EXPORT EMSCRIPTEN_KEEPALIVE
+
+#include <emscripten/emscripten.h>
+
+#include <lib.cpp>

package/wasmer.toml

@@ -0,0 +1,8 @@
+[package]
+name="unum/usearch"
+version="0.19.3"
+description="Smaller & Faster Single-File Vector Search Engine from Unum"
+
+[[module]]
+name="index"
+source="build/wasm/index.wasm"

package/python/usearch/join.py

@@ -1,98 +0,0 @@
-from collections import deque
-
-import numpy as np
-from numba import njit
-
-from index import Index, Matches, Label
-
-
-class bidict(dict):
-    def __init__(self, *args, **kwargs):
-        super(bidict, self).__init__(*args, **kwargs)
-        self.inverse = {}
-        for key, value in self.items():
-            self.inverse.setdefault(value, []).append(key)
-
-    def __setitem__(self, key, value):
-        if key in self:
-            self.inverse[self[key]].remove(key)
-        super(bidict, self).__setitem__(key, value)
-        self.inverse.setdefault(value, []).append(key)
-
-    def __delitem__(self, key):
-        self.inverse.setdefault(self[key], []).remove(key)
-        if self[key] in self.inverse and not self.inverse[self[key]]:
-            del self.inverse[self[key]]
-        super(bidict, self).__delitem__(key)
-
-
-@njit
-def index_in_array(array: np.ndarray, item: Label) -> int:
-    for idx, val in np.ndenumerate(array):
-        if val == item:
-            return idx
-
-    return len(array)
-
-
-def semantic_join(a: Index, b: Index, resolution: int = 10) -> bidict:
-    """Performs a Semantic Join, matching one entry from `a` with
-    one entry from `b`, converging towards a Stable Marriage.
-    Assuming the collections can be different in size, classical solution
-    doesn't provide
-
-    :param resolution: Approximate matches per member to consider, defaults to 10
-    :type resolution: int, optional
-    :return: Bidirectional mapping from men to women and `.inverse`
-    :rtype: bidict
-    """
-
-    men, women = (a, b) if len(a) > len(b) else (b, a)
-    matches = bidict()
-
-    men_vectors = np.vstack(men[i] for i in range(len(men)))
-    women_vectors = np.vstack(women[i] for i in range(len(women)))
-
-    man_count_proposed = np.zeros(len(men))
-    man_to_women_preferences: Matches = men.search(women_vectors, resolution)
-    woman_to_men_preferences: Matches = women.search(men_vectors, resolution)
-
-    # A nice optimization is to prioritize free men by the quality of their
-    # best remaining variant. TODO: Replace with `heapq`
-    free_men = deque(range(len(men)))
-
-    idle_cycles: int = 0
-    hopeless_men_count = len(men) - len(women)
-    while len(free_men) > hopeless_men_count:
-        # In the worst case scenario we may need to match more candidates for every
-        # remaining man. This, however, may drastically increase the runtime.
-        if len(free_men) == idle_cycles:
-            break
-
-        man: Label = free_men.popleft()
-        count_proposals = man_count_proposed[man]
-        if count_proposals == man_to_women_preferences.counts[man]:
-            free_men.append(man)
-            idle_cycles += 1
-            continue
-
-        woman: Label = man_to_women_preferences.labels[man, count_proposals]
-        man_count_proposed[man] += 1
-
-        if woman not in matches.inverse:
-            matches[man] = woman
-
-        else:
-            her_preferences = woman_to_men_preferences.labels[woman, :]
-            husband: Label = matches.inverse[woman]
-            husband_rank = index_in_array(her_preferences, husband)
-            challenger_rank = index_in_array(her_preferences, man)
-            if challenger_rank < husband_rank:
-                del matches[husband]
-                free_men.append(husband)
-                matches[man] = woman
-
-            else:
-                free_men.append(man)
-
-    return matches