usearch 0.22.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.releaserc +10 -2
- package/.vscode/launch.json +10 -10
- package/.vscode/settings.json +9 -1
- package/.vscode/tasks.json +4 -4
- package/CMakeLists.txt +6 -1
- package/Cargo.toml +2 -2
- package/README.md +35 -41
- package/build.rs +2 -2
- package/c/CMakeLists.txt +48 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +83 -72
- package/c/test.c +20 -20
- package/c/usearch.h +132 -32
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +3 -1
- package/cpp/README.md +16 -16
- package/cpp/bench.cpp +50 -126
- package/cpp/test.cpp +164 -80
- package/docs/benchmarks.md +26 -5
- package/docs/c/reference.rst +3 -3
- package/docs/compilation.md +31 -49
- package/docs/cpp/index.rst +1 -1
- package/docs/cpp/reference.rst +4 -4
- package/golang/lib.go +275 -141
- package/golang/lib_test.go +105 -0
- package/include/usearch/index.hpp +1681 -1656
- package/include/usearch/index_dense.hpp +1400 -0
- package/include/usearch/index_plugins.hpp +1263 -0
- package/java/README.md +1 -1
- package/java/cloud/unum/usearch/Index.java +10 -10
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
- package/java/test/IndexTest.java +1 -1
- package/javascript/README.md +6 -6
- package/javascript/lib.cpp +156 -55
- package/javascript/test.js +26 -11
- package/javascript/usearch.d.ts +102 -0
- package/javascript/usearch.js +2 -118
- package/objc/USearchObjective.mm +57 -31
- package/objc/include/USearchObjective.h +32 -22
- package/package.json +5 -3
- package/python/README.md +23 -14
- package/python/lib.cpp +468 -444
- package/python/scripts/bench.ipynb +3 -2
- package/python/scripts/bench.py +4 -4
- package/python/scripts/index_faiss.py +5 -5
- package/python/scripts/join.py +4 -4
- package/python/scripts/test.py +152 -61
- package/python/scripts/test_jit.py +10 -12
- package/python/scripts/test_sparse.py +32 -0
- package/python/usearch/client.py +26 -28
- package/python/usearch/eval.py +73 -23
- package/python/usearch/index.py +520 -228
- package/python/usearch/io.py +57 -25
- package/python/usearch/numba.py +4 -4
- package/python/usearch/server.py +10 -10
- package/rust/lib.cpp +34 -30
- package/rust/lib.hpp +14 -12
- package/rust/lib.rs +8 -7
- package/setup.py +5 -2
- package/simsimd/.github/workflows/prerelease.yml +73 -0
- package/simsimd/.github/workflows/release.yml +110 -0
- package/simsimd/.github/workflows/update_version.sh +3 -0
- package/simsimd/.releaserc +77 -0
- package/simsimd/.vscode/launch.json +27 -1
- package/simsimd/.vscode/settings.json +22 -1
- package/simsimd/.vscode/tasks.json +6 -0
- package/simsimd/CMakeLists.txt +3 -2
- package/simsimd/README.md +82 -46
- package/simsimd/bench.cxx +63 -24
- package/simsimd/include/simsimd/simsimd.h +88 -66
- package/simsimd/include/simsimd/simsimd_chem.h +126 -0
- package/simsimd/package.json +8 -0
- package/simsimd/pyproject.toml +42 -0
- package/simsimd/python/python.c +81 -0
- package/simsimd/python/run.sh +1 -0
- package/simsimd/python/test.c +109 -0
- package/simsimd/python/test.py +23 -0
- package/simsimd/setup.py +44 -0
- package/swift/Index+Sugar.swift +75 -23
- package/swift/README.md +26 -2
- package/swift/Test.swift +19 -7
- package/swift/USearch.swift +2 -2
- package/tsconfig.json +8 -0
- package/wasm/CMakeLists.txt +1 -0
- package/wasmer.toml +1 -1
- package/wolfram/CMakeLists.txt +1 -0
- package/wolfram/lib.cpp +11 -13
- package/golang/usearch.h +0 -9
- package/golang/usearch_par_test.go +0 -40
- package/golang/usearch_test.go +0 -82
- package/include/usearch/index_punned_dense.hpp +0 -1001
- package/include/usearch/index_punned_helpers.hpp +0 -676
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
"metadata": {},
|
|
7
7
|
"outputs": [],
|
|
8
8
|
"source": [
|
|
9
|
-
"!pip install faiss-cpu numba pandas plotly nbformat jinja2 kaleido tqdm
|
|
9
|
+
"!pip install faiss-cpu numba pandas plotly nbformat jinja2 kaleido tqdm scikit-learn scipy matplotlib"
|
|
10
10
|
]
|
|
11
11
|
},
|
|
12
12
|
{
|
|
@@ -61,6 +61,7 @@
|
|
|
61
61
|
"metadata": {},
|
|
62
62
|
"outputs": [],
|
|
63
63
|
"source": [
|
|
64
|
+
"from usearch.eval import Dataset\n",
|
|
64
65
|
"data = Dataset.build(\n",
|
|
65
66
|
" vectors='../../datasets/wiki_1M/base.1M.fbin',\n",
|
|
66
67
|
" queries='../../datasets/wiki_1M/query.public.100K.fbin',\n",
|
|
@@ -161,7 +162,7 @@
|
|
|
161
162
|
"name": "python",
|
|
162
163
|
"nbconvert_exporter": "python",
|
|
163
164
|
"pygments_lexer": "ipython3",
|
|
164
|
-
"version": "3.10.
|
|
165
|
+
"version": "3.10.11"
|
|
165
166
|
},
|
|
166
167
|
"orig_nbformat": 4
|
|
167
168
|
},
|
package/python/scripts/bench.py
CHANGED
|
@@ -5,7 +5,7 @@ from dataclasses import asdict
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
import pandas as pd
|
|
7
7
|
|
|
8
|
-
from usearch.index import Index,
|
|
8
|
+
from usearch.index import Index, Key, MetricKind, ScalarKind
|
|
9
9
|
from usearch.numba import jit as njit
|
|
10
10
|
from usearch.eval import Evaluation, AddTask
|
|
11
11
|
from usearch.index import (
|
|
@@ -26,7 +26,7 @@ def bench_speed(
|
|
|
26
26
|
# Build various indexes:
|
|
27
27
|
indexes = []
|
|
28
28
|
jit_options = [False, True] if jit else [False]
|
|
29
|
-
dtype_options = [ScalarKind.F32, ScalarKind.F16, ScalarKind.
|
|
29
|
+
dtype_options = [ScalarKind.F32, ScalarKind.F16, ScalarKind.I8]
|
|
30
30
|
for jit, dtype in itertools.product(jit_options, dtype_options):
|
|
31
31
|
metric = MetricKind.IP
|
|
32
32
|
if jit:
|
|
@@ -38,7 +38,7 @@ def bench_speed(
|
|
|
38
38
|
expansion_add=expansion_add,
|
|
39
39
|
expansion_search=expansion_search,
|
|
40
40
|
connectivity=connectivity,
|
|
41
|
-
path="USearch" + ["", "+JIT"][jit] + ":" + dtype,
|
|
41
|
+
path="USearch" + ["", "+JIT"][jit] + ":" + str(dtype),
|
|
42
42
|
)
|
|
43
43
|
|
|
44
44
|
# Skip the cases, where JIT-ing is impossible
|
|
@@ -115,7 +115,7 @@ def bench_params(
|
|
|
115
115
|
results = []
|
|
116
116
|
for connectivity, ndim in itertools.product(connectivities, dimensions):
|
|
117
117
|
task = AddTask(
|
|
118
|
-
|
|
118
|
+
keys=np.arange(count, dtype=Key),
|
|
119
119
|
vectors=np.random.rand(count, ndim).astype(np.float32),
|
|
120
120
|
)
|
|
121
121
|
index = Index(
|
|
@@ -40,14 +40,14 @@ class IndexFAISS:
|
|
|
40
40
|
|
|
41
41
|
self.path = path
|
|
42
42
|
|
|
43
|
-
def add(self,
|
|
44
|
-
# Adding
|
|
45
|
-
# self._faiss.add_with_ids(vectors,
|
|
43
|
+
def add(self, keys, vectors):
|
|
44
|
+
# Adding keys isn't supported for most index types
|
|
45
|
+
# self._faiss.add_with_ids(vectors, keys)
|
|
46
46
|
self._faiss.add(vectors)
|
|
47
47
|
|
|
48
48
|
def search(self, queries, k: int) -> Matches:
|
|
49
|
-
distances,
|
|
50
|
-
return Matches(
|
|
49
|
+
distances, keys = self._faiss.search(queries, k)
|
|
50
|
+
return Matches(keys, distances, np.array([k] * queries.shape[0]))
|
|
51
51
|
|
|
52
52
|
def __len__(self) -> int:
|
|
53
53
|
return self._faiss.ntotal
|
package/python/scripts/join.py
CHANGED
|
@@ -110,28 +110,28 @@ if len(run_diagnostics) == 0 or run_diagnostics.lower() == "y":
|
|
|
110
110
|
exact=exact,
|
|
111
111
|
)
|
|
112
112
|
|
|
113
|
-
secs, a_self_recall = dt(lambda: a.search(a.vectors, **args).recall(a.
|
|
113
|
+
secs, a_self_recall = dt(lambda: a.search(a.vectors, **args).recall(a.keys))
|
|
114
114
|
print(
|
|
115
115
|
"Self-recall @{} of {} index: {:.2f}%, took {:.2f}s".format(
|
|
116
116
|
k, a_name, a_self_recall * 100, secs
|
|
117
117
|
)
|
|
118
118
|
)
|
|
119
119
|
|
|
120
|
-
secs, b_self_recall = dt(lambda: b.search(b.vectors, **args).recall(b.
|
|
120
|
+
secs, b_self_recall = dt(lambda: b.search(b.vectors, **args).recall(b.keys))
|
|
121
121
|
print(
|
|
122
122
|
"Self-recall @{} of {} index: {:.2f}%, took {:.2f}s".format(
|
|
123
123
|
k, b_name, b_self_recall * 100, secs
|
|
124
124
|
)
|
|
125
125
|
)
|
|
126
126
|
|
|
127
|
-
secs, ab_recall = dt(lambda: b.search(a.vectors, **args).recall(b.
|
|
127
|
+
secs, ab_recall = dt(lambda: b.search(a.vectors, **args).recall(b.keys))
|
|
128
128
|
print(
|
|
129
129
|
"Cross-recall @{} of {} in {}: {:.2f}%, took {:.2f}s".format(
|
|
130
130
|
k, a_name, b_name, ab_recall * 100, secs
|
|
131
131
|
)
|
|
132
132
|
)
|
|
133
133
|
|
|
134
|
-
secs, ba_recall = dt(lambda: a.search(b.vectors, **args).recall(a.
|
|
134
|
+
secs, ba_recall = dt(lambda: a.search(b.vectors, **args).recall(a.keys))
|
|
135
135
|
print(
|
|
136
136
|
"Cross-recall @{} of {} in {}: {:.2f}%, took {:.2f}s".format(
|
|
137
137
|
k, b_name, a_name, ba_recall * 100, secs
|
package/python/scripts/test.py
CHANGED
|
@@ -5,13 +5,16 @@ import numpy as np
|
|
|
5
5
|
|
|
6
6
|
from usearch.io import load_matrix, save_matrix
|
|
7
7
|
from usearch.eval import random_vectors
|
|
8
|
+
from usearch.index import search
|
|
8
9
|
|
|
9
10
|
from usearch.index import (
|
|
10
11
|
Index,
|
|
11
|
-
|
|
12
|
+
Indexes,
|
|
12
13
|
MetricKind,
|
|
13
14
|
ScalarKind,
|
|
15
|
+
Match,
|
|
14
16
|
Matches,
|
|
17
|
+
BatchMatches,
|
|
15
18
|
)
|
|
16
19
|
from usearch.index import (
|
|
17
20
|
DEFAULT_CONNECTIVITY,
|
|
@@ -19,6 +22,11 @@ from usearch.index import (
|
|
|
19
22
|
DEFAULT_EXPANSION_SEARCH,
|
|
20
23
|
)
|
|
21
24
|
|
|
25
|
+
# When running the tests concurrently in different VMs, one can get access violation errors.
|
|
26
|
+
# To avoid that, we embed the OS name into the temporary files name.
|
|
27
|
+
# https://github.com/unum-cloud/usearch/actions/runs/5705359698/job/15459945738
|
|
28
|
+
temporary_filename = f"tmp-{os.name}"
|
|
29
|
+
temporary_usearch_filename = temporary_filename + ".usearch"
|
|
22
30
|
|
|
23
31
|
dimensions = [3, 97, 256]
|
|
24
32
|
batch_sizes = [1, 77]
|
|
@@ -26,7 +34,7 @@ index_types = [
|
|
|
26
34
|
ScalarKind.F32,
|
|
27
35
|
ScalarKind.F64,
|
|
28
36
|
ScalarKind.F16,
|
|
29
|
-
ScalarKind.
|
|
37
|
+
ScalarKind.I8,
|
|
30
38
|
]
|
|
31
39
|
numpy_types = [np.float32, np.float64, np.float16]
|
|
32
40
|
|
|
@@ -52,10 +60,10 @@ def test_serializing_fbin_matrix(rows: int, cols: int):
|
|
|
52
60
|
:param int cols: The number of columns in the matrix.
|
|
53
61
|
"""
|
|
54
62
|
original = np.random.rand(rows, cols).astype(np.float32)
|
|
55
|
-
save_matrix(original, "
|
|
56
|
-
reconstructed = load_matrix("
|
|
63
|
+
save_matrix(original, temporary_filename + ".fbin")
|
|
64
|
+
reconstructed = load_matrix(temporary_filename + ".fbin")
|
|
57
65
|
assert np.allclose(original, reconstructed)
|
|
58
|
-
os.remove("
|
|
66
|
+
os.remove(temporary_filename + ".fbin")
|
|
59
67
|
|
|
60
68
|
|
|
61
69
|
@pytest.mark.parametrize("rows", batch_sizes)
|
|
@@ -68,10 +76,31 @@ def test_serializing_ibin_matrix(rows: int, cols: int):
|
|
|
68
76
|
:param int cols: The number of columns in the matrix.
|
|
69
77
|
"""
|
|
70
78
|
original = np.random.randint(0, rows + 1, size=(rows, cols)).astype(np.int32)
|
|
71
|
-
save_matrix(original, "
|
|
72
|
-
reconstructed = load_matrix("
|
|
79
|
+
save_matrix(original, temporary_filename + ".ibin")
|
|
80
|
+
reconstructed = load_matrix(temporary_filename + ".ibin")
|
|
73
81
|
assert np.allclose(original, reconstructed)
|
|
74
|
-
os.remove("
|
|
82
|
+
os.remove(temporary_filename + ".ibin")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@pytest.mark.parametrize("rows", batch_sizes)
|
|
86
|
+
@pytest.mark.parametrize("cols", dimensions)
|
|
87
|
+
def test_exact_search(rows: int, cols: int):
|
|
88
|
+
"""
|
|
89
|
+
Test exact search.
|
|
90
|
+
|
|
91
|
+
:param int rows: The number of rows in the matrix.
|
|
92
|
+
:param int cols: The number of columns in the matrix.
|
|
93
|
+
"""
|
|
94
|
+
original = np.random.rand(rows, cols)
|
|
95
|
+
matches: BatchMatches = search(original, original, 10, exact=True)
|
|
96
|
+
top_matches = (
|
|
97
|
+
[int(m.keys[0]) for m in matches] if rows > 1 else int(matches.keys[0])
|
|
98
|
+
)
|
|
99
|
+
assert np.all(top_matches == np.arange(rows))
|
|
100
|
+
|
|
101
|
+
matches: Matches = search(original, original[0], 10, exact=True)
|
|
102
|
+
top_match = int(matches.keys[0])
|
|
103
|
+
assert top_match == 0
|
|
75
104
|
|
|
76
105
|
|
|
77
106
|
@pytest.mark.parametrize("ndim", dimensions)
|
|
@@ -100,23 +129,26 @@ def test_index(
|
|
|
100
129
|
vector = random_vectors(count=1, ndim=ndim, dtype=numpy_type).flatten()
|
|
101
130
|
index.add(42, vector)
|
|
102
131
|
|
|
103
|
-
assert
|
|
104
|
-
assert 42 in index
|
|
105
|
-
assert
|
|
106
|
-
assert index
|
|
107
|
-
assert index[
|
|
132
|
+
assert len(index) == 1, "Size after addition"
|
|
133
|
+
assert 42 in index, "Presence in the index"
|
|
134
|
+
assert 42 in index.keys, "Presence among keys"
|
|
135
|
+
assert 43 not in index, "Presence in the index, false positive"
|
|
136
|
+
assert index[42] is not None, "Vector recovery"
|
|
137
|
+
assert index[43] is None, "Vector recovery, false positive"
|
|
108
138
|
assert len(index[42]) == ndim
|
|
109
139
|
if numpy_type != np.byte:
|
|
110
140
|
assert np.allclose(index[42], vector, atol=0.1)
|
|
111
141
|
|
|
112
|
-
matches
|
|
113
|
-
assert len(
|
|
114
|
-
assert len(matches) ==
|
|
115
|
-
assert len(
|
|
116
|
-
assert
|
|
117
|
-
assert matches[0] ==
|
|
118
|
-
assert
|
|
142
|
+
matches: Matches = index.search(vector, 10)
|
|
143
|
+
assert len(matches.keys) == 1, "Number of matches"
|
|
144
|
+
assert len(matches.keys) == len(matches.distances), "Symmetric match sub-arrays"
|
|
145
|
+
assert len({match.key for match in matches}) == 1, "Iteration over matches"
|
|
146
|
+
assert matches[0].key == 42
|
|
147
|
+
assert matches[0].distance == pytest.approx(0, abs=1e-3)
|
|
148
|
+
assert matches.computed_distances != 0
|
|
149
|
+
assert matches.visited_members != 0
|
|
119
150
|
|
|
151
|
+
# Validating the index structure and metadata:
|
|
120
152
|
assert index.max_level >= 0
|
|
121
153
|
assert index.levels_stats.nodes >= 1
|
|
122
154
|
assert index.level_stats(0).nodes == 1
|
|
@@ -129,30 +161,61 @@ def test_index(
|
|
|
129
161
|
index.remove(43)
|
|
130
162
|
assert len(index) == 1
|
|
131
163
|
|
|
132
|
-
|
|
164
|
+
# Try inserting back
|
|
165
|
+
index.add(43, other_vector)
|
|
166
|
+
assert len(index) == 2
|
|
167
|
+
index.remove(43)
|
|
168
|
+
assert len(index) == 1
|
|
169
|
+
|
|
170
|
+
index.save(temporary_usearch_filename)
|
|
133
171
|
|
|
134
172
|
# Re-populate cleared index
|
|
135
173
|
index.clear()
|
|
136
174
|
assert len(index) == 0
|
|
137
175
|
index.add(42, vector)
|
|
138
176
|
assert len(index) == 1
|
|
139
|
-
matches
|
|
140
|
-
assert
|
|
177
|
+
matches: Matches = index.search(vector, 10)
|
|
178
|
+
assert len(matches) == 1
|
|
179
|
+
|
|
180
|
+
index_copy = index.copy()
|
|
181
|
+
assert len(index_copy) == 1
|
|
182
|
+
assert len(index_copy[42]) == ndim
|
|
183
|
+
matches_copy: Matches = index_copy.search(vector, 10)
|
|
184
|
+
assert np.all(matches_copy.keys == matches.keys)
|
|
141
185
|
|
|
142
|
-
index.load(
|
|
186
|
+
index.load(temporary_usearch_filename)
|
|
143
187
|
assert len(index) == 1
|
|
144
188
|
assert len(index[42]) == ndim
|
|
145
189
|
|
|
146
|
-
|
|
190
|
+
matches_loaded: Matches = index.search(vector, 10)
|
|
191
|
+
assert np.all(matches_loaded.keys == matches.keys)
|
|
192
|
+
|
|
193
|
+
index = Index.restore(temporary_usearch_filename, view=True)
|
|
147
194
|
assert len(index) == 1
|
|
148
195
|
assert len(index[42]) == ndim
|
|
149
196
|
|
|
150
|
-
|
|
151
|
-
assert
|
|
152
|
-
assert len(index_copy[42]) == ndim
|
|
197
|
+
matches_viewed: Matches = index.search(vector, 10)
|
|
198
|
+
assert np.all(matches_viewed.keys == matches.keys)
|
|
153
199
|
|
|
154
200
|
# Cleanup
|
|
155
|
-
|
|
201
|
+
index.reset()
|
|
202
|
+
os.remove(temporary_usearch_filename)
|
|
203
|
+
|
|
204
|
+
# Try opening a missing file
|
|
205
|
+
meta = Index.metadata(temporary_usearch_filename)
|
|
206
|
+
assert meta is None
|
|
207
|
+
index = Index.restore(temporary_usearch_filename)
|
|
208
|
+
assert index is None
|
|
209
|
+
|
|
210
|
+
# Try opening a corrupt file
|
|
211
|
+
with open(temporary_usearch_filename, "w") as file:
|
|
212
|
+
file.write("Some random string")
|
|
213
|
+
meta = Index.metadata(temporary_usearch_filename)
|
|
214
|
+
assert meta is None
|
|
215
|
+
index = Index.restore(temporary_usearch_filename)
|
|
216
|
+
|
|
217
|
+
assert index is None
|
|
218
|
+
os.remove(temporary_usearch_filename)
|
|
156
219
|
|
|
157
220
|
|
|
158
221
|
@pytest.mark.parametrize("ndim", dimensions)
|
|
@@ -169,36 +232,50 @@ def test_index_batch(
|
|
|
169
232
|
):
|
|
170
233
|
index = Index(ndim=ndim, metric=metric, dtype=index_type)
|
|
171
234
|
|
|
172
|
-
|
|
235
|
+
keys = np.arange(batch_size)
|
|
173
236
|
vectors = random_vectors(count=batch_size, ndim=ndim, dtype=numpy_type)
|
|
174
237
|
|
|
175
|
-
index.add(
|
|
238
|
+
index.add(keys, vectors, threads=2)
|
|
176
239
|
assert len(index) == batch_size
|
|
177
|
-
assert np.allclose(index.get_vectors(
|
|
240
|
+
assert np.allclose(index.get_vectors(keys).astype(numpy_type), vectors, atol=0.1)
|
|
178
241
|
|
|
179
|
-
matches:
|
|
180
|
-
assert matches.
|
|
181
|
-
assert matches
|
|
182
|
-
assert np.all(np.sort(index.
|
|
242
|
+
matches: BatchMatches = index.search(vectors, 10, threads=2)
|
|
243
|
+
assert matches.keys.shape[0] == matches.distances.shape[0]
|
|
244
|
+
assert len(matches) == batch_size
|
|
245
|
+
assert np.all(np.sort(index.keys) == np.sort(keys))
|
|
183
246
|
|
|
184
|
-
|
|
247
|
+
if batch_size > 1:
|
|
248
|
+
assert index.max_level >= 1
|
|
249
|
+
else:
|
|
250
|
+
assert index.max_level >= 0
|
|
185
251
|
assert index.levels_stats.nodes >= batch_size
|
|
186
252
|
assert index.level_stats(0).nodes == batch_size
|
|
187
253
|
|
|
188
|
-
index.save(
|
|
254
|
+
index.save(temporary_usearch_filename)
|
|
189
255
|
index.clear()
|
|
190
256
|
assert len(index) == 0
|
|
191
257
|
|
|
192
|
-
index.load(
|
|
258
|
+
index.load(temporary_usearch_filename)
|
|
193
259
|
assert len(index) == batch_size
|
|
194
260
|
assert len(index[0]) == ndim
|
|
195
261
|
|
|
196
|
-
|
|
262
|
+
if batch_size > 1:
|
|
263
|
+
matches_loaded: BatchMatches = index.search(vectors, 10, threads=2)
|
|
264
|
+
for idx in range(len(matches_loaded)):
|
|
265
|
+
assert np.all(matches_loaded[idx].keys == matches[idx].keys)
|
|
266
|
+
|
|
267
|
+
index = Index.restore(temporary_usearch_filename, view=True)
|
|
197
268
|
assert len(index) == batch_size
|
|
198
269
|
assert len(index[0]) == ndim
|
|
199
270
|
|
|
271
|
+
if batch_size > 1:
|
|
272
|
+
matches_viewed: BatchMatches = index.search(vectors, 10, threads=2)
|
|
273
|
+
for idx in range(len(matches_viewed)):
|
|
274
|
+
assert np.all(matches_viewed[idx].keys == matches[idx].keys)
|
|
275
|
+
|
|
200
276
|
# Cleanup
|
|
201
|
-
|
|
277
|
+
index.reset()
|
|
278
|
+
os.remove(temporary_usearch_filename)
|
|
202
279
|
|
|
203
280
|
|
|
204
281
|
@pytest.mark.parametrize("metric", [MetricKind.L2sq])
|
|
@@ -213,33 +290,54 @@ def test_exact_recall(
|
|
|
213
290
|
):
|
|
214
291
|
ndim: int = batch_size
|
|
215
292
|
index = Index(ndim=ndim, metric=metric, dtype=index_type)
|
|
293
|
+
assert index.ndim == ndim
|
|
216
294
|
|
|
217
295
|
vectors = np.zeros((batch_size, ndim), dtype=numpy_type)
|
|
218
296
|
for i in range(batch_size):
|
|
219
297
|
vectors[i, i] = 1
|
|
220
|
-
|
|
221
|
-
index.add(
|
|
298
|
+
keys = np.arange(batch_size)
|
|
299
|
+
index.add(keys, vectors)
|
|
222
300
|
assert len(index) == batch_size
|
|
223
301
|
|
|
224
302
|
# Search one at a time
|
|
225
303
|
for i in range(batch_size):
|
|
226
304
|
matches: Matches = index.search(vectors[i], 10, exact=True)
|
|
227
|
-
found_labels = matches.
|
|
305
|
+
found_labels = matches.keys
|
|
228
306
|
assert found_labels[0] == i
|
|
307
|
+
assert matches.computed_distances == len(index)
|
|
308
|
+
assert matches.visited_members == 0, "Exact search won't traverse the graph"
|
|
229
309
|
|
|
230
310
|
# Search the whole batch
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
assert
|
|
311
|
+
if batch_size > 1:
|
|
312
|
+
matches: BatchMatches = index.search(vectors, 10, exact=True)
|
|
313
|
+
assert matches.computed_distances == len(index) * len(vectors)
|
|
314
|
+
assert matches.visited_members == 0, "Exact search won't traverse the graph"
|
|
235
315
|
|
|
236
|
-
|
|
316
|
+
found_labels = matches.keys
|
|
317
|
+
for i in range(batch_size):
|
|
318
|
+
assert found_labels[i, 0] == i
|
|
319
|
+
|
|
320
|
+
# Match entries against themselves
|
|
237
321
|
index_copy: Index = index.copy()
|
|
238
322
|
mapping: dict = index.join(index_copy, exact=True)
|
|
239
323
|
for man, woman in mapping.items():
|
|
240
324
|
assert man == woman, "Stable marriage failed"
|
|
241
325
|
|
|
242
326
|
|
|
327
|
+
def test_indexes():
|
|
328
|
+
ndim = 10
|
|
329
|
+
index_a = Index(ndim=ndim)
|
|
330
|
+
index_b = Index(ndim=ndim)
|
|
331
|
+
|
|
332
|
+
vectors = random_vectors(count=3, ndim=ndim)
|
|
333
|
+
index_a.add(42, vectors[0])
|
|
334
|
+
index_b.add(43, vectors[1])
|
|
335
|
+
|
|
336
|
+
indexes = Indexes([index_a, index_b])
|
|
337
|
+
matches = indexes.search(vectors[2], 10)
|
|
338
|
+
assert len(matches) == 2
|
|
339
|
+
|
|
340
|
+
|
|
243
341
|
@pytest.mark.parametrize("bits", dimensions)
|
|
244
342
|
@pytest.mark.parametrize("metric", hash_metrics)
|
|
245
343
|
@pytest.mark.parametrize("connectivity", connectivity_options)
|
|
@@ -252,22 +350,15 @@ def test_bitwise_index(
|
|
|
252
350
|
):
|
|
253
351
|
index = Index(ndim=bits, metric=metric, connectivity=connectivity)
|
|
254
352
|
|
|
255
|
-
|
|
353
|
+
keys = np.arange(batch_size)
|
|
256
354
|
byte_vectors = np.random.randint(2, size=(batch_size, bits))
|
|
257
355
|
bit_vectors = np.packbits(byte_vectors, axis=1)
|
|
258
356
|
|
|
259
|
-
index.add(
|
|
260
|
-
assert np.
|
|
261
|
-
assert np.all(index.get_vectors(labels, ScalarKind.B1) == bit_vectors)
|
|
357
|
+
index.add(keys, bit_vectors)
|
|
358
|
+
assert np.all(index.get_vectors(keys, ScalarKind.B1) == bit_vectors)
|
|
262
359
|
|
|
263
360
|
index.search(bit_vectors, 10)
|
|
264
361
|
|
|
265
362
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
def test_sets_index(connectivity: int):
|
|
269
|
-
index = SparseIndex(connectivity=connectivity)
|
|
270
|
-
index.add(10, np.array([10, 12, 15], dtype=np.uint32))
|
|
271
|
-
index.add(11, np.array([11, 12, 15, 16], dtype=np.uint32))
|
|
272
|
-
results = index.search(np.array([12, 15], dtype=np.uint32), 10)
|
|
273
|
-
assert list(results) == [10, 11]
|
|
363
|
+
if __name__ == "__main__":
|
|
364
|
+
pytest.main(args=["python/scripts/test.py", "-s", "-x", "-v"])
|
|
@@ -85,13 +85,12 @@ def test_index_numba(ndim: int, batch_size: int):
|
|
|
85
85
|
)
|
|
86
86
|
index = Index(ndim=ndim, metric=metric)
|
|
87
87
|
|
|
88
|
-
|
|
88
|
+
keys = np.arange(batch_size)
|
|
89
89
|
vectors = random_vectors(count=batch_size, ndim=ndim)
|
|
90
90
|
|
|
91
|
-
index.add(
|
|
92
|
-
matches
|
|
93
|
-
assert matches
|
|
94
|
-
assert count.shape[0] == batch_size
|
|
91
|
+
index.add(keys, vectors)
|
|
92
|
+
matches = index.search(vectors, 10)
|
|
93
|
+
assert len(matches) == batch_size
|
|
95
94
|
|
|
96
95
|
|
|
97
96
|
@pytest.mark.parametrize("ndim", dimensions[-1:])
|
|
@@ -149,13 +148,12 @@ def test_index_cppyy(ndim: int, batch_size: int):
|
|
|
149
148
|
)
|
|
150
149
|
index = Index(ndim=ndim, metric=metric)
|
|
151
150
|
|
|
152
|
-
|
|
151
|
+
keys = np.arange(batch_size)
|
|
153
152
|
vectors = random_vectors(count=batch_size, ndim=ndim)
|
|
154
153
|
|
|
155
|
-
index.add(
|
|
156
|
-
matches
|
|
157
|
-
assert matches
|
|
158
|
-
assert count.shape[0] == batch_size
|
|
154
|
+
index.add(keys, vectors)
|
|
155
|
+
matches = index.search(vectors, 10)
|
|
156
|
+
assert len(matches) == batch_size
|
|
159
157
|
|
|
160
158
|
|
|
161
159
|
@pytest.mark.parametrize("ndim", [8])
|
|
@@ -235,10 +233,10 @@ def test_index_peachpy(ndim: int, batch_size: int):
|
|
|
235
233
|
)
|
|
236
234
|
index = Index(ndim=ndim, metric=metric)
|
|
237
235
|
|
|
238
|
-
|
|
236
|
+
keys = np.arange(batch_size)
|
|
239
237
|
vectors = random_vectors(count=batch_size, ndim=ndim)
|
|
240
238
|
|
|
241
|
-
index.add(
|
|
239
|
+
index.add(keys, vectors)
|
|
242
240
|
matches, distances, count = index.search(vectors, 10)
|
|
243
241
|
assert matches.shape[0] == distances.shape[0]
|
|
244
242
|
assert count.shape[0] == batch_size
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from usearch.index import (
|
|
7
|
+
SparseIndex,
|
|
8
|
+
)
|
|
9
|
+
from usearch.index import (
|
|
10
|
+
DEFAULT_CONNECTIVITY,
|
|
11
|
+
DEFAULT_EXPANSION_ADD,
|
|
12
|
+
DEFAULT_EXPANSION_SEARCH,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
dimensions = [3, 97, 256]
|
|
17
|
+
batch_sizes = [1, 77]
|
|
18
|
+
connectivity_options = [3, 13, 50, DEFAULT_CONNECTIVITY]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.mark.parametrize("connectivity", connectivity_options)
|
|
22
|
+
@pytest.mark.skipif(os.name == "nt", reason="Spurious behaviour on windows")
|
|
23
|
+
def test_sets_index(connectivity: int):
|
|
24
|
+
index = SparseIndex(connectivity=connectivity)
|
|
25
|
+
index.add(10, np.array([10, 12, 15], dtype=np.uint32))
|
|
26
|
+
index.add(11, np.array([11, 12, 15, 16], dtype=np.uint32))
|
|
27
|
+
results = index.search(np.array([12, 15], dtype=np.uint32), 10)
|
|
28
|
+
assert list(results) == [10, 11]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
if __name__ == "__main__":
|
|
32
|
+
pytest.main(args=["python/scripts/test.py", "-s", "-x", "-v"])
|
package/python/usearch/client.py
CHANGED
|
@@ -21,36 +21,33 @@ def _vector_to_ascii(vector: np.ndarray) -> Optional[str]:
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class IndexClient:
|
|
24
|
-
|
|
25
24
|
def __init__(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
port: int = 8545,
|
|
29
|
-
use_http: bool = True) -> None:
|
|
25
|
+
self, uri: str = "127.0.0.1", port: int = 8545, use_http: bool = True
|
|
26
|
+
) -> None:
|
|
30
27
|
self.client = Client(uri=uri, port=port, use_http=use_http)
|
|
31
28
|
|
|
32
|
-
def add_one(self,
|
|
33
|
-
assert isinstance(
|
|
29
|
+
def add_one(self, key: int, vector: np.ndarray):
|
|
30
|
+
assert isinstance(key, int)
|
|
34
31
|
assert isinstance(vector, np.ndarray)
|
|
35
32
|
vector = vector.flatten()
|
|
36
33
|
ascii = _vector_to_ascii(vector)
|
|
37
34
|
if ascii:
|
|
38
|
-
self.client.add_ascii(
|
|
35
|
+
self.client.add_ascii(key=key, string=ascii)
|
|
39
36
|
else:
|
|
40
|
-
self.client.add_one(
|
|
37
|
+
self.client.add_one(key=key, vectors=vector)
|
|
41
38
|
|
|
42
|
-
def add_many(self,
|
|
43
|
-
assert isinstance(
|
|
39
|
+
def add_many(self, keys: np.ndarray, vectors: np.ndarray):
|
|
40
|
+
assert isinstance(keys, int)
|
|
44
41
|
assert isinstance(vectors, np.ndarray)
|
|
45
|
-
assert
|
|
46
|
-
assert
|
|
47
|
-
self.client.add_many(
|
|
42
|
+
assert keys.ndim == 1 and vectors.ndim == 2
|
|
43
|
+
assert keys.shape[0] == vectors.shape[0]
|
|
44
|
+
self.client.add_many(keys=keys, vectors=vectors)
|
|
48
45
|
|
|
49
|
-
def add(self,
|
|
50
|
-
if isinstance(
|
|
51
|
-
return self.add_one(
|
|
46
|
+
def add(self, keys: Union[np.ndarray, int], vectors: np.ndarray):
|
|
47
|
+
if isinstance(keys, int) or len(keys) == 1:
|
|
48
|
+
return self.add_one(keys, vectors)
|
|
52
49
|
else:
|
|
53
|
-
return self.add_many(
|
|
50
|
+
return self.add_many(keys, vectors)
|
|
54
51
|
|
|
55
52
|
def search_one(self, vector: np.ndarray, count: int) -> Matches:
|
|
56
53
|
matches: List[dict] = []
|
|
@@ -64,31 +61,32 @@ class IndexClient:
|
|
|
64
61
|
print(matches.data)
|
|
65
62
|
matches = matches.json
|
|
66
63
|
|
|
67
|
-
|
|
64
|
+
keys = np.array((1, count), dtype=np.uint32)
|
|
68
65
|
distances = np.array((1, count), dtype=np.float32)
|
|
69
66
|
counts = np.array((1), dtype=np.uint32)
|
|
70
67
|
for col, result in enumerate(matches):
|
|
71
|
-
|
|
72
|
-
distances[0, col] = result[
|
|
68
|
+
keys[0, col] = result["key"]
|
|
69
|
+
distances[0, col] = result["distance"]
|
|
73
70
|
counts[0] = len(matches)
|
|
74
71
|
|
|
75
|
-
return
|
|
72
|
+
return keys, distances, counts
|
|
76
73
|
|
|
77
74
|
def search_many(self, vectors: np.ndarray, count: int) -> Matches:
|
|
78
75
|
batch_size: int = vectors.shape[0]
|
|
79
76
|
list_of_matches: List[List[dict]] = self.client.search_many(
|
|
80
|
-
vectors=vectors, count=count
|
|
77
|
+
vectors=vectors, count=count
|
|
78
|
+
)
|
|
81
79
|
|
|
82
|
-
|
|
80
|
+
keys = np.array((batch_size, count), dtype=np.uint32)
|
|
83
81
|
distances = np.array((batch_size, count), dtype=np.float32)
|
|
84
82
|
counts = np.array((batch_size), dtype=np.uint32)
|
|
85
83
|
for row, matches in enumerate(list_of_matches):
|
|
86
84
|
for col, result in enumerate(matches):
|
|
87
|
-
|
|
88
|
-
distances[row, col] = result[
|
|
85
|
+
keys[row, col] = result["key"]
|
|
86
|
+
distances[row, col] = result["distance"]
|
|
89
87
|
counts[row] = len(results)
|
|
90
88
|
|
|
91
|
-
return
|
|
89
|
+
return keys, distances, counts
|
|
92
90
|
|
|
93
91
|
def search(self, vectors: np.ndarray, count: int) -> Matches:
|
|
94
92
|
if vectors.ndim == 1 or (vectors.ndim == 2 and vectors.shape[0] == 1):
|
|
@@ -119,7 +117,7 @@ class IndexClient:
|
|
|
119
117
|
raise NotImplementedError()
|
|
120
118
|
|
|
121
119
|
|
|
122
|
-
if __name__ ==
|
|
120
|
+
if __name__ == "__main__":
|
|
123
121
|
index = IndexClient()
|
|
124
122
|
index.add(42, np.array([0.4] * 256, dtype=np.float32))
|
|
125
123
|
results = index.search(np.array([0.4] * 256, dtype=np.float32), 10)
|