usearch 0.2.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.clang-tidy +60 -0
- package/.github/ISSUE_TEMPLATE/bug_report.yml +90 -0
- package/.github/ISSUE_TEMPLATE/config.yml +5 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +64 -0
- package/.github/workflows/release.yml +47 -20
- package/.github/workflows/update_version.sh +1 -0
- package/.vscode/launch.json +22 -0
- package/.vscode/settings.json +9 -1
- package/.vscode/tasks.json +4 -4
- package/CITATION.cff +11 -0
- package/CMakeLists.txt +108 -95
- package/CODE_OF_CONDUCT.md +128 -0
- package/CONTRIBUTING.md +12 -0
- package/Cargo.toml +1 -1
- package/Dockerfile +14 -0
- package/README.md +21 -12
- package/VERSION +1 -1
- package/binding.gyp +13 -9
- package/build.rs +2 -1
- package/cmake/config.cmake.in +15 -0
- package/cmake/pkg-config.pc.in +4 -0
- package/cmake/usearchConfigVersion.cmake.in +20 -0
- package/conanfile.py +1 -1
- package/docs/{Benchmarking.md → benchmarks.md} +11 -2
- package/docs/benchmarks.rst +5 -0
- package/docs/{Compilation.md → compilation.md} +54 -27
- package/docs/compilation.rst +5 -0
- package/docs/conf.dox +2668 -0
- package/docs/index.rst +7 -0
- package/include/usearch/usearch.hpp +833 -439
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +17 -16
- package/javascript/lib.cpp +94 -20
- package/javascript/test.js +1 -1
- package/objc/USearchObjective.mm +47 -30
- package/objc/include/USearchObjective.h +18 -26
- package/package.json +1 -1
- package/pyproject.toml +3 -2
- package/python/lib.cpp +216 -130
- package/python/{bench.py → scripts/bench.py} +6 -1
- package/python/scripts/bench_params.py +112 -0
- package/python/{test.py → scripts/test.py} +32 -13
- package/python/usearch/client.py +100 -14
- package/python/usearch/server.py +104 -23
- package/rust/lib.cpp +20 -15
- package/rust/lib.hpp +5 -5
- package/rust/lib.rs +3 -3
- package/setup.py +7 -0
- package/src/CMakeLists.txt +107 -0
- package/src/bench.cpp +54 -19
- package/src/{advanced.hpp → punned.hpp} +235 -203
- package/src/test.cpp +62 -0
- package/swift/Index+Sugar.swift +1 -0
- package/swift/Test.swift +2 -1
- package/wolfram/lib.cpp +14 -14
package/python/lib.cpp
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
#include <pybind11/numpy.h>
|
|
19
19
|
#include <pybind11/pybind11.h>
|
|
20
20
|
|
|
21
|
-
#include "
|
|
21
|
+
#include "punned.hpp"
|
|
22
22
|
|
|
23
23
|
using namespace unum::usearch;
|
|
24
24
|
using namespace unum;
|
|
@@ -29,28 +29,55 @@ using py_shape_t = py::array::ShapeContainer;
|
|
|
29
29
|
using label_t = Py_ssize_t;
|
|
30
30
|
using distance_t = punned_distance_t;
|
|
31
31
|
using id_t = std::uint32_t;
|
|
32
|
-
using
|
|
32
|
+
using punned_t = punned_gt<label_t, id_t>;
|
|
33
|
+
|
|
34
|
+
struct punned_py_t : public punned_t {
|
|
35
|
+
using native_t = punned_t;
|
|
36
|
+
using native_t::add;
|
|
37
|
+
using native_t::capacity;
|
|
38
|
+
using native_t::reserve;
|
|
39
|
+
using native_t::search;
|
|
40
|
+
using native_t::size;
|
|
41
|
+
|
|
42
|
+
std::vector<char> ascii_normalization_buffer_;
|
|
43
|
+
|
|
44
|
+
punned_py_t(native_t&& base) : native_t(std::move(base)) {}
|
|
45
|
+
};
|
|
33
46
|
|
|
34
47
|
using set_member_t = std::uint32_t;
|
|
35
48
|
using set_view_t = span_gt<set_member_t const>;
|
|
36
49
|
using sets_index_t = index_gt<jaccard_gt<set_member_t>, label_t, id_t, set_member_t>;
|
|
37
50
|
|
|
51
|
+
struct sets_index_py_t : public sets_index_t {
|
|
52
|
+
using native_t = sets_index_t;
|
|
53
|
+
using native_t::add;
|
|
54
|
+
using native_t::capacity;
|
|
55
|
+
using native_t::reserve;
|
|
56
|
+
using native_t::search;
|
|
57
|
+
using native_t::size;
|
|
58
|
+
|
|
59
|
+
sets_index_py_t(native_t&& base) : native_t(std::move(base)) {}
|
|
60
|
+
};
|
|
61
|
+
|
|
38
62
|
using hash_word_t = std::uint64_t;
|
|
39
63
|
using hash_index_t = index_gt<bit_hamming_gt<hash_word_t>, label_t, id_t, hash_word_t>;
|
|
40
64
|
static constexpr std::size_t bits_per_hash_word_k = sizeof(hash_word_t) * CHAR_BIT;
|
|
41
65
|
|
|
42
|
-
struct
|
|
43
|
-
using hash_index_t
|
|
44
|
-
using
|
|
45
|
-
using
|
|
46
|
-
using
|
|
47
|
-
using
|
|
66
|
+
struct hash_index_py_t : public hash_index_t {
|
|
67
|
+
using native_t = hash_index_t;
|
|
68
|
+
using native_t::add;
|
|
69
|
+
using native_t::capacity;
|
|
70
|
+
using native_t::reserve;
|
|
71
|
+
using native_t::search;
|
|
72
|
+
using native_t::size;
|
|
48
73
|
|
|
49
74
|
std::vector<hash_word_t> buffer_;
|
|
50
75
|
std::size_t words_;
|
|
51
76
|
std::size_t bits_;
|
|
52
77
|
|
|
53
|
-
|
|
78
|
+
hash_index_py_t(native_t&& base) : native_t(std::move(base)) {}
|
|
79
|
+
|
|
80
|
+
hash_index_py_t(config_t config, std::size_t bits) : hash_index_t(config) {
|
|
54
81
|
words_ = divide_round_up<bits_per_hash_word_k>(bits);
|
|
55
82
|
bits_ = words_ * bits_per_hash_word_k;
|
|
56
83
|
buffer_.resize(words_);
|
|
@@ -78,7 +105,7 @@ punned_stateful_metric_t udf(std::size_t metric_uintptr, accuracy_t accuracy) {
|
|
|
78
105
|
}
|
|
79
106
|
}
|
|
80
107
|
|
|
81
|
-
static
|
|
108
|
+
static punned_py_t make_index( //
|
|
82
109
|
std::size_t dimensions, //
|
|
83
110
|
std::size_t capacity, //
|
|
84
111
|
std::string const& scalar_type, //
|
|
@@ -86,8 +113,8 @@ static native_index_t make_index( //
|
|
|
86
113
|
std::size_t connectivity, //
|
|
87
114
|
std::size_t expansion_add, //
|
|
88
115
|
std::size_t expansion_search, //
|
|
89
|
-
std::size_t metric_uintptr
|
|
90
|
-
) {
|
|
116
|
+
std::size_t metric_uintptr, //
|
|
117
|
+
bool tune) {
|
|
91
118
|
|
|
92
119
|
config_t config;
|
|
93
120
|
config.expansion_add = expansion_add;
|
|
@@ -97,18 +124,21 @@ static native_index_t make_index( //
|
|
|
97
124
|
config.max_threads_add = std::thread::hardware_concurrency();
|
|
98
125
|
config.max_threads_search = std::thread::hardware_concurrency();
|
|
99
126
|
|
|
127
|
+
if (tune)
|
|
128
|
+
config = punned_t::optimize(config);
|
|
129
|
+
|
|
100
130
|
accuracy_t accuracy = accuracy_from_name(scalar_type.c_str(), scalar_type.size());
|
|
101
131
|
if (metric_uintptr)
|
|
102
|
-
return
|
|
132
|
+
return punned_t::udf(dimensions, udf(metric_uintptr, accuracy), accuracy, config);
|
|
103
133
|
else
|
|
104
|
-
return index_from_name<
|
|
134
|
+
return index_from_name<punned_t>(metric.c_str(), metric.size(), dimensions, accuracy, config);
|
|
105
135
|
}
|
|
106
136
|
|
|
107
|
-
static std::unique_ptr<
|
|
108
|
-
std::size_t capacity,
|
|
109
|
-
std::size_t connectivity,
|
|
110
|
-
std::size_t expansion_add,
|
|
111
|
-
std::size_t expansion_search
|
|
137
|
+
static std::unique_ptr<sets_index_py_t> make_sets_index( //
|
|
138
|
+
std::size_t capacity, //
|
|
139
|
+
std::size_t connectivity, //
|
|
140
|
+
std::size_t expansion_add, //
|
|
141
|
+
std::size_t expansion_search //
|
|
112
142
|
) {
|
|
113
143
|
config_t config;
|
|
114
144
|
config.expansion_add = expansion_add;
|
|
@@ -118,15 +148,15 @@ static std::unique_ptr<sets_index_t> make_sets_index( //
|
|
|
118
148
|
config.max_threads_add = 1;
|
|
119
149
|
config.max_threads_search = 1;
|
|
120
150
|
|
|
121
|
-
return std::unique_ptr<
|
|
151
|
+
return std::unique_ptr<sets_index_py_t>(new sets_index_py_t(sets_index_t(config)));
|
|
122
152
|
}
|
|
123
153
|
|
|
124
|
-
static std::unique_ptr<
|
|
125
|
-
std::size_t bits,
|
|
126
|
-
std::size_t capacity,
|
|
127
|
-
std::size_t connectivity,
|
|
128
|
-
std::size_t expansion_add,
|
|
129
|
-
std::size_t expansion_search
|
|
154
|
+
static std::unique_ptr<hash_index_py_t> make_hash_index( //
|
|
155
|
+
std::size_t bits, //
|
|
156
|
+
std::size_t capacity, //
|
|
157
|
+
std::size_t connectivity, //
|
|
158
|
+
std::size_t expansion_add, //
|
|
159
|
+
std::size_t expansion_search //
|
|
130
160
|
) {
|
|
131
161
|
config_t config;
|
|
132
162
|
config.expansion_add = expansion_add;
|
|
@@ -136,10 +166,10 @@ static std::unique_ptr<hash_index_w_meta_t> make_hash_index( //
|
|
|
136
166
|
config.max_threads_add = 1;
|
|
137
167
|
config.max_threads_search = 1;
|
|
138
168
|
|
|
139
|
-
return std::unique_ptr<
|
|
169
|
+
return std::unique_ptr<hash_index_py_t>(new hash_index_py_t(config, bits));
|
|
140
170
|
}
|
|
141
171
|
|
|
142
|
-
static void add_one_to_index(
|
|
172
|
+
static void add_one_to_index(punned_py_t& index, label_t label, py::buffer vector, bool copy) {
|
|
143
173
|
|
|
144
174
|
py::buffer_info vector_info = vector.request();
|
|
145
175
|
if (vector_info.ndim != 1)
|
|
@@ -153,19 +183,24 @@ static void add_one_to_index(native_index_t& index, label_t label, py::buffer ve
|
|
|
153
183
|
if (index.size() + 1 >= index.capacity())
|
|
154
184
|
index.reserve(ceil2(index.size() + 1));
|
|
155
185
|
|
|
186
|
+
add_config_t config;
|
|
187
|
+
config.store_vector = copy;
|
|
188
|
+
|
|
156
189
|
// https://docs.python.org/3/library/struct.html#format-characters
|
|
157
|
-
if (vector_info.format == "
|
|
158
|
-
index.add(label, reinterpret_cast<
|
|
190
|
+
if (vector_info.format == "c" || vector_info.format == "b")
|
|
191
|
+
index.add(label, reinterpret_cast<f8_bits_t const*>(vector_data), config);
|
|
192
|
+
else if (vector_info.format == "e")
|
|
193
|
+
index.add(label, reinterpret_cast<f16_bits_t const*>(vector_data), config);
|
|
159
194
|
else if (vector_info.format == "f")
|
|
160
|
-
index.add(label, reinterpret_cast<float const*>(vector_data),
|
|
195
|
+
index.add(label, reinterpret_cast<float const*>(vector_data), config);
|
|
161
196
|
else if (vector_info.format == "d")
|
|
162
|
-
index.add(label, reinterpret_cast<double const*>(vector_data),
|
|
197
|
+
index.add(label, reinterpret_cast<double const*>(vector_data), config);
|
|
163
198
|
else
|
|
164
199
|
throw std::invalid_argument("Incompatible scalars in the vector!");
|
|
165
200
|
}
|
|
166
201
|
|
|
167
|
-
static void add_many_to_index(
|
|
168
|
-
|
|
202
|
+
static void add_many_to_index( //
|
|
203
|
+
punned_py_t& index, py::buffer labels, py::buffer vectors, //
|
|
169
204
|
bool copy, std::size_t threads = 0) {
|
|
170
205
|
|
|
171
206
|
py::buffer_info labels_info = labels.request();
|
|
@@ -196,30 +231,49 @@ static void add_many_to_index( //
|
|
|
196
231
|
char const* labels_data = reinterpret_cast<char const*>(labels_info.ptr);
|
|
197
232
|
|
|
198
233
|
// https://docs.python.org/3/library/struct.html#format-characters
|
|
199
|
-
if (vectors_info.format == "
|
|
200
|
-
multithreaded(index.concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
234
|
+
if (vectors_info.format == "c" || vectors_info.format == "b")
|
|
235
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
236
|
+
add_config_t config;
|
|
237
|
+
config.store_vector = copy;
|
|
238
|
+
config.thread = thread_idx;
|
|
239
|
+
label_t label = *reinterpret_cast<label_t const*>(labels_data + task_idx * labels_info.strides[0]);
|
|
240
|
+
f8_bits_t const* vector =
|
|
241
|
+
reinterpret_cast<f8_bits_t const*>(vectors_data + task_idx * vectors_info.strides[0]);
|
|
242
|
+
index.add(label, vector, config);
|
|
243
|
+
});
|
|
244
|
+
else if (vectors_info.format == "e")
|
|
245
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
246
|
+
add_config_t config;
|
|
247
|
+
config.store_vector = copy;
|
|
248
|
+
config.thread = thread_idx;
|
|
201
249
|
label_t label = *reinterpret_cast<label_t const*>(labels_data + task_idx * labels_info.strides[0]);
|
|
202
250
|
f16_bits_t const* vector =
|
|
203
251
|
reinterpret_cast<f16_bits_t const*>(vectors_data + task_idx * vectors_info.strides[0]);
|
|
204
|
-
index.add(label, vector,
|
|
252
|
+
index.add(label, vector, config);
|
|
205
253
|
});
|
|
206
254
|
else if (vectors_info.format == "f")
|
|
207
|
-
multithreaded(index.concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
255
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
256
|
+
add_config_t config;
|
|
257
|
+
config.store_vector = copy;
|
|
258
|
+
config.thread = thread_idx;
|
|
208
259
|
label_t label = *reinterpret_cast<label_t const*>(labels_data + task_idx * labels_info.strides[0]);
|
|
209
260
|
float const* vector = reinterpret_cast<float const*>(vectors_data + task_idx * vectors_info.strides[0]);
|
|
210
|
-
index.add(label, vector,
|
|
261
|
+
index.add(label, vector, config);
|
|
211
262
|
});
|
|
212
263
|
else if (vectors_info.format == "d")
|
|
213
|
-
multithreaded(index.concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
264
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
265
|
+
add_config_t config;
|
|
266
|
+
config.store_vector = copy;
|
|
267
|
+
config.thread = thread_idx;
|
|
214
268
|
label_t label = *reinterpret_cast<label_t const*>(labels_data + task_idx * labels_info.strides[0]);
|
|
215
269
|
double const* vector = reinterpret_cast<double const*>(vectors_data + task_idx * vectors_info.strides[0]);
|
|
216
|
-
index.add(label, vector,
|
|
270
|
+
index.add(label, vector, config);
|
|
217
271
|
});
|
|
218
272
|
else
|
|
219
273
|
throw std::invalid_argument("Incompatible scalars in the vectors matrix!");
|
|
220
274
|
}
|
|
221
275
|
|
|
222
|
-
static py::tuple search_one_in_index(
|
|
276
|
+
static py::tuple search_one_in_index(punned_py_t& index, py::buffer vector, std::size_t wanted, bool exact) {
|
|
223
277
|
|
|
224
278
|
py::buffer_info vector_info = vector.request();
|
|
225
279
|
Py_ssize_t vector_dimensions = vector_info.shape[0];
|
|
@@ -233,16 +287,26 @@ static py::tuple search_one_in_index(native_index_t& index, py::buffer vector, s
|
|
|
233
287
|
auto labels_py1d = labels_py.mutable_unchecked<1>();
|
|
234
288
|
auto distances_py1d = distances_py.mutable_unchecked<1>();
|
|
235
289
|
|
|
290
|
+
search_config_t config;
|
|
291
|
+
config.exact = exact;
|
|
292
|
+
|
|
236
293
|
// https://docs.python.org/3/library/struct.html#format-characters
|
|
237
|
-
if (vector_info.format == "
|
|
238
|
-
count = index
|
|
239
|
-
|
|
294
|
+
if (vector_info.format == "c" || vector_info.format == "b")
|
|
295
|
+
count = index //
|
|
296
|
+
.search(reinterpret_cast<f8_bits_t const*>(vector_data), wanted, config)
|
|
297
|
+
.dump_to(&labels_py1d(0), &distances_py1d(0));
|
|
298
|
+
else if (vector_info.format == "e")
|
|
299
|
+
count = index //
|
|
300
|
+
.search(reinterpret_cast<f16_bits_t const*>(vector_data), wanted, config)
|
|
301
|
+
.dump_to(&labels_py1d(0), &distances_py1d(0));
|
|
240
302
|
else if (vector_info.format == "f")
|
|
241
|
-
count = index
|
|
242
|
-
|
|
303
|
+
count = index //
|
|
304
|
+
.search(reinterpret_cast<float const*>(vector_data), wanted, config)
|
|
305
|
+
.dump_to(&labels_py1d(0), &distances_py1d(0));
|
|
243
306
|
else if (vector_info.format == "d")
|
|
244
|
-
count = index
|
|
245
|
-
|
|
307
|
+
count = index //
|
|
308
|
+
.search(reinterpret_cast<double const*>(vector_data), wanted, config)
|
|
309
|
+
.dump_to(&labels_py1d(0), &distances_py1d(0));
|
|
246
310
|
else
|
|
247
311
|
throw std::invalid_argument("Incompatible scalars in the query vector!");
|
|
248
312
|
|
|
@@ -265,14 +329,14 @@ static py::tuple search_one_in_index(native_index_t& index, py::buffer vector, s
|
|
|
265
329
|
* 2. matrix of distances,
|
|
266
330
|
* 3. array with match counts.
|
|
267
331
|
*/
|
|
268
|
-
static py::tuple search_many_in_index(
|
|
332
|
+
static py::tuple search_many_in_index(punned_py_t& index, py::buffer vectors, std::size_t wanted, bool exact) {
|
|
269
333
|
|
|
270
334
|
if (wanted == 0)
|
|
271
335
|
return py::tuple(3);
|
|
272
336
|
|
|
273
337
|
py::buffer_info vectors_info = vectors.request();
|
|
274
338
|
if (vectors_info.ndim == 1)
|
|
275
|
-
return search_one_in_index(index, vectors, wanted);
|
|
339
|
+
return search_one_in_index(index, vectors, wanted, exact);
|
|
276
340
|
if (vectors_info.ndim != 2)
|
|
277
341
|
throw std::invalid_argument("Expects a matrix of vectors to add!");
|
|
278
342
|
|
|
@@ -290,23 +354,41 @@ static py::tuple search_many_in_index(native_index_t& index, py::buffer vectors,
|
|
|
290
354
|
auto counts_py1d = counts_py.mutable_unchecked<1>();
|
|
291
355
|
|
|
292
356
|
// https://docs.python.org/3/library/struct.html#format-characters
|
|
293
|
-
if (vectors_info.format == "
|
|
294
|
-
multithreaded(index.concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
357
|
+
if (vectors_info.format == "c" || vectors_info.format == "b")
|
|
358
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
359
|
+
search_config_t config;
|
|
360
|
+
config.thread = thread_idx;
|
|
361
|
+
config.exact = exact;
|
|
362
|
+
f8_bits_t const* vector = (f8_bits_t const*)(vectors_data + task_idx * vectors_info.strides[0]);
|
|
363
|
+
counts_py1d(task_idx) = static_cast<Py_ssize_t>(
|
|
364
|
+
index.search(vector, wanted, config).dump_to(&labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)));
|
|
365
|
+
});
|
|
366
|
+
else if (vectors_info.format == "e")
|
|
367
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
368
|
+
search_config_t config;
|
|
369
|
+
config.thread = thread_idx;
|
|
370
|
+
config.exact = exact;
|
|
295
371
|
f16_bits_t const* vector = (f16_bits_t const*)(vectors_data + task_idx * vectors_info.strides[0]);
|
|
296
372
|
counts_py1d(task_idx) = static_cast<Py_ssize_t>(
|
|
297
|
-
index.search(vector, wanted, &labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)
|
|
373
|
+
index.search(vector, wanted, config).dump_to(&labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)));
|
|
298
374
|
});
|
|
299
375
|
else if (vectors_info.format == "f")
|
|
300
|
-
multithreaded(index.concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
376
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
377
|
+
search_config_t config;
|
|
378
|
+
config.thread = thread_idx;
|
|
379
|
+
config.exact = exact;
|
|
301
380
|
float const* vector = (float const*)(vectors_data + task_idx * vectors_info.strides[0]);
|
|
302
381
|
counts_py1d(task_idx) = static_cast<Py_ssize_t>(
|
|
303
|
-
index.search(vector, wanted, &labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)
|
|
382
|
+
index.search(vector, wanted, config).dump_to(&labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)));
|
|
304
383
|
});
|
|
305
384
|
else if (vectors_info.format == "d")
|
|
306
|
-
multithreaded(index.concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
385
|
+
multithreaded(index.config().concurrency(), vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
|
|
386
|
+
search_config_t config;
|
|
387
|
+
config.thread = thread_idx;
|
|
388
|
+
config.exact = exact;
|
|
307
389
|
double const* vector = (double const*)(vectors_data + task_idx * vectors_info.strides[0]);
|
|
308
390
|
counts_py1d(task_idx) = static_cast<Py_ssize_t>(
|
|
309
|
-
index.search(vector, wanted, &labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)
|
|
391
|
+
index.search(vector, wanted, config).dump_to(&labels_py2d(task_idx, 0), &distances_py2d(task_idx, 0)));
|
|
310
392
|
});
|
|
311
393
|
else
|
|
312
394
|
throw std::invalid_argument("Incompatible scalars in the query matrix!");
|
|
@@ -362,7 +444,7 @@ inline std::uint64_t hash(std::uint64_t v) noexcept {
|
|
|
362
444
|
}
|
|
363
445
|
|
|
364
446
|
template <typename scalar_at>
|
|
365
|
-
inline void hash_typed_buffer(
|
|
447
|
+
inline void hash_typed_buffer(hash_index_py_t& index, py::buffer_info const& vector_info) noexcept {
|
|
366
448
|
char const* vector_data = reinterpret_cast<char const*>(vector_info.ptr);
|
|
367
449
|
Py_ssize_t vector_dimensions = vector_info.shape[0];
|
|
368
450
|
Py_ssize_t vector_stride = vector_info.strides[0];
|
|
@@ -375,7 +457,7 @@ inline void hash_typed_buffer(hash_index_w_meta_t& index, py::buffer_info const&
|
|
|
375
457
|
}
|
|
376
458
|
}
|
|
377
459
|
|
|
378
|
-
void hash_buffer(
|
|
460
|
+
void hash_buffer(hash_index_py_t& index, py::buffer vector) {
|
|
379
461
|
py::buffer_info info = vector.request();
|
|
380
462
|
if (info.ndim != 1)
|
|
381
463
|
throw std::invalid_argument("Array can't be multi-dimensional!");
|
|
@@ -394,18 +476,19 @@ void hash_buffer(hash_index_w_meta_t& index, py::buffer vector) {
|
|
|
394
476
|
PYBIND11_MODULE(index, m) {
|
|
395
477
|
m.doc() = "Unum USearch Python bindings";
|
|
396
478
|
|
|
397
|
-
auto i = py::class_<
|
|
398
|
-
|
|
399
|
-
i.def(py::init(&make_index),
|
|
400
|
-
py::kw_only(),
|
|
401
|
-
py::arg("ndim") = 0,
|
|
402
|
-
py::arg("capacity") = 0,
|
|
403
|
-
py::arg("dtype") = std::string("f32"),
|
|
404
|
-
py::arg("metric") = std::string("ip"),
|
|
405
|
-
py::arg("connectivity") =
|
|
406
|
-
py::arg("expansion_add") =
|
|
407
|
-
py::arg("expansion_search") =
|
|
408
|
-
py::arg("metric_pointer") = 0
|
|
479
|
+
auto i = py::class_<punned_py_t>(m, "Index");
|
|
480
|
+
|
|
481
|
+
i.def(py::init(&make_index), //
|
|
482
|
+
py::kw_only(), //
|
|
483
|
+
py::arg("ndim") = 0, //
|
|
484
|
+
py::arg("capacity") = 0, //
|
|
485
|
+
py::arg("dtype") = std::string("f32"), //
|
|
486
|
+
py::arg("metric") = std::string("ip"), //
|
|
487
|
+
py::arg("connectivity") = default_connectivity(), //
|
|
488
|
+
py::arg("expansion_add") = default_expansion_add(), //
|
|
489
|
+
py::arg("expansion_search") = default_expansion_search(), //
|
|
490
|
+
py::arg("metric_pointer") = 0, //
|
|
491
|
+
py::arg("tune") = false //
|
|
409
492
|
);
|
|
410
493
|
|
|
411
494
|
i.def( //
|
|
@@ -428,42 +511,45 @@ PYBIND11_MODULE(index, m) {
|
|
|
428
511
|
i.def( //
|
|
429
512
|
"search", &search_many_in_index, //
|
|
430
513
|
py::arg("query"), //
|
|
431
|
-
py::arg("count") = 10
|
|
514
|
+
py::arg("count") = 10, //
|
|
515
|
+
py::arg("exact") = false //
|
|
432
516
|
);
|
|
433
517
|
|
|
434
|
-
i.def("__len__", &
|
|
435
|
-
i.def_property_readonly("size", &
|
|
436
|
-
i.def_property_readonly("ndim", &
|
|
437
|
-
i.def_property_readonly("connectivity", &
|
|
438
|
-
i.def_property_readonly("capacity", &
|
|
518
|
+
i.def("__len__", &punned_py_t::size);
|
|
519
|
+
i.def_property_readonly("size", &punned_py_t::size);
|
|
520
|
+
i.def_property_readonly("ndim", &punned_py_t::dimensions);
|
|
521
|
+
i.def_property_readonly("connectivity", &punned_py_t::connectivity);
|
|
522
|
+
i.def_property_readonly("capacity", &punned_py_t::capacity);
|
|
439
523
|
i.def_property_readonly( //
|
|
440
|
-
"dtype", [](
|
|
441
|
-
|
|
442
|
-
i.def("save", &save_index<
|
|
443
|
-
i.def("load", &load_index<
|
|
444
|
-
i.def("view", &view_index<
|
|
445
|
-
i.def("clear", &clear_index<
|
|
446
|
-
|
|
447
|
-
auto si = py::class_<
|
|
448
|
-
|
|
449
|
-
si.def(
|
|
450
|
-
py::init(&make_sets_index),
|
|
451
|
-
py::kw_only(),
|
|
452
|
-
py::arg("capacity") = 0,
|
|
453
|
-
py::arg("connectivity") =
|
|
454
|
-
py::arg("expansion_add") =
|
|
455
|
-
py::arg("expansion_search") =
|
|
524
|
+
"dtype", [](punned_py_t const& index) -> std::string { return accuracy_name(index.accuracy()); });
|
|
525
|
+
|
|
526
|
+
i.def("save", &save_index<punned_py_t>, py::arg("path"));
|
|
527
|
+
i.def("load", &load_index<punned_py_t>, py::arg("path"));
|
|
528
|
+
i.def("view", &view_index<punned_py_t>, py::arg("path"));
|
|
529
|
+
i.def("clear", &clear_index<punned_py_t>);
|
|
530
|
+
|
|
531
|
+
auto si = py::class_<sets_index_py_t>(m, "SetsIndex");
|
|
532
|
+
|
|
533
|
+
si.def( //
|
|
534
|
+
py::init(&make_sets_index), //
|
|
535
|
+
py::kw_only(), //
|
|
536
|
+
py::arg("capacity") = 0, //
|
|
537
|
+
py::arg("connectivity") = default_connectivity(), //
|
|
538
|
+
py::arg("expansion_add") = default_expansion_add(), //
|
|
539
|
+
py::arg("expansion_search") = default_expansion_search() //
|
|
456
540
|
);
|
|
457
541
|
|
|
458
542
|
si.def( //
|
|
459
543
|
"add",
|
|
460
|
-
[](
|
|
544
|
+
[](sets_index_py_t& index, label_t label, py::array_t<set_member_t> set, bool copy) {
|
|
461
545
|
validate_set(set);
|
|
462
546
|
if (index.size() + 1 >= index.capacity())
|
|
463
547
|
index.reserve(ceil2(index.size() + 1));
|
|
464
548
|
auto proxy = set.unchecked<1>();
|
|
465
549
|
auto view = set_view_t{proxy.data(0), static_cast<std::size_t>(proxy.shape(0))};
|
|
466
|
-
|
|
550
|
+
add_config_t config;
|
|
551
|
+
config.store_vector = copy;
|
|
552
|
+
index.add(label, view, config);
|
|
467
553
|
}, //
|
|
468
554
|
py::arg("label"), //
|
|
469
555
|
py::arg("set"), //
|
|
@@ -473,13 +559,13 @@ PYBIND11_MODULE(index, m) {
|
|
|
473
559
|
|
|
474
560
|
si.def( //
|
|
475
561
|
"search",
|
|
476
|
-
[](
|
|
562
|
+
[](sets_index_py_t& index, py::array_t<set_member_t> set, std::size_t count) -> py::array_t<label_t> {
|
|
477
563
|
validate_set(set);
|
|
478
564
|
auto proxy = set.unchecked<1>();
|
|
479
565
|
auto view = set_view_t{proxy.data(0), static_cast<std::size_t>(proxy.shape(0))};
|
|
480
566
|
auto labels_py = py::array_t<label_t>(py_shape_t{static_cast<Py_ssize_t>(count)});
|
|
481
567
|
auto labels_proxy = labels_py.mutable_unchecked<1>();
|
|
482
|
-
auto found = index.search(view, count
|
|
568
|
+
auto found = index.search(view, count).dump_to(&labels_proxy(0), nullptr);
|
|
483
569
|
labels_py.resize(py_shape_t{static_cast<Py_ssize_t>(found)});
|
|
484
570
|
return labels_py;
|
|
485
571
|
},
|
|
@@ -487,35 +573,35 @@ PYBIND11_MODULE(index, m) {
|
|
|
487
573
|
py::arg("count") = 10 //
|
|
488
574
|
);
|
|
489
575
|
|
|
490
|
-
si.def("__len__", &
|
|
491
|
-
si.def_property_readonly("size", &
|
|
492
|
-
si.def_property_readonly("connectivity", &
|
|
493
|
-
si.def_property_readonly("capacity", &
|
|
494
|
-
|
|
495
|
-
si.def("save", &save_index<
|
|
496
|
-
si.def("load", &load_index<
|
|
497
|
-
si.def("view", &view_index<
|
|
498
|
-
si.def("clear", &clear_index<
|
|
499
|
-
|
|
500
|
-
auto hi = py::class_<
|
|
501
|
-
|
|
502
|
-
hi.def(
|
|
503
|
-
py::init(&make_hash_index),
|
|
504
|
-
py::kw_only(),
|
|
505
|
-
py::arg("bits"),
|
|
506
|
-
py::arg("capacity") = 0,
|
|
507
|
-
py::arg("connectivity") =
|
|
508
|
-
py::arg("expansion_add") =
|
|
509
|
-
py::arg("expansion_search") =
|
|
576
|
+
si.def("__len__", &sets_index_py_t::size);
|
|
577
|
+
si.def_property_readonly("size", &sets_index_py_t::size);
|
|
578
|
+
si.def_property_readonly("connectivity", &sets_index_py_t::connectivity);
|
|
579
|
+
si.def_property_readonly("capacity", &sets_index_py_t::capacity);
|
|
580
|
+
|
|
581
|
+
si.def("save", &save_index<sets_index_py_t>, py::arg("path"));
|
|
582
|
+
si.def("load", &load_index<sets_index_py_t>, py::arg("path"));
|
|
583
|
+
si.def("view", &view_index<sets_index_py_t>, py::arg("path"));
|
|
584
|
+
si.def("clear", &clear_index<sets_index_py_t>);
|
|
585
|
+
|
|
586
|
+
auto hi = py::class_<hash_index_py_t>(m, "HashIndex");
|
|
587
|
+
|
|
588
|
+
hi.def( //
|
|
589
|
+
py::init(&make_hash_index), //
|
|
590
|
+
py::kw_only(), //
|
|
591
|
+
py::arg("bits"), //
|
|
592
|
+
py::arg("capacity") = 0, //
|
|
593
|
+
py::arg("connectivity") = default_connectivity(), //
|
|
594
|
+
py::arg("expansion_add") = default_expansion_add(), //
|
|
595
|
+
py::arg("expansion_search") = default_expansion_search() //
|
|
510
596
|
);
|
|
511
597
|
|
|
512
598
|
hi.def( //
|
|
513
599
|
"add",
|
|
514
|
-
[](
|
|
600
|
+
[](hash_index_py_t& index, label_t label, py::buffer array) {
|
|
515
601
|
if (index.size() + 1 >= index.capacity())
|
|
516
602
|
index.reserve(ceil2(index.size() + 1));
|
|
517
603
|
hash_buffer(index, array);
|
|
518
|
-
index.add(label, index.buffer()
|
|
604
|
+
index.add(label, index.buffer());
|
|
519
605
|
}, //
|
|
520
606
|
py::arg("label"), //
|
|
521
607
|
py::arg("array") //
|
|
@@ -523,13 +609,13 @@ PYBIND11_MODULE(index, m) {
|
|
|
523
609
|
|
|
524
610
|
hi.def( //
|
|
525
611
|
"search",
|
|
526
|
-
[](
|
|
612
|
+
[](hash_index_py_t& index, py::buffer array, std::size_t count) -> py::array_t<label_t> {
|
|
527
613
|
if (index.size() + 1 >= index.capacity())
|
|
528
614
|
index.reserve(ceil2(index.size() + 1));
|
|
529
615
|
hash_buffer(index, array);
|
|
530
616
|
auto labels_py = py::array_t<label_t>(py_shape_t{static_cast<Py_ssize_t>(count)});
|
|
531
617
|
auto labels_proxy = labels_py.mutable_unchecked<1>();
|
|
532
|
-
auto found = index.search(index.buffer(), count
|
|
618
|
+
auto found = index.search(index.buffer(), count).dump_to(&labels_proxy(0), nullptr);
|
|
533
619
|
labels_py.resize(py_shape_t{static_cast<Py_ssize_t>(found)});
|
|
534
620
|
return labels_py;
|
|
535
621
|
},
|
|
@@ -537,13 +623,13 @@ PYBIND11_MODULE(index, m) {
|
|
|
537
623
|
py::arg("count") = 10 //
|
|
538
624
|
);
|
|
539
625
|
|
|
540
|
-
hi.def("__len__", &
|
|
541
|
-
hi.def_property_readonly("size", &
|
|
542
|
-
hi.def_property_readonly("connectivity", &
|
|
543
|
-
hi.def_property_readonly("capacity", &
|
|
626
|
+
hi.def("__len__", &hash_index_py_t::size);
|
|
627
|
+
hi.def_property_readonly("size", &hash_index_py_t::size);
|
|
628
|
+
hi.def_property_readonly("connectivity", &hash_index_py_t::connectivity);
|
|
629
|
+
hi.def_property_readonly("capacity", &hash_index_py_t::capacity);
|
|
544
630
|
|
|
545
|
-
hi.def("save", &save_index<
|
|
546
|
-
hi.def("load", &load_index<
|
|
547
|
-
hi.def("view", &view_index<
|
|
548
|
-
hi.def("clear", &clear_index<
|
|
631
|
+
hi.def("save", &save_index<hash_index_py_t>, py::arg("path"));
|
|
632
|
+
hi.def("load", &load_index<hash_index_py_t>, py::arg("path"));
|
|
633
|
+
hi.def("view", &view_index<hash_index_py_t>, py::arg("path"));
|
|
634
|
+
hi.def("clear", &clear_index<hash_index_py_t>);
|
|
549
635
|
}
|
|
@@ -5,7 +5,6 @@ import fire
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
from numba import cfunc, types, carray
|
|
7
7
|
|
|
8
|
-
from faiss import IndexHNSWFlat, IndexIVFPQ
|
|
9
8
|
from usearch.index import Index
|
|
10
9
|
from usearch.io import load_matrix
|
|
11
10
|
|
|
@@ -125,6 +124,12 @@ def main(
|
|
|
125
124
|
)
|
|
126
125
|
bench_usearch(index, vectors_mat, queries_mat, neighbors_mat)
|
|
127
126
|
|
|
127
|
+
# Don't depend on the FAISS installation for benchmarks
|
|
128
|
+
try:
|
|
129
|
+
from faiss import IndexHNSWFlat, IndexIVFPQ
|
|
130
|
+
except ImportError:
|
|
131
|
+
return
|
|
132
|
+
|
|
128
133
|
print('FAISS: HNSW')
|
|
129
134
|
index = IndexHNSWFlat(dim, connectivity)
|
|
130
135
|
index.hnsw.efSearch = expansion_search
|