usearch 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release.yml +56 -4
- package/.github/workflows/update_version.sh +6 -1
- package/.vscode/launch.json +21 -22
- package/.vscode/settings.json +7 -1
- package/CITATION.cff +4 -4
- package/CMakeLists.txt +1 -1
- package/Cargo.toml +1 -1
- package/Package.swift +0 -1
- package/README.md +100 -27
- package/binding.gyp +0 -1
- package/build.gradle +1 -1
- package/build.rs +3 -3
- package/c/lib.cpp +32 -9
- package/c/usearch.h +67 -36
- package/conanfile.py +1 -1
- package/{src → cpp}/bench.cpp +52 -36
- package/{src → cpp}/test.cpp +17 -7
- package/docs/benchmarks.md +33 -10
- package/docs/c/index.rst +11 -0
- package/docs/c/reference.rst +7 -0
- package/docs/compilation.md +24 -9
- package/docs/conf.dox +6 -6
- package/docs/conf.py +3 -1
- package/docs/cpp/index.rst +11 -0
- package/docs/cpp/reference.rst +7 -0
- package/docs/index.rst +10 -0
- package/docs/java/index.rst +11 -0
- package/docs/java/reference.rst +7 -0
- package/docs/javascript/index.rst +11 -0
- package/docs/javascript/reference.rst +9 -0
- package/docs/python/index.rst +11 -0
- package/docs/python/reference.rst +35 -0
- package/golang/lib.go +1 -1
- package/include/usearch/{usearch.hpp → index.hpp} +152 -148
- package/include/usearch/index_punned_dense.hpp +633 -0
- package/include/usearch/index_punned_helpers.hpp +528 -0
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +8 -7
- package/javascript/lib.cpp +26 -50
- package/javascript/usearch.js +117 -2
- package/objc/USearchObjective.mm +145 -49
- package/objc/include/USearchObjective.h +106 -30
- package/package.json +2 -2
- package/python/lib.cpp +134 -331
- package/python/scripts/bench.ipynb +170 -0
- package/python/scripts/bench.py +87 -175
- package/python/scripts/index_faiss.py +100 -0
- package/python/scripts/test.py +41 -37
- package/python/usearch/eval.py +353 -0
- package/python/usearch/index.py +117 -113
- package/python/usearch/io.py +7 -2
- package/python/usearch/join.py +100 -0
- package/python/usearch/numba.py +101 -0
- package/rust/lib.cpp +27 -14
- package/rust/lib.hpp +4 -10
- package/rust/lib.rs +50 -17
- package/setup.py +12 -2
- package/simsimd/include/simsimd/simsimd.h +32 -3
- package/swift/Index+Sugar.swift +55 -29
- package/swift/Test.swift +1 -1
- package/wolfram/lib.cpp +4 -4
- package/python/scripts/bench_params.py +0 -201
- package/python/usearch/synthetic.py +0 -31
- package/src/punned.hpp +0 -906
- /package/{src → cpp}/CMakeLists.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @file
|
|
2
|
+
* @file index.hpp
|
|
3
3
|
* @author Ashot Vardanian
|
|
4
4
|
* @brief Single-header Vector Search.
|
|
5
5
|
* @date 2023-04-26
|
|
@@ -9,17 +9,9 @@
|
|
|
9
9
|
#ifndef UNUM_USEARCH_HPP
|
|
10
10
|
#define UNUM_USEARCH_HPP
|
|
11
11
|
|
|
12
|
-
#if !defined(USEARCH_VERSION_MAJOR)
|
|
13
12
|
#define USEARCH_VERSION_MAJOR 0
|
|
14
|
-
#endif
|
|
15
|
-
|
|
16
|
-
#if !defined(USEARCH_VERSION_MINOR)
|
|
17
13
|
#define USEARCH_VERSION_MINOR 0
|
|
18
|
-
#endif
|
|
19
|
-
|
|
20
|
-
#if !defined(USEARCH_VERSION_PATCH)
|
|
21
14
|
#define USEARCH_VERSION_PATCH 0
|
|
22
|
-
#endif
|
|
23
15
|
|
|
24
16
|
// Inferring C++ version
|
|
25
17
|
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
@@ -73,8 +65,8 @@
|
|
|
73
65
|
#include <mutex> // `std::unique_lock` - replacement candidate
|
|
74
66
|
#include <random> // `std::default_random_engine` - replacement candidate
|
|
75
67
|
#include <stdexcept> // `std::runtime_exception`
|
|
68
|
+
#include <thread> // `std::thread`
|
|
76
69
|
#include <utility> // `std::pair`
|
|
77
|
-
#include <vector> // `std::vector`
|
|
78
70
|
|
|
79
71
|
// Prefetching
|
|
80
72
|
#if defined(USEARCH_IS_GCC)
|
|
@@ -205,15 +197,33 @@ template <typename at> class misaligned_ptr_gt {
|
|
|
205
197
|
bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
|
|
206
198
|
};
|
|
207
199
|
|
|
200
|
+
/**
|
|
201
|
+
* @brief Non-owning memory range view, similar to `std::span`, but for C++11.
|
|
202
|
+
*/
|
|
203
|
+
template <typename scalar_at> class span_gt {
|
|
204
|
+
scalar_at* data_;
|
|
205
|
+
std::size_t size_;
|
|
206
|
+
|
|
207
|
+
public:
|
|
208
|
+
span_gt(scalar_at* begin, scalar_at const* end) noexcept : data_(begin), size_(end - begin) {}
|
|
209
|
+
span_gt(scalar_at* begin, std::size_t size) noexcept : data_(begin), size_(size) {}
|
|
210
|
+
scalar_at* data() const noexcept { return data_; }
|
|
211
|
+
std::size_t size() const noexcept { return size_; }
|
|
212
|
+
operator scalar_at*() const noexcept { return data(); }
|
|
213
|
+
};
|
|
214
|
+
|
|
208
215
|
/**
|
|
209
216
|
* @brief Inner (Dot) Product distance.
|
|
210
217
|
*/
|
|
211
|
-
template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
|
|
218
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct ip_gt {
|
|
212
219
|
using scalar_t = scalar_at;
|
|
220
|
+
using view_t = span_gt<scalar_t const>;
|
|
213
221
|
using result_t = result_at;
|
|
214
222
|
using result_type = result_t;
|
|
215
223
|
|
|
216
|
-
inline result_t operator()(
|
|
224
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
225
|
+
|
|
226
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
217
227
|
result_type ab{};
|
|
218
228
|
#if defined(USEARCH_USE_OPENMP)
|
|
219
229
|
#pragma omp simd reduction(+ : ab)
|
|
@@ -234,12 +244,15 @@ template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
|
|
|
234
244
|
* Unless you are running on an tiny embedded platform, this metric
|
|
235
245
|
* is recommended over `::ip_gt` for low-precision scalars.
|
|
236
246
|
*/
|
|
237
|
-
template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
|
|
247
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct cos_gt {
|
|
238
248
|
using scalar_t = scalar_at;
|
|
249
|
+
using view_t = span_gt<scalar_t const>;
|
|
239
250
|
using result_t = result_at;
|
|
240
251
|
using result_type = result_t;
|
|
241
252
|
|
|
242
|
-
inline result_t operator()(
|
|
253
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
254
|
+
|
|
255
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
243
256
|
result_t ab{}, a2{}, b2{};
|
|
244
257
|
#if defined(USEARCH_USE_OPENMP)
|
|
245
258
|
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
@@ -260,12 +273,15 @@ template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
|
|
|
260
273
|
* @brief Squared Euclidean (L2) distance.
|
|
261
274
|
* Square root is avoided at the end, as it won't affect the ordering.
|
|
262
275
|
*/
|
|
263
|
-
template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
|
|
276
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct l2sq_gt {
|
|
264
277
|
using scalar_t = scalar_at;
|
|
278
|
+
using view_t = span_gt<scalar_t const>;
|
|
265
279
|
using result_t = result_at;
|
|
266
280
|
using result_type = result_t;
|
|
267
281
|
|
|
268
|
-
inline result_t operator()(
|
|
282
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
283
|
+
|
|
284
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
269
285
|
result_t ab_deltas_sq{};
|
|
270
286
|
#if defined(USEARCH_USE_OPENMP)
|
|
271
287
|
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
@@ -280,43 +296,21 @@ template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
|
|
|
280
296
|
}
|
|
281
297
|
};
|
|
282
298
|
|
|
283
|
-
/**
|
|
284
|
-
* @brief Hamming distance computes the number of differing elements in
|
|
285
|
-
* two arrays. An example would be a chess board.
|
|
286
|
-
*/
|
|
287
|
-
template <typename scalar_at, typename result_at = std::size_t> struct hamming_gt {
|
|
288
|
-
using scalar_t = scalar_at;
|
|
289
|
-
using result_t = result_at;
|
|
290
|
-
using result_type = result_t;
|
|
291
|
-
|
|
292
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
|
|
293
|
-
result_t matches{};
|
|
294
|
-
#if defined(USEARCH_USE_OPENMP)
|
|
295
|
-
#pragma omp simd reduction(+ : matches)
|
|
296
|
-
#elif defined(USEARCH_IS_CLANG)
|
|
297
|
-
#pragma clang loop vectorize(enable)
|
|
298
|
-
#elif defined(USEARCH_IS_GCC)
|
|
299
|
-
#pragma GCC ivdep
|
|
300
|
-
#endif
|
|
301
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
302
|
-
matches += a[i] != b[i];
|
|
303
|
-
return matches;
|
|
304
|
-
}
|
|
305
|
-
};
|
|
306
|
-
|
|
307
299
|
/**
|
|
308
300
|
* @brief Hamming distance computes the number of differing bits in
|
|
309
301
|
* two arrays of integers. An example would be a textual document,
|
|
310
302
|
* tokenized and hashed into a fixed-capacity bitset.
|
|
311
303
|
*/
|
|
312
|
-
template <typename scalar_at, typename result_at = std::size_t> struct
|
|
304
|
+
template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct hamming_gt {
|
|
313
305
|
using scalar_t = scalar_at;
|
|
306
|
+
using view_t = span_gt<scalar_t const>;
|
|
314
307
|
using result_t = result_at;
|
|
315
308
|
using result_type = result_t;
|
|
316
309
|
static_assert(std::is_unsigned<scalar_t>::value, "Hamming distance requires unsigned integral words");
|
|
317
310
|
|
|
318
|
-
inline result_t operator()(
|
|
319
|
-
|
|
311
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
312
|
+
|
|
313
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
320
314
|
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
321
315
|
result_t matches{};
|
|
322
316
|
#if defined(USEARCH_USE_OPENMP)
|
|
@@ -336,20 +330,22 @@ template <typename scalar_at, typename result_at = std::size_t> struct bitwise_h
|
|
|
336
330
|
* @brief Tanimoto distance is the intersection over bitwise union.
|
|
337
331
|
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
338
332
|
*/
|
|
339
|
-
template <typename scalar_at, typename result_at = float> struct
|
|
333
|
+
template <typename scalar_at = std::uint64_t, typename result_at = float> struct tanimoto_gt {
|
|
340
334
|
using scalar_t = scalar_at;
|
|
335
|
+
using view_t = span_gt<scalar_t const>;
|
|
341
336
|
using result_t = result_at;
|
|
342
337
|
using result_type = result_t;
|
|
343
338
|
static_assert(std::is_unsigned<scalar_t>::value, "Tanimoto distance requires unsigned integral words");
|
|
344
339
|
static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
|
|
345
340
|
|
|
346
|
-
inline result_t operator()(
|
|
347
|
-
|
|
341
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
342
|
+
|
|
343
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
348
344
|
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
349
345
|
result_t and_count{};
|
|
350
346
|
result_t or_count{};
|
|
351
347
|
#if defined(USEARCH_USE_OPENMP)
|
|
352
|
-
#pragma omp simd reduction(+ :
|
|
348
|
+
#pragma omp simd reduction(+ : and_count, or_count)
|
|
353
349
|
#elif defined(USEARCH_IS_CLANG)
|
|
354
350
|
#pragma clang loop vectorize(enable)
|
|
355
351
|
#elif defined(USEARCH_IS_GCC)
|
|
@@ -366,20 +362,22 @@ template <typename scalar_at, typename result_at = float> struct bitwise_tanimot
|
|
|
366
362
|
* @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
|
|
367
363
|
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
368
364
|
*/
|
|
369
|
-
template <typename scalar_at, typename result_at = float> struct
|
|
365
|
+
template <typename scalar_at = std::uint64_t, typename result_at = float> struct sorensen_gt {
|
|
370
366
|
using scalar_t = scalar_at;
|
|
367
|
+
using view_t = span_gt<scalar_t const>;
|
|
371
368
|
using result_t = result_at;
|
|
372
369
|
using result_type = result_t;
|
|
373
370
|
static_assert(std::is_unsigned<scalar_t>::value, "Sorensen-Dice distance requires unsigned integral words");
|
|
374
371
|
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
375
372
|
|
|
376
|
-
inline result_t operator()(
|
|
377
|
-
|
|
373
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
374
|
+
|
|
375
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
378
376
|
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
379
377
|
result_t and_count{};
|
|
380
378
|
result_t any_count{};
|
|
381
379
|
#if defined(USEARCH_USE_OPENMP)
|
|
382
|
-
#pragma omp simd reduction(+ :
|
|
380
|
+
#pragma omp simd reduction(+ : and_count, any_count)
|
|
383
381
|
#elif defined(USEARCH_IS_CLANG)
|
|
384
382
|
#pragma clang loop vectorize(enable)
|
|
385
383
|
#elif defined(USEARCH_IS_GCC)
|
|
@@ -396,13 +394,19 @@ template <typename scalar_at, typename result_at = float> struct bitwise_sorense
|
|
|
396
394
|
* @brief Counts the number of matching elements in two unique sorted sets.
|
|
397
395
|
* Can be used to compute the similarity between two textual documents
|
|
398
396
|
* using the IDs of tokens present in them.
|
|
397
|
+
* Similar to `tanimoto_gt` for dense representations.
|
|
399
398
|
*/
|
|
400
|
-
template <typename scalar_at, typename result_at = float> struct jaccard_gt {
|
|
399
|
+
template <typename scalar_at = std::int32_t, typename result_at = float> struct jaccard_gt {
|
|
401
400
|
using scalar_t = scalar_at;
|
|
401
|
+
using view_t = span_gt<scalar_t const>;
|
|
402
402
|
using result_t = result_at;
|
|
403
403
|
using result_type = result_t;
|
|
404
404
|
static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
|
|
405
405
|
|
|
406
|
+
inline result_t operator()(view_t a, view_t b) const noexcept {
|
|
407
|
+
return operator()(a.data(), b.data(), a.size(), b.size());
|
|
408
|
+
}
|
|
409
|
+
|
|
406
410
|
inline result_t operator()( //
|
|
407
411
|
scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
|
|
408
412
|
result_t intersection{};
|
|
@@ -422,13 +426,15 @@ template <typename scalar_at, typename result_at = float> struct jaccard_gt {
|
|
|
422
426
|
* Can be used to compute the similarity between two textual documents
|
|
423
427
|
* using the IDs of tokens present in them.
|
|
424
428
|
*/
|
|
425
|
-
template <typename scalar_at, typename result_at = float> struct pearson_correlation_gt {
|
|
429
|
+
template <typename scalar_at = float, typename result_at = float> struct pearson_correlation_gt {
|
|
426
430
|
using scalar_t = scalar_at;
|
|
431
|
+
using view_t = span_gt<scalar_t const>;
|
|
427
432
|
using result_t = result_at;
|
|
428
433
|
using result_type = result_t;
|
|
429
434
|
|
|
430
|
-
inline result_t operator()(
|
|
431
|
-
|
|
435
|
+
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
436
|
+
|
|
437
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
432
438
|
result_t a_sum{}, b_sum{}, ab_sum{};
|
|
433
439
|
result_t a_sq_sum{}, b_sq_sum{};
|
|
434
440
|
#if defined(USEARCH_USE_OPENMP)
|
|
@@ -439,11 +445,11 @@ template <typename scalar_at, typename result_at = float> struct pearson_correla
|
|
|
439
445
|
#pragma GCC ivdep
|
|
440
446
|
#endif
|
|
441
447
|
for (std::size_t i = 0; i != dim; ++i) {
|
|
442
|
-
a_sum += a[i];
|
|
443
|
-
b_sum += b[i];
|
|
444
|
-
ab_sum += a[i] * b[i];
|
|
445
|
-
a_sq_sum += a[i] * a[i];
|
|
446
|
-
b_sq_sum += b[i] * b[i];
|
|
448
|
+
a_sum += result_t(a[i]);
|
|
449
|
+
b_sum += result_t(b[i]);
|
|
450
|
+
ab_sum += result_t(a[i]) * result_t(b[i]);
|
|
451
|
+
a_sq_sum += result_t(a[i]) * result_t(a[i]);
|
|
452
|
+
b_sq_sum += result_t(b[i]) * result_t(b[i]);
|
|
447
453
|
}
|
|
448
454
|
result_t denom = std::sqrt((dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum));
|
|
449
455
|
result_t corr = (dim * ab_sum - a_sum * b_sum) / denom;
|
|
@@ -455,13 +461,14 @@ template <typename scalar_at, typename result_at = float> struct pearson_correla
|
|
|
455
461
|
* @brief Haversine distance for the shortest distance between two nodes on
|
|
456
462
|
* the surface of a 3D sphere, defined with latitude and longitude.
|
|
457
463
|
*/
|
|
458
|
-
template <typename scalar_at, typename result_at = scalar_at> struct haversine_gt {
|
|
464
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct haversine_gt {
|
|
459
465
|
using scalar_t = scalar_at;
|
|
466
|
+
using view_t = span_gt<scalar_t const>;
|
|
460
467
|
using result_t = result_at;
|
|
461
468
|
using result_type = result_t;
|
|
462
469
|
static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
|
|
463
470
|
|
|
464
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b
|
|
471
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b) const noexcept {
|
|
465
472
|
result_t lat_a = a[0], lon_a = a[1];
|
|
466
473
|
result_t lat_b = b[0], lon_b = b[1];
|
|
467
474
|
|
|
@@ -494,6 +501,7 @@ class error_t {
|
|
|
494
501
|
return *this;
|
|
495
502
|
}
|
|
496
503
|
explicit operator bool() const noexcept { return message_ != nullptr; }
|
|
504
|
+
char const* what() const noexcept { return message_; }
|
|
497
505
|
|
|
498
506
|
#if defined(__cpp_exceptions) && (1 == __cpp_exceptions)
|
|
499
507
|
~error_t() noexcept(false) {
|
|
@@ -526,6 +534,7 @@ template <typename result_at> struct expected_gt {
|
|
|
526
534
|
error.raise();
|
|
527
535
|
return std::move(result);
|
|
528
536
|
}
|
|
537
|
+
result_at const& operator*() const noexcept { return result; }
|
|
529
538
|
explicit operator bool() const noexcept { return !error; }
|
|
530
539
|
expected_gt failed(char const* message) noexcept {
|
|
531
540
|
error = message;
|
|
@@ -932,31 +941,28 @@ class usearch_pack_m uint40_t {
|
|
|
932
941
|
|
|
933
942
|
static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
|
|
934
943
|
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
template <typename scalar_at> class span_gt {
|
|
939
|
-
scalar_at* data_;
|
|
940
|
-
std::size_t size_;
|
|
941
|
-
|
|
942
|
-
public:
|
|
943
|
-
span_gt(scalar_at* begin, scalar_at const* end) noexcept : data_(begin), size_(end - begin) {}
|
|
944
|
-
span_gt(scalar_at* begin, std::size_t size) noexcept : data_(begin), size_(size) {}
|
|
945
|
-
scalar_at* data() const noexcept { return data_; }
|
|
946
|
-
std::size_t size() const noexcept { return size_; }
|
|
947
|
-
operator scalar_at*() const noexcept { return data(); }
|
|
948
|
-
};
|
|
949
|
-
|
|
944
|
+
/// @brief Number of neighbors per graph node.
|
|
945
|
+
/// Defaults to 32 in FAISS and 16 in hnswlib.
|
|
946
|
+
/// > It is called `M` in the paper.
|
|
950
947
|
constexpr std::size_t default_connectivity() { return 16; }
|
|
948
|
+
|
|
949
|
+
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
950
|
+
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
951
|
+
/// > It is called `efConstruction` in the paper.
|
|
951
952
|
constexpr std::size_t default_expansion_add() { return 128; }
|
|
953
|
+
|
|
954
|
+
/// @brief Hyper-parameter controlling the quality of search.
|
|
955
|
+
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
956
|
+
/// > It is called `ef` in the paper.
|
|
952
957
|
constexpr std::size_t default_expansion_search() { return 64; }
|
|
958
|
+
|
|
953
959
|
constexpr std::size_t default_allocator_entry_bytes() { return 64; }
|
|
954
960
|
|
|
955
961
|
/**
|
|
956
962
|
* @brief The "magic" sequence helps infer the type of the file.
|
|
957
963
|
* USearch indexes start with the "unumusearch" string.
|
|
958
964
|
*/
|
|
959
|
-
|
|
965
|
+
constexpr char const* default_magic() { return "unumusearch"; }
|
|
960
966
|
|
|
961
967
|
/**
|
|
962
968
|
* @brief Configuration settings for the index construction.
|
|
@@ -964,22 +970,11 @@ static constexpr char const* default_magic() { return "unumusearch"; }
|
|
|
964
970
|
* and two expansion factors - for construction and search.
|
|
965
971
|
*/
|
|
966
972
|
struct index_config_t {
|
|
967
|
-
|
|
968
973
|
/// @brief Number of neighbors per graph node.
|
|
969
974
|
/// Defaults to 32 in FAISS and 16 in hnswlib.
|
|
970
975
|
/// > It is called `M` in the paper.
|
|
971
976
|
std::size_t connectivity = default_connectivity();
|
|
972
977
|
|
|
973
|
-
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
974
|
-
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
975
|
-
/// > It is called `efConstruction` in the paper.
|
|
976
|
-
std::size_t expansion_add = default_expansion_add();
|
|
977
|
-
|
|
978
|
-
/// @brief Hyper-parameter controlling the quality of search.
|
|
979
|
-
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
980
|
-
/// > It is called `ef` in the paper.
|
|
981
|
-
std::size_t expansion_search = default_expansion_search();
|
|
982
|
-
|
|
983
978
|
/// @brief Parameter controlling the physical layout of vectors
|
|
984
979
|
/// in memory. When using multi-byte scalar types, like `float`,
|
|
985
980
|
/// dereferencing at mis-aligned locations may cause UB. So you
|
|
@@ -1002,18 +997,32 @@ struct index_limits_t {
|
|
|
1002
997
|
};
|
|
1003
998
|
|
|
1004
999
|
struct add_config_t {
|
|
1000
|
+
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
1001
|
+
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
1002
|
+
/// > It is called `efConstruction` in the paper.
|
|
1003
|
+
std::size_t expansion = default_expansion_add();
|
|
1004
|
+
|
|
1005
1005
|
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1006
|
-
std::size_t thread
|
|
1006
|
+
std::size_t thread = 0;
|
|
1007
|
+
|
|
1007
1008
|
/// @brief Don't copy the ::vector, if it's persisted elsewhere.
|
|
1008
|
-
bool store_vector
|
|
1009
|
+
bool store_vector = true;
|
|
1009
1010
|
};
|
|
1010
1011
|
|
|
1011
1012
|
struct search_config_t {
|
|
1012
|
-
|
|
1013
|
-
|
|
1013
|
+
/// @brief Hyper-parameter controlling the quality of search.
|
|
1014
|
+
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1015
|
+
/// > It is called `ef` in the paper.
|
|
1016
|
+
std::size_t expansion = default_expansion_search();
|
|
1017
|
+
|
|
1018
|
+
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1019
|
+
std::size_t thread = 0;
|
|
1020
|
+
|
|
1021
|
+
/// @brief Brute-forces exhaustive search over all entries in the index.
|
|
1022
|
+
bool exact = false;
|
|
1014
1023
|
};
|
|
1015
1024
|
|
|
1016
|
-
enum class
|
|
1025
|
+
enum class metric_kind_t : std::uint8_t {
|
|
1017
1026
|
unknown_k = 0,
|
|
1018
1027
|
// Classics:
|
|
1019
1028
|
ip_k = 'i',
|
|
@@ -1026,9 +1035,9 @@ enum class common_metric_kind_t : std::uint8_t {
|
|
|
1026
1035
|
|
|
1027
1036
|
// Sets:
|
|
1028
1037
|
jaccard_k = 'j',
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1038
|
+
hamming_k = 'b',
|
|
1039
|
+
tanimoto_k = 't',
|
|
1040
|
+
sorensen_k = 's',
|
|
1032
1041
|
};
|
|
1033
1042
|
|
|
1034
1043
|
using file_header_t = byte_t[64];
|
|
@@ -1057,7 +1066,7 @@ struct file_head_t {
|
|
|
1057
1066
|
misaligned_ref_gt<version_major_t> version_major;
|
|
1058
1067
|
misaligned_ref_gt<version_minor_t> version_minor;
|
|
1059
1068
|
misaligned_ref_gt<version_patch_t> version_patch;
|
|
1060
|
-
misaligned_ref_gt<
|
|
1069
|
+
misaligned_ref_gt<metric_kind_t> metric;
|
|
1061
1070
|
misaligned_ref_gt<connectivity_t> connectivity;
|
|
1062
1071
|
misaligned_ref_gt<max_level_t> max_level;
|
|
1063
1072
|
misaligned_ref_gt<vector_alignment_t> vector_alignment;
|
|
@@ -1071,8 +1080,7 @@ struct file_head_t {
|
|
|
1071
1080
|
version_major(exchange(ptr, ptr + sizeof(version_major_t))),
|
|
1072
1081
|
version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
|
|
1073
1082
|
version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
|
|
1074
|
-
metric(exchange(ptr, ptr + sizeof(
|
|
1075
|
-
connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
|
|
1083
|
+
metric(exchange(ptr, ptr + sizeof(metric_kind_t))), connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
|
|
1076
1084
|
max_level(exchange(ptr, ptr + sizeof(max_level_t))),
|
|
1077
1085
|
vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
|
|
1078
1086
|
bytes_per_label(exchange(ptr, ptr + sizeof(bytes_per_label_t))),
|
|
@@ -1082,6 +1090,15 @@ struct file_head_t {
|
|
|
1082
1090
|
|
|
1083
1091
|
static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 bytes");
|
|
1084
1092
|
|
|
1093
|
+
/// @brief C++17 and newer version deprecate the `std::result_of`
|
|
1094
|
+
template <typename metric_at, typename... args_at>
|
|
1095
|
+
using return_type_gt =
|
|
1096
|
+
#if defined(USEARCH_IS_CPP17)
|
|
1097
|
+
typename std::invoke_result<metric_at, args_at...>::type;
|
|
1098
|
+
#else
|
|
1099
|
+
typename std::result_of<metric_at(args_at...)>::type;
|
|
1100
|
+
#endif
|
|
1101
|
+
|
|
1085
1102
|
/**
|
|
1086
1103
|
* @brief Approximate Nearest Neighbors Search index using the
|
|
1087
1104
|
* Hierarchical Navigable Small World graph algorithm.
|
|
@@ -1092,9 +1109,10 @@ static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 byt
|
|
|
1092
1109
|
* - Search for vectors of different dimensionality, if ::`metric_at` supports that.
|
|
1093
1110
|
*
|
|
1094
1111
|
* @tparam metric_at
|
|
1095
|
-
* A function
|
|
1096
|
-
*
|
|
1097
|
-
* - `
|
|
1112
|
+
* A function responsible for computing the distance (dis-similarity) between two vectors.
|
|
1113
|
+
* Can have following signatures:
|
|
1114
|
+
* - `distance_t (*) (span_gt<scalar_t const>, span_gt<scalar_t const>)`
|
|
1115
|
+
* - `distance_t (*) (scalar_t const *, scalar_t const *)`
|
|
1098
1116
|
*
|
|
1099
1117
|
* @tparam label_at
|
|
1100
1118
|
* The type of unique labels to assign to vectors.
|
|
@@ -1140,18 +1158,15 @@ static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 byt
|
|
|
1140
1158
|
* @subsection Missing Functionality
|
|
1141
1159
|
*
|
|
1142
1160
|
* To simplify the implementation, the `index_gt` lacks endpoints to remove existing
|
|
1143
|
-
* vectors. That, however, is solved by `
|
|
1144
|
-
*
|
|
1145
|
-
* @subsection Ugly Implementation Parts
|
|
1146
|
-
*
|
|
1147
|
-
* Internally, a number of `std::vector`-s is created, often `mutable`.
|
|
1161
|
+
* vectors. That, however, is solved by `index_punned_dense_gt`, which also adds automatic casting.
|
|
1148
1162
|
*
|
|
1149
1163
|
*/
|
|
1150
1164
|
template <typename metric_at = ip_gt<float>, //
|
|
1151
1165
|
typename label_at = std::size_t, //
|
|
1152
1166
|
typename id_at = std::uint32_t, //
|
|
1153
1167
|
typename scalar_at = float, //
|
|
1154
|
-
typename allocator_at = std::allocator<char
|
|
1168
|
+
typename allocator_at = std::allocator<char>, //
|
|
1169
|
+
typename point_allocator_at = allocator_at> //
|
|
1155
1170
|
class index_gt {
|
|
1156
1171
|
public:
|
|
1157
1172
|
using metric_t = metric_at;
|
|
@@ -1159,16 +1174,10 @@ class index_gt {
|
|
|
1159
1174
|
using label_t = label_at;
|
|
1160
1175
|
using id_t = id_at;
|
|
1161
1176
|
using allocator_t = allocator_at;
|
|
1162
|
-
|
|
1163
|
-
/// C++17 and newer version deprecate the `std::result_of`
|
|
1164
|
-
using distance_t =
|
|
1165
|
-
#if defined(USEARCH_IS_CPP17)
|
|
1166
|
-
typename std::invoke_result<metric_t, scalar_t const*, scalar_t const*, std::size_t, std::size_t>::type;
|
|
1167
|
-
#else
|
|
1168
|
-
typename std::result_of<metric_t(scalar_t const*, scalar_t const*, std::size_t, std::size_t)>::type;
|
|
1169
|
-
#endif
|
|
1177
|
+
using point_allocator_t = point_allocator_at;
|
|
1170
1178
|
|
|
1171
1179
|
using vector_view_t = span_gt<scalar_t const>;
|
|
1180
|
+
using distance_t = return_type_gt<metric_t, vector_view_t, vector_view_t>;
|
|
1172
1181
|
|
|
1173
1182
|
struct member_ref_t {
|
|
1174
1183
|
misaligned_ref_gt<label_t> label;
|
|
@@ -1253,6 +1262,10 @@ class index_gt {
|
|
|
1253
1262
|
using byte_t = typename allocator_t::value_type;
|
|
1254
1263
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
1255
1264
|
|
|
1265
|
+
using point_allocator_traits_t = std::allocator_traits<point_allocator_t>;
|
|
1266
|
+
static_assert(sizeof(typename point_allocator_traits_t::value_type) == 1,
|
|
1267
|
+
"Allocator must allocate separate addressable bytes");
|
|
1268
|
+
|
|
1256
1269
|
/**
|
|
1257
1270
|
* @brief How much larger (number of neighbors per node) will
|
|
1258
1271
|
* the base level be compared to other levels.
|
|
@@ -1363,7 +1376,7 @@ class index_gt {
|
|
|
1363
1376
|
|
|
1364
1377
|
inline distance_t measure(vector_view_t a, vector_view_t b) noexcept {
|
|
1365
1378
|
measurements_count++;
|
|
1366
|
-
return metric(a
|
|
1379
|
+
return metric(a, b);
|
|
1367
1380
|
}
|
|
1368
1381
|
};
|
|
1369
1382
|
|
|
@@ -1371,6 +1384,7 @@ class index_gt {
|
|
|
1371
1384
|
index_limits_t limits_{};
|
|
1372
1385
|
metric_t metric_{};
|
|
1373
1386
|
allocator_t allocator_{};
|
|
1387
|
+
point_allocator_t point_allocator_{};
|
|
1374
1388
|
precomputed_constants_t pre_{};
|
|
1375
1389
|
int viewed_file_descriptor_{};
|
|
1376
1390
|
|
|
@@ -1403,10 +1417,11 @@ class index_gt {
|
|
|
1403
1417
|
* @section Exceptions
|
|
1404
1418
|
* Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
|
|
1405
1419
|
*/
|
|
1406
|
-
explicit index_gt(index_config_t config = {}, metric_t metric = {}, allocator_t allocator = {}
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1420
|
+
explicit index_gt(index_config_t config = {}, metric_t metric = {}, allocator_t allocator = {},
|
|
1421
|
+
point_allocator_t point_allocator = {}) noexcept
|
|
1422
|
+
: config_(config), limits_(0, 0), metric_(metric), allocator_(std::move(allocator)),
|
|
1423
|
+
point_allocator_(std::move(point_allocator)), pre_(precompute_(config)), viewed_file_descriptor_(0),
|
|
1424
|
+
size_(0u), max_level_(-1), entry_id_(0u), nodes_(nullptr), nodes_mutexes_(), contexts_(nullptr) {}
|
|
1410
1425
|
|
|
1411
1426
|
/**
|
|
1412
1427
|
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
@@ -1558,16 +1573,6 @@ class index_gt {
|
|
|
1558
1573
|
return config;
|
|
1559
1574
|
}
|
|
1560
1575
|
|
|
1561
|
-
/**
|
|
1562
|
-
* @brief Validates/normalizes the configuration options.
|
|
1563
|
-
*/
|
|
1564
|
-
static index_config_t normalize(index_config_t config) noexcept {
|
|
1565
|
-
config.expansion_add = (std::max<std::size_t>)(config.expansion_add, config.connectivity);
|
|
1566
|
-
// config.threads_add = (std::max<std::size_t>)(1u, config.threads_add);
|
|
1567
|
-
// config.threads_search = (std::max<std::size_t>)(1u, config.threads_search);
|
|
1568
|
-
return config;
|
|
1569
|
-
}
|
|
1570
|
-
|
|
1571
1576
|
#pragma endregion
|
|
1572
1577
|
|
|
1573
1578
|
#pragma region Construction and Search
|
|
@@ -1657,10 +1662,10 @@ class index_gt {
|
|
|
1657
1662
|
|
|
1658
1663
|
// The top list needs one more slot than the connectivity of the base level
|
|
1659
1664
|
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
1660
|
-
std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1,
|
|
1665
|
+
std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config.expansion);
|
|
1661
1666
|
if (!top.reserve(top_limit))
|
|
1662
1667
|
return result.failed("Out of memory!");
|
|
1663
|
-
if (!next.reserve(
|
|
1668
|
+
if (!next.reserve(config.expansion))
|
|
1664
1669
|
return result.failed("Out of memory!");
|
|
1665
1670
|
|
|
1666
1671
|
// Determining how much memory to allocate for the node depends on the target level
|
|
@@ -1698,7 +1703,8 @@ class index_gt {
|
|
|
1698
1703
|
|
|
1699
1704
|
// From `target_level` down perform proper extensive search
|
|
1700
1705
|
for (level_t level = (std::min)(target_level, max_level_copy); level >= 0; --level) {
|
|
1701
|
-
|
|
1706
|
+
// TODO: Handle out of memory conditions
|
|
1707
|
+
search_to_insert_(closest_id, vector, level, config.expansion, context);
|
|
1702
1708
|
closest_id = connect_new_node_(new_id, level, context);
|
|
1703
1709
|
}
|
|
1704
1710
|
|
|
@@ -1740,7 +1746,7 @@ class index_gt {
|
|
|
1740
1746
|
search_exact_(query, wanted, context);
|
|
1741
1747
|
} else {
|
|
1742
1748
|
next_candidates_t& next = context.next_candidates;
|
|
1743
|
-
std::size_t expansion = (std::max)(
|
|
1749
|
+
std::size_t expansion = (std::max)(config.expansion, wanted);
|
|
1744
1750
|
if (!next.reserve(expansion))
|
|
1745
1751
|
return result.failed("Out of memory!");
|
|
1746
1752
|
if (!top.reserve(expansion))
|
|
@@ -1774,7 +1780,7 @@ class index_gt {
|
|
|
1774
1780
|
if (!size_)
|
|
1775
1781
|
return result;
|
|
1776
1782
|
|
|
1777
|
-
std::size_t expansion = (std::max)(
|
|
1783
|
+
std::size_t expansion = (std::max)(config.expansion, wanted);
|
|
1778
1784
|
if (!next.reserve(expansion))
|
|
1779
1785
|
return result.failed("Out of memory!");
|
|
1780
1786
|
if (!top.reserve(expansion))
|
|
@@ -1854,13 +1860,10 @@ class index_gt {
|
|
|
1854
1860
|
total += limits_.elements * sizeof(node_t) + allocator_entry_bytes;
|
|
1855
1861
|
|
|
1856
1862
|
// Temporary data-structures, proportional to the number of threads:
|
|
1857
|
-
|
|
1858
|
-
total += limits_.threads() * (sizeof(context_t) + per_thread) + allocator_entry_bytes * 3;
|
|
1863
|
+
total += limits_.threads() * sizeof(context_t) + allocator_entry_bytes * 3;
|
|
1859
1864
|
return total;
|
|
1860
1865
|
}
|
|
1861
1866
|
|
|
1862
|
-
void change_expansion_add(std::size_t n) noexcept { config_.expansion_add = n; }
|
|
1863
|
-
void change_expansion_search(std::size_t n) noexcept { config_.expansion_search = n; }
|
|
1864
1867
|
void change_metric(metric_t const& m) noexcept {
|
|
1865
1868
|
metric_ = m;
|
|
1866
1869
|
for (std::size_t i = 0; i != limits_.threads(); ++i)
|
|
@@ -1898,7 +1901,7 @@ class index_gt {
|
|
|
1898
1901
|
state.version_major = USEARCH_VERSION_MAJOR;
|
|
1899
1902
|
state.version_minor = USEARCH_VERSION_MINOR;
|
|
1900
1903
|
state.version_patch = USEARCH_VERSION_PATCH;
|
|
1901
|
-
state.metric =
|
|
1904
|
+
state.metric = metric_kind_t::unknown_k;
|
|
1902
1905
|
|
|
1903
1906
|
// Describe state
|
|
1904
1907
|
state.connectivity = config_.connectivity;
|
|
@@ -2133,17 +2136,18 @@ class index_gt {
|
|
|
2133
2136
|
|
|
2134
2137
|
node_t& node = nodes_[id];
|
|
2135
2138
|
std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_stored_(node);
|
|
2136
|
-
|
|
2139
|
+
point_allocator_.deallocate(node.tape(), node_bytes);
|
|
2137
2140
|
node = node_t{};
|
|
2138
2141
|
}
|
|
2139
2142
|
|
|
2140
2143
|
node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
|
|
2141
2144
|
|
|
2142
2145
|
std::size_t dim = vector.size();
|
|
2143
|
-
std::size_t stored_vector_bytes = node_vector_bytes_(dim) * store_vector;
|
|
2144
|
-
std::size_t node_bytes = node_bytes_(dim, level) -
|
|
2146
|
+
std::size_t stored_vector_bytes = node_vector_bytes_(static_cast<dim_t>(dim)) * std::size_t(store_vector);
|
|
2147
|
+
std::size_t node_bytes = node_bytes_(static_cast<dim_t>(dim), level) -
|
|
2148
|
+
node_vector_bytes_(static_cast<dim_t>(dim)) * std::size_t(!store_vector);
|
|
2145
2149
|
|
|
2146
|
-
byte_t* data = (byte_t*)
|
|
2150
|
+
byte_t* data = (byte_t*)point_allocator_.allocate(node_bytes);
|
|
2147
2151
|
if (!data)
|
|
2148
2152
|
return {};
|
|
2149
2153
|
|
|
@@ -2277,12 +2281,12 @@ class index_gt {
|
|
|
2277
2281
|
* Locks the nodes in the process, assuming other threads are updating neighbors lists.
|
|
2278
2282
|
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
2279
2283
|
*/
|
|
2280
|
-
bool search_to_insert_(
|
|
2284
|
+
bool search_to_insert_( //
|
|
2285
|
+
id_t start_id, vector_view_t query, level_t level, std::size_t top_limit, context_t& context) noexcept {
|
|
2281
2286
|
|
|
2282
2287
|
visits_bitset_t& visits = context.visits;
|
|
2283
2288
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
2284
2289
|
top_candidates_t& top = context.top_candidates; // pop max, push
|
|
2285
|
-
std::size_t const top_limit = config_.expansion_add;
|
|
2286
2290
|
|
|
2287
2291
|
visits.clear();
|
|
2288
2292
|
next.clear();
|