usearch 2.11.0 → 2.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -37
- package/binding.gyp +11 -12
- package/c/README.md +9 -3
- package/c/lib.cpp +50 -13
- package/c/test.c +60 -55
- package/c/usearch.h +16 -0
- package/cpp/CMakeLists.txt +6 -0
- package/cpp/README.md +46 -14
- package/cpp/bench.cpp +99 -47
- package/cpp/test.cpp +948 -132
- package/include/usearch/index.hpp +894 -272
- package/include/usearch/index_dense.hpp +419 -152
- package/include/usearch/index_plugins.hpp +99 -49
- package/javascript/dist/cjs/usearch.d.ts +39 -1
- package/javascript/dist/cjs/usearch.js +11 -1
- package/javascript/dist/esm/usearch.d.ts +39 -1
- package/javascript/dist/esm/usearch.js +11 -1
- package/javascript/tsconfig-esm.json +3 -2
- package/javascript/usearch.ts +18 -7
- package/package.json +2 -2
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.cmake-format.py +19 -0
- package/simsimd/CONTRIBUTING.md +49 -8
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +10 -5
- package/simsimd/Package.swift +16 -0
- package/simsimd/README.md +69 -15
- package/simsimd/binding.gyp +14 -19
- package/simsimd/build.rs +2 -0
- package/simsimd/c/lib.c +18 -0
- package/simsimd/include/module.modulemap +4 -0
- package/simsimd/include/simsimd/binary.h +2 -0
- package/simsimd/include/simsimd/dot.h +466 -32
- package/simsimd/include/simsimd/probability.h +20 -10
- package/simsimd/include/simsimd/simsimd.h +444 -98
- package/simsimd/include/simsimd/spatial.h +480 -44
- package/simsimd/include/simsimd/types.h +161 -39
- package/simsimd/javascript/simsimd.ts +26 -2
- package/simsimd/package.json +1 -1
- package/simsimd/rust/benches/cosine.rs +8 -2
- package/simsimd/rust/benches/native.rs +240 -2
- package/simsimd/rust/benches/sqeuclidean.rs +8 -2
- package/simsimd/rust/lib.rs +50 -13
- package/simsimd/swift/source/SimSIMD.swift +123 -0
- package/simsimd/swift/tests/SimSIMDTests.swift +92 -0
- package/prebuilds/win32-ia32/usearch.node +0 -0
- package/prebuilds/win32-x64/usearch.node +0 -0
- package/simsimd/.releaserc +0 -87
|
@@ -1,16 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @file
|
|
3
|
-
* @author
|
|
4
|
-
* @brief
|
|
5
|
-
* @date 2023
|
|
6
|
-
*
|
|
7
|
-
* @copyright Copyright (c) 2023
|
|
2
|
+
* @file index.hpp
|
|
3
|
+
* @author Ash Vardanian
|
|
4
|
+
* @brief Single-header Vector Search engine.
|
|
5
|
+
* @date April 26, 2023
|
|
8
6
|
*/
|
|
9
7
|
#ifndef UNUM_USEARCH_HPP
|
|
10
8
|
#define UNUM_USEARCH_HPP
|
|
11
9
|
|
|
12
10
|
#define USEARCH_VERSION_MAJOR 2
|
|
13
|
-
#define USEARCH_VERSION_MINOR
|
|
11
|
+
#define USEARCH_VERSION_MINOR 13
|
|
14
12
|
#define USEARCH_VERSION_PATCH 0
|
|
15
13
|
|
|
16
14
|
// Inferring C++ version
|
|
@@ -18,6 +16,9 @@
|
|
|
18
16
|
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
19
17
|
#define USEARCH_DEFINED_CPP17
|
|
20
18
|
#endif
|
|
19
|
+
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) || __cplusplus >= 202002L)
|
|
20
|
+
#define USEARCH_DEFINED_CPP20
|
|
21
|
+
#endif
|
|
21
22
|
|
|
22
23
|
// Inferring target OS: Windows, MacOS, or Linux
|
|
23
24
|
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
|
|
@@ -35,6 +36,14 @@
|
|
|
35
36
|
#define USEARCH_DEFINED_GCC
|
|
36
37
|
#endif
|
|
37
38
|
|
|
39
|
+
// The `#pragma region` and `#pragma endregion` are not supported by GCC 12 and older.
|
|
40
|
+
// But they are supported by GCC 13, all recent Clang versions, and MSVC.
|
|
41
|
+
#if defined(__GNUC__) && ((__GNUC__ > 13) || (__GNUC__ == 13 && __GNUC_MINOR__ >= 0))
|
|
42
|
+
#define USEARCH_USE_PRAGMA_REGION
|
|
43
|
+
#elif defined(__clang__) || defined(_MSC_VER)
|
|
44
|
+
#define USEARCH_USE_PRAGMA_REGION
|
|
45
|
+
#endif
|
|
46
|
+
|
|
38
47
|
// Inferring hardware architecture: x86 vs Arm
|
|
39
48
|
#if defined(__x86_64__)
|
|
40
49
|
#define USEARCH_DEFINED_X86
|
|
@@ -86,17 +95,39 @@
|
|
|
86
95
|
#include <thread> // `std::thread`
|
|
87
96
|
#include <utility> // `std::pair`
|
|
88
97
|
|
|
98
|
+
// Helper macros for concatenation and stringification
|
|
99
|
+
#define usearch_concat_helper_m(a, b) a##b
|
|
100
|
+
#define usearch_concat_m(a, b) usearch_concat_helper_m(a, b)
|
|
101
|
+
#define usearch_stringify_helper_m(x) #x
|
|
102
|
+
#define usearch_stringify_m(x) usearch_stringify_helper_m(x)
|
|
103
|
+
|
|
89
104
|
// Prefetching
|
|
90
105
|
#if defined(USEARCH_DEFINED_GCC)
|
|
91
106
|
// https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
|
|
92
107
|
// Zero means we are only going to read from that memory.
|
|
93
108
|
// Three means high temporal locality and suggests to keep
|
|
94
109
|
// the data in all layers of cache.
|
|
95
|
-
#define
|
|
110
|
+
#define usearch_prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
|
|
96
111
|
#elif defined(USEARCH_DEFINED_X86)
|
|
97
|
-
#define
|
|
112
|
+
#define usearch_prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
|
|
98
113
|
#else
|
|
99
|
-
#define
|
|
114
|
+
#define usearch_prefetch_m(ptr)
|
|
115
|
+
#endif
|
|
116
|
+
|
|
117
|
+
// Function profiling
|
|
118
|
+
#if defined(usearch_defined_x86)
|
|
119
|
+
#define usearch_profiled_m __attribute__((noinline))
|
|
120
|
+
#define usearch_profile_name_m(name) \
|
|
121
|
+
__asm__ volatile(".globl " usearch_stringify_m(usearch_concat_m(name, __COUNTER__)) "\n" usearch_stringify_m( \
|
|
122
|
+
usearch_concat_m(name, __COUNTER__)) ":")
|
|
123
|
+
#elif defined(usearch_defined_arm)
|
|
124
|
+
#define usearch_profiled_m __attribute__((noinline))
|
|
125
|
+
#define usearch_profile_name_m(name) \
|
|
126
|
+
__asm__ volatile(".global " usearch_stringify_m(usearch_concat_m(name, __COUNTER__)) "\n" usearch_stringify_m( \
|
|
127
|
+
usearch_concat_m(name, __COUNTER__)) ":")
|
|
128
|
+
#else
|
|
129
|
+
#define usearch_profiled_m
|
|
130
|
+
#define usearch_profile_name_m(name)
|
|
100
131
|
#endif
|
|
101
132
|
|
|
102
133
|
// Alignment
|
|
@@ -115,11 +146,25 @@
|
|
|
115
146
|
#else
|
|
116
147
|
#define usearch_assert_m(must_be_true, message) \
|
|
117
148
|
if (!(must_be_true)) { \
|
|
118
|
-
|
|
149
|
+
__usearch_raise_runtime_error(message); \
|
|
119
150
|
}
|
|
120
151
|
#define usearch_noexcept_m
|
|
121
152
|
#endif
|
|
122
153
|
|
|
154
|
+
extern "C" {
|
|
155
|
+
/// @brief Helper function to simplify debugging - trace just one symbol - `__usearch_raise_runtime_error`.
|
|
156
|
+
/// Assuming the `extern C` block, the name won't be mangled.
|
|
157
|
+
inline static void __usearch_raise_runtime_error(char const* message) {
|
|
158
|
+
// On Windows we compile with `/EHc` flag, which specifies that functions
|
|
159
|
+
// with C linkage do not throw C++ exceptions.
|
|
160
|
+
#if !defined(__cpp_exceptions) || defined(USEARCH_DEFINED_WINDOWS)
|
|
161
|
+
std::terminate();
|
|
162
|
+
#else
|
|
163
|
+
throw std::runtime_error(message);
|
|
164
|
+
#endif
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
123
168
|
namespace unum {
|
|
124
169
|
namespace usearch {
|
|
125
170
|
|
|
@@ -168,6 +213,13 @@ template <typename at, typename other_at = at> at exchange(at& obj, other_at&& n
|
|
|
168
213
|
return old_value;
|
|
169
214
|
}
|
|
170
215
|
|
|
216
|
+
#if defined(USEARCH_DEFINED_CPP20)
|
|
217
|
+
|
|
218
|
+
template <typename at> void destroy_at(at* obj) { std::destroy_at(obj); }
|
|
219
|
+
template <typename at> void construct_at(at* obj) { std::construct_at(obj); }
|
|
220
|
+
|
|
221
|
+
#else
|
|
222
|
+
|
|
171
223
|
/// @brief The `std::destroy_at` alternative for C++11.
|
|
172
224
|
template <typename at, typename sfinae_at = at>
|
|
173
225
|
typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
|
|
@@ -184,6 +236,8 @@ typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* o
|
|
|
184
236
|
new (obj) at();
|
|
185
237
|
}
|
|
186
238
|
|
|
239
|
+
#endif
|
|
240
|
+
|
|
187
241
|
/**
|
|
188
242
|
* @brief A reference to a misaligned memory location with a specific type.
|
|
189
243
|
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
@@ -223,27 +277,57 @@ template <typename at> class misaligned_ptr_gt {
|
|
|
223
277
|
using pointer = misaligned_ptr_gt<element_t>;
|
|
224
278
|
using reference = misaligned_ref_gt<element_t>;
|
|
225
279
|
|
|
280
|
+
misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
281
|
+
|
|
226
282
|
reference operator*() const noexcept { return {ptr_}; }
|
|
227
283
|
reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
|
|
228
284
|
value_type operator[](std::size_t i) const noexcept {
|
|
229
285
|
return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
|
|
230
286
|
}
|
|
231
287
|
|
|
232
|
-
misaligned_ptr_gt(
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
misaligned_ptr_gt operator
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
misaligned_ptr_gt
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
288
|
+
misaligned_ptr_gt& operator++() noexcept {
|
|
289
|
+
ptr_ += sizeof(element_t);
|
|
290
|
+
return *this;
|
|
291
|
+
}
|
|
292
|
+
misaligned_ptr_gt& operator--() noexcept {
|
|
293
|
+
ptr_ -= sizeof(element_t);
|
|
294
|
+
return *this;
|
|
295
|
+
}
|
|
296
|
+
misaligned_ptr_gt operator++(int) noexcept {
|
|
297
|
+
misaligned_ptr_gt tmp = *this;
|
|
298
|
+
++(*this);
|
|
299
|
+
return tmp;
|
|
300
|
+
}
|
|
301
|
+
misaligned_ptr_gt operator--(int) noexcept {
|
|
302
|
+
misaligned_ptr_gt tmp = *this;
|
|
303
|
+
--(*this);
|
|
304
|
+
return tmp;
|
|
305
|
+
}
|
|
306
|
+
misaligned_ptr_gt operator+(difference_type d) const noexcept {
|
|
307
|
+
return misaligned_ptr_gt(ptr_ + d * sizeof(element_t));
|
|
308
|
+
}
|
|
309
|
+
misaligned_ptr_gt operator-(difference_type d) const noexcept {
|
|
310
|
+
return misaligned_ptr_gt(ptr_ - d * sizeof(element_t));
|
|
311
|
+
}
|
|
312
|
+
difference_type operator-(const misaligned_ptr_gt& other) const noexcept {
|
|
313
|
+
return (ptr_ - other.ptr_) / sizeof(element_t);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
misaligned_ptr_gt& operator+=(difference_type d) noexcept {
|
|
317
|
+
ptr_ += d * sizeof(element_t);
|
|
318
|
+
return *this;
|
|
319
|
+
}
|
|
320
|
+
misaligned_ptr_gt& operator-=(difference_type d) noexcept {
|
|
321
|
+
ptr_ -= d * sizeof(element_t);
|
|
322
|
+
return *this;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
bool operator==(misaligned_ptr_gt const& other) const noexcept { return ptr_ == other.ptr_; }
|
|
326
|
+
bool operator!=(misaligned_ptr_gt const& other) const noexcept { return ptr_ != other.ptr_; }
|
|
327
|
+
bool operator<(misaligned_ptr_gt const& other) const noexcept { return ptr_ < other.ptr_; }
|
|
328
|
+
bool operator<=(misaligned_ptr_gt const& other) const noexcept { return ptr_ <= other.ptr_; }
|
|
329
|
+
bool operator>(misaligned_ptr_gt const& other) const noexcept { return ptr_ > other.ptr_; }
|
|
330
|
+
bool operator>=(misaligned_ptr_gt const& other) const noexcept { return ptr_ >= other.ptr_; }
|
|
247
331
|
};
|
|
248
332
|
|
|
249
333
|
/**
|
|
@@ -279,7 +363,8 @@ template <typename scalar_at, typename allocator_at = std::allocator<scalar_at>>
|
|
|
279
363
|
for (std::size_t i = 0; i != size_; ++i)
|
|
280
364
|
construct_at(data_ + i);
|
|
281
365
|
}
|
|
282
|
-
~buffer_gt() noexcept {
|
|
366
|
+
~buffer_gt() noexcept { reset(); }
|
|
367
|
+
void reset() noexcept {
|
|
283
368
|
if (!std::is_trivially_destructible<scalar_at>::value)
|
|
284
369
|
for (std::size_t i = 0; i != size_; ++i)
|
|
285
370
|
destroy_at(data_ + i);
|
|
@@ -319,7 +404,8 @@ class error_t {
|
|
|
319
404
|
char const* message_{};
|
|
320
405
|
|
|
321
406
|
public:
|
|
322
|
-
error_t(
|
|
407
|
+
error_t() noexcept : message_(nullptr) {}
|
|
408
|
+
error_t(char const* message) noexcept : message_(message) {}
|
|
323
409
|
error_t& operator=(char const* message) noexcept {
|
|
324
410
|
message_ = message;
|
|
325
411
|
return *this;
|
|
@@ -332,11 +418,18 @@ class error_t {
|
|
|
332
418
|
std::swap(message_, other.message_);
|
|
333
419
|
return *this;
|
|
334
420
|
}
|
|
421
|
+
|
|
422
|
+
/// @brief Checks if there was an error.
|
|
335
423
|
explicit operator bool() const noexcept { return message_ != nullptr; }
|
|
424
|
+
|
|
425
|
+
/// @brief Returns the error message.
|
|
336
426
|
char const* what() const noexcept { return message_; }
|
|
427
|
+
|
|
428
|
+
/// @brief Releases the error message, meaning the caller takes ownership.
|
|
337
429
|
char const* release() noexcept { return exchange(message_, nullptr); }
|
|
338
430
|
|
|
339
431
|
#if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
|
|
432
|
+
/// @brief Destructor raises an exception if an error was recorded.
|
|
340
433
|
~error_t() noexcept(false) {
|
|
341
434
|
#if defined(USEARCH_DEFINED_CPP17)
|
|
342
435
|
if (message_ && std::uncaught_exceptions() == 0)
|
|
@@ -345,12 +438,17 @@ class error_t {
|
|
|
345
438
|
#endif
|
|
346
439
|
raise();
|
|
347
440
|
}
|
|
441
|
+
|
|
442
|
+
/// @brief Throws an exception using to be caught by `try` / `catch`.
|
|
348
443
|
void raise() noexcept(false) {
|
|
349
444
|
if (message_)
|
|
350
445
|
throw std::runtime_error(exchange(message_, nullptr));
|
|
351
446
|
}
|
|
352
447
|
#else
|
|
448
|
+
/// @brief Destructor terminates if an error was recorded.
|
|
353
449
|
~error_t() noexcept { raise(); }
|
|
450
|
+
|
|
451
|
+
/// @brief Terminates if an error was recorded.
|
|
354
452
|
void raise() noexcept {
|
|
355
453
|
if (message_)
|
|
356
454
|
std::terminate();
|
|
@@ -363,7 +461,7 @@ class error_t {
|
|
|
363
461
|
* or an error. It's used to avoid raising exception, and gracefully propagate
|
|
364
462
|
* the error.
|
|
365
463
|
*
|
|
366
|
-
*
|
|
464
|
+
* @tparam result_at The type of the expected result.
|
|
367
465
|
*/
|
|
368
466
|
template <typename result_at> struct expected_gt {
|
|
369
467
|
result_at result;
|
|
@@ -497,6 +595,64 @@ using bitset_t = bitset_gt<>;
|
|
|
497
595
|
* @brief Similar to `std::priority_queue`, but allows raw access to underlying
|
|
498
596
|
* memory, in case you want to shuffle it or sort. Good for collections
|
|
499
597
|
* from 100s to 10'000s elements.
|
|
598
|
+
*
|
|
599
|
+
* In a max-heap, the heap property ensures that the value of each node is greater
|
|
600
|
+
* than or equal to the values of its children. This means that the largest element
|
|
601
|
+
* is always at the root of the heap.
|
|
602
|
+
*
|
|
603
|
+
* @section Heap Structures
|
|
604
|
+
*
|
|
605
|
+
* There are several designs of heaps. Binary heaps are the simplest & most common
|
|
606
|
+
* variant, that is easy to implement as a succint array. However, they are not the
|
|
607
|
+
* most efficient for all operations. Most importantly, @b melding (merging) of
|
|
608
|
+
* two heaps has linear complexity in time.
|
|
609
|
+
*
|
|
610
|
+
* +-----------------+---------+-----------+---------+--------------+---------+
|
|
611
|
+
* | Operation | find-max| delete-max| insert | increase-key | meld |
|
|
612
|
+
* +-----------------+---------+-----------+---------+--------------+---------+
|
|
613
|
+
* | Binary | Θ(1) | Θ(log n) | O(log n)| O(log n) | Θ(n) |
|
|
614
|
+
* | Leftist | Θ(1) | Θ(log n) | O(log n)| Θ(log n) | Θ(log n)|
|
|
615
|
+
* | Binomial | Θ(1) | Θ(log n) | Θ(1) | Θ(log n) | O(log n)|
|
|
616
|
+
* | Skew binomial | Θ(1) | Θ(log n) | Θ(1) | O(log n) | O(log n)|
|
|
617
|
+
* | Pairing | Θ(1) | O(log n) | Θ(1) | o(log n) | Θ(1) |
|
|
618
|
+
* | Rank-pairing | Θ(1) | O(log n) | Θ(1) | Θ(1) | Θ(1) |
|
|
619
|
+
* | Fibonacci | Θ(1) | O(log n) | Θ(1) | Θ(1) | Θ(1) |
|
|
620
|
+
* | Strict Fibonacci| Θ(1) | O(log n) | Θ(1) | Θ(1) | Θ(1) |
|
|
621
|
+
* | Brodal | Θ(1) | Θ(log n) | Θ(1) | Θ(1) | Θ(1) |
|
|
622
|
+
* | 2–3 heap | Θ(1) | O(log n) | Θ(1) | Θ(1) | O(log n)|
|
|
623
|
+
* +-----------------+---------+-----------+---------+--------------+---------+
|
|
624
|
+
*
|
|
625
|
+
* It's well known, that improved priority queue structures translate into better
|
|
626
|
+
* graph-transversal algorithms. For example, Dijkstra's algorithm can be sped up
|
|
627
|
+
* by using a Fibonacci heap for arbitrary weights. For integer weight bounded
|
|
628
|
+
* by L, Schrijver reported following time complexities in 2004:
|
|
629
|
+
*
|
|
630
|
+
* +------------+-------------------------------------+----------------------------+--------------------------+
|
|
631
|
+
* | Weights | Algorithm | Time complexity | Author |
|
|
632
|
+
* +------------+-------------------------------------+----------------------------+--------------------------+
|
|
633
|
+
* | R | | O(V^2 EL) | Ford 1956 |
|
|
634
|
+
* | R | Bellman–Ford algorithm | O(VE) | Shimbel 1955, Bellman |
|
|
635
|
+
* | | | | 1958, Moore 1959 |
|
|
636
|
+
* | R | | O(V^2 log V) | Dantzig 1960 |
|
|
637
|
+
* | R | Dijkstra's with list | O(V^2) | Leyzorek et al. 1957, |
|
|
638
|
+
* | | | | Dijkstra 1959... |
|
|
639
|
+
* | R | Dijkstra's with binary heap | O((E + V) log V) | Johnson 1977 |
|
|
640
|
+
* | R | Dijkstra's with Fibonacci heap | O(E + V log V) | Fredman & Tarjan 1984, |
|
|
641
|
+
* | | | | Fredman & Tarjan 1987 |
|
|
642
|
+
* | R | Quantum Dijkstra | O(√VE log^2 V) | Dürr et al. 2006 |
|
|
643
|
+
* | R | Dial's algorithm (Dijkstra's using | O(E + LV) | Dial 1969 |
|
|
644
|
+
* | | a bucket queue with L buckets) | | |
|
|
645
|
+
* | N | | O(E log log L) | Johnson 1981, Karlsson & |
|
|
646
|
+
* | | | | Poblete 1983 |
|
|
647
|
+
* | N | Gabow's algorithm | O(E log_E/V L) | Gabow 1983, Gabow 1985 |
|
|
648
|
+
* | N | | O(E + V √log L) | Ahuja et al. 1990 |
|
|
649
|
+
* | N | Thorup | O(E + V log log V) | Thorup 2004 |
|
|
650
|
+
* +------------+-------------------------------------+----------------------------+--------------------------+
|
|
651
|
+
*
|
|
652
|
+
* Possible improvements:
|
|
653
|
+
* - Randomized meldable heaps: https://en.wikipedia.org/wiki/Randomized_meldable_heap
|
|
654
|
+
* - D-ary heaps: https://en.wikipedia.org/wiki/D-ary_heap
|
|
655
|
+
* - B-heap: https://en.wikipedia.org/wiki/B-heap
|
|
500
656
|
*/
|
|
501
657
|
template <typename element_at, //
|
|
502
658
|
typename comparator_at = std::less<void>, // <void> is needed before C++14.
|
|
@@ -547,13 +703,25 @@ class max_heap_gt {
|
|
|
547
703
|
inline bool empty() const noexcept { return !size_; }
|
|
548
704
|
inline std::size_t size() const noexcept { return size_; }
|
|
549
705
|
inline std::size_t capacity() const noexcept { return capacity_; }
|
|
706
|
+
inline element_t* data() noexcept { return elements_; }
|
|
707
|
+
inline element_t const* data() const noexcept { return elements_; }
|
|
708
|
+
inline void clear() noexcept { size_ = 0; }
|
|
709
|
+
inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
550
710
|
|
|
551
711
|
/// @brief Selects the largest element in the heap.
|
|
552
712
|
/// @return Reference to the stored element.
|
|
553
713
|
inline element_t const& top() const noexcept { return elements_[0]; }
|
|
554
|
-
inline void clear() noexcept { size_ = 0; }
|
|
555
714
|
|
|
556
|
-
|
|
715
|
+
/// @brief Invalidates the "max-heap" property, transforming into ascending range.
|
|
716
|
+
inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* @brief Ensures the heap has enough capacity for the specified number of elements.
|
|
720
|
+
* @param new_capacity The desired minimum capacity.
|
|
721
|
+
* @return True if the capacity was successfully increased, false otherwise.
|
|
722
|
+
*/
|
|
723
|
+
usearch_profiled_m bool reserve(std::size_t new_capacity) noexcept {
|
|
724
|
+
usearch_profile_name_m(max_heap_reserve);
|
|
557
725
|
if (new_capacity < capacity_)
|
|
558
726
|
return true;
|
|
559
727
|
|
|
@@ -573,6 +741,11 @@ class max_heap_gt {
|
|
|
573
741
|
return new_elements;
|
|
574
742
|
}
|
|
575
743
|
|
|
744
|
+
/**
|
|
745
|
+
* @brief Inserts an element into the heap.
|
|
746
|
+
* @param element The element to be inserted.
|
|
747
|
+
* @return True if the element was successfully inserted, false otherwise.
|
|
748
|
+
*/
|
|
576
749
|
bool insert(element_t&& element) noexcept {
|
|
577
750
|
if (!reserve(size_ + 1))
|
|
578
751
|
return false;
|
|
@@ -581,13 +754,33 @@ class max_heap_gt {
|
|
|
581
754
|
return true;
|
|
582
755
|
}
|
|
583
756
|
|
|
584
|
-
|
|
757
|
+
/**
|
|
758
|
+
* @brief Inserts an element into the heap without reserving additional space.
|
|
759
|
+
* @param element The element to be inserted.
|
|
760
|
+
*/
|
|
761
|
+
usearch_profiled_m void insert_reserved(element_t&& element) noexcept {
|
|
762
|
+
usearch_profile_name_m(max_heap_insert_reserved);
|
|
585
763
|
new (&elements_[size_]) element_t(element);
|
|
586
764
|
size_++;
|
|
587
765
|
shift_up(size_ - 1);
|
|
588
766
|
}
|
|
589
767
|
|
|
590
|
-
|
|
768
|
+
/**
|
|
769
|
+
* @brief Inserts multiple elements into the heap.
|
|
770
|
+
* @param elements Pointer to the elements to be inserted.
|
|
771
|
+
* @return True if the elements were successfully inserted, false otherwise.
|
|
772
|
+
*/
|
|
773
|
+
inline bool insert_many(element_t const* elements) noexcept {
|
|
774
|
+
// Wikipedia describes a procedure, due to Floyd, which constructs a heap from an array in linear time.
|
|
775
|
+
// It also mentions a procedure for merging two heaps, of sizes 𝑛 and 𝑘, in time 𝑂(𝑘+log𝑘log𝑛).
|
|
776
|
+
// Altogether, we can add 𝑘 elements to a heap of length 𝑛 in time 𝑂(𝑘+log𝑘log𝑛): first build a heap containing
|
|
777
|
+
// 𝑘 elements to be inserted (takes 𝑂(𝑘) time), then merge that with the heap of size 𝑛 (takes 𝑂(𝑘+log𝑘log𝑛)
|
|
778
|
+
// time). Compare this to repeated insertion, which would run in time 𝑂(𝑘log𝑛).
|
|
779
|
+
return false;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
usearch_profiled_m element_t pop() noexcept {
|
|
783
|
+
usearch_profile_name_m(max_heap_pop);
|
|
591
784
|
element_t result = top();
|
|
592
785
|
std::swap(elements_[0], elements_[size_ - 1]);
|
|
593
786
|
size_--;
|
|
@@ -596,24 +789,29 @@ class max_heap_gt {
|
|
|
596
789
|
return result;
|
|
597
790
|
}
|
|
598
791
|
|
|
599
|
-
/** @brief Invalidates the "max-heap" property, transforming into ascending range. */
|
|
600
|
-
inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
|
|
601
|
-
inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
602
|
-
|
|
603
|
-
inline element_t* data() noexcept { return elements_; }
|
|
604
|
-
inline element_t const* data() const noexcept { return elements_; }
|
|
605
|
-
|
|
606
792
|
private:
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
793
|
+
static std::size_t parent_idx(std::size_t i) noexcept { return (i - 1u) / 2u; }
|
|
794
|
+
static std::size_t left_child_idx(std::size_t i) noexcept { return (i * 2u) + 1u; }
|
|
795
|
+
static std::size_t right_child_idx(std::size_t i) noexcept { return (i * 2u) + 2u; }
|
|
610
796
|
static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
|
|
611
797
|
|
|
798
|
+
/**
|
|
799
|
+
* @brief Shifts an element up to maintain the heap property.
|
|
800
|
+
* This operation is called when a new element is @b added at the end of the heap.
|
|
801
|
+
* The element is moved up until the heap property is restored.
|
|
802
|
+
* @param i Index of the element to be shifted up.
|
|
803
|
+
*/
|
|
612
804
|
void shift_up(std::size_t i) noexcept {
|
|
613
805
|
for (; i && less(elements_[parent_idx(i)], elements_[i]); i = parent_idx(i))
|
|
614
806
|
std::swap(elements_[parent_idx(i)], elements_[i]);
|
|
615
807
|
}
|
|
616
808
|
|
|
809
|
+
/**
|
|
810
|
+
* @brief Shifts an element down to maintain the heap property.
|
|
811
|
+
* This operation is called when the root element is @b removed and the last element is moved to the root.
|
|
812
|
+
* The element is moved down until the heap property is restored.
|
|
813
|
+
* @param i Index of the element to be shifted down.
|
|
814
|
+
*/
|
|
617
815
|
void shift_down(std::size_t i) noexcept {
|
|
618
816
|
std::size_t max_idx = i;
|
|
619
817
|
|
|
@@ -736,6 +934,18 @@ class sorted_buffer_gt {
|
|
|
736
934
|
return true;
|
|
737
935
|
}
|
|
738
936
|
|
|
937
|
+
inline bool insert_sorted(element_t const* elements, std::size_t elements_count, std::size_t limit) noexcept {
|
|
938
|
+
if (!size_) {
|
|
939
|
+
}
|
|
940
|
+
// If we are inserting elements, we only perform full-scale binary search once,
|
|
941
|
+
// and then only compute successive insertion offsets based only on the tail.
|
|
942
|
+
std::size_t slot = std::lower_bound(elements_, elements_ + size_, elements[0], &less) - elements_;
|
|
943
|
+
if (slot == limit)
|
|
944
|
+
return false;
|
|
945
|
+
std::size_t to_move = size_ - slot - (size_ == limit);
|
|
946
|
+
std::size_t next_slot = std::lower_bound(elements_ + slot, elements_ + size_, elements[1], &less) - elements_;
|
|
947
|
+
}
|
|
948
|
+
|
|
739
949
|
inline element_t pop() noexcept {
|
|
740
950
|
size_--;
|
|
741
951
|
element_t result = elements_[size_];
|
|
@@ -758,9 +968,7 @@ class sorted_buffer_gt {
|
|
|
758
968
|
#endif
|
|
759
969
|
|
|
760
970
|
/**
|
|
761
|
-
* @brief
|
|
762
|
-
*
|
|
763
|
-
* @note Avoid usage in 32bit environment
|
|
971
|
+
* @brief Five-byte integer type to address node clouds with over 4B entries.
|
|
764
972
|
*/
|
|
765
973
|
class usearch_pack_m uint40_t {
|
|
766
974
|
unsigned char octets[5];
|
|
@@ -772,7 +980,10 @@ class usearch_pack_m uint40_t {
|
|
|
772
980
|
|
|
773
981
|
public:
|
|
774
982
|
inline uint40_t() noexcept { broadcast(0); }
|
|
775
|
-
inline uint40_t(std::uint32_t n) noexcept {
|
|
983
|
+
inline uint40_t(std::uint32_t n) noexcept {
|
|
984
|
+
std::memcpy(&octets, &n, 4);
|
|
985
|
+
octets[4] = 0;
|
|
986
|
+
}
|
|
776
987
|
|
|
777
988
|
#ifdef USEARCH_64BIT_ENV
|
|
778
989
|
inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
|
|
@@ -789,22 +1000,38 @@ class usearch_pack_m uint40_t {
|
|
|
789
1000
|
std::memcpy(octets, &n, 5);
|
|
790
1001
|
#else
|
|
791
1002
|
std::memcpy(octets, &n, 4);
|
|
792
|
-
|
|
1003
|
+
octets[4] = 0;
|
|
1004
|
+
#endif // USEARCH_64BIT_ENV
|
|
793
1005
|
}
|
|
794
|
-
#endif
|
|
1006
|
+
#endif // USEARCH_DEFINED_CLANG && USEARCH_DEFINED_APPLE
|
|
795
1007
|
|
|
796
1008
|
inline operator std::size_t() const noexcept {
|
|
797
1009
|
std::size_t result = 0;
|
|
798
1010
|
#ifdef USEARCH_64BIT_ENV
|
|
799
1011
|
std::memcpy(&result, octets, 5);
|
|
800
1012
|
#else
|
|
801
|
-
std::memcpy(&result, octets
|
|
1013
|
+
std::memcpy(&result, octets, 4);
|
|
802
1014
|
#endif
|
|
803
1015
|
return result;
|
|
804
1016
|
}
|
|
805
1017
|
|
|
806
1018
|
inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
|
|
807
1019
|
inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
|
|
1020
|
+
|
|
1021
|
+
inline bool operator==(uint40_t const& other) const noexcept { return std::memcmp(octets, other.octets, 5) == 0; }
|
|
1022
|
+
inline bool operator!=(uint40_t const& other) const noexcept { return !(*this == other); }
|
|
1023
|
+
inline bool operator>(uint40_t const& other) const noexcept { return other < *this; }
|
|
1024
|
+
inline bool operator<=(uint40_t const& other) const noexcept { return !(*this > other); }
|
|
1025
|
+
inline bool operator>=(uint40_t const& other) const noexcept { return !(*this < other); }
|
|
1026
|
+
inline bool operator<(uint40_t const& other) const noexcept {
|
|
1027
|
+
for (int i = 0; i < 5; ++i) {
|
|
1028
|
+
if (octets[4 - i] < other.octets[4 - i])
|
|
1029
|
+
return true;
|
|
1030
|
+
if (octets[4 - i] > other.octets[4 - i])
|
|
1031
|
+
return false;
|
|
1032
|
+
}
|
|
1033
|
+
return false;
|
|
1034
|
+
}
|
|
808
1035
|
};
|
|
809
1036
|
|
|
810
1037
|
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
@@ -813,12 +1040,33 @@ class usearch_pack_m uint40_t {
|
|
|
813
1040
|
|
|
814
1041
|
static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
|
|
815
1042
|
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
1043
|
+
/**
|
|
1044
|
+
* @brief Reflection-helper to get the default "unused" value for a given type.
|
|
1045
|
+
* Needed to initialize hash-sets and bit-sets.
|
|
1046
|
+
*/
|
|
1047
|
+
template <typename element_at> struct default_free_value_gt {
|
|
1048
|
+
template <typename sfinae_element_at = element_at,
|
|
1049
|
+
typename std::enable_if<std::is_integral<sfinae_element_at>::value>::type* = nullptr>
|
|
1050
|
+
static sfinae_element_at value() noexcept {
|
|
1051
|
+
return std::numeric_limits<element_at>::max();
|
|
1052
|
+
}
|
|
1053
|
+
template <typename sfinae_element_at = element_at,
|
|
1054
|
+
typename std::enable_if<!std::is_integral<sfinae_element_at>::value>::type* = nullptr>
|
|
1055
|
+
static sfinae_element_at value() noexcept {
|
|
1056
|
+
return element_at();
|
|
1057
|
+
}
|
|
1058
|
+
};
|
|
1059
|
+
|
|
1060
|
+
template <> struct default_free_value_gt<uint40_t> {
|
|
1061
|
+
static uint40_t value() noexcept { return uint40_t::max(); }
|
|
1062
|
+
};
|
|
1063
|
+
|
|
1064
|
+
template <typename element_at> element_at default_free_value() { return default_free_value_gt<element_at>::value(); }
|
|
821
1065
|
|
|
1066
|
+
/**
|
|
1067
|
+
* @brief Adapter to allow definining arbitrary hash functions for keys and slots.
|
|
1068
|
+
* It's added, as overloading `std::hash` is not recommended by the standard.
|
|
1069
|
+
*/
|
|
822
1070
|
template <typename element_at> struct hash_gt {
|
|
823
1071
|
std::size_t operator()(element_at const& element) const noexcept { return std::hash<element_at>{}(element); }
|
|
824
1072
|
};
|
|
@@ -829,6 +1077,7 @@ template <> struct hash_gt<uint40_t> {
|
|
|
829
1077
|
|
|
830
1078
|
/**
|
|
831
1079
|
* @brief Minimalistic hash-set implementation to track visited nodes during graph traversal.
|
|
1080
|
+
* In our primary usecase, its a sparse alternative to a bit-set.
|
|
832
1081
|
*
|
|
833
1082
|
* It doesn't support deletion of separate objects, but supports `clear`-ing all at once.
|
|
834
1083
|
* It expects `reserve` to be called ahead of all insertions, so no resizes are needed.
|
|
@@ -896,6 +1145,10 @@ class growing_hash_set_gt {
|
|
|
896
1145
|
growing_hash_set_gt(growing_hash_set_gt const&) = delete;
|
|
897
1146
|
growing_hash_set_gt& operator=(growing_hash_set_gt const&) = delete;
|
|
898
1147
|
|
|
1148
|
+
/**
|
|
1149
|
+
* @brief Checks if the element is already in the hash-set.
|
|
1150
|
+
* @return `true` if the element is already in the hash-set.
|
|
1151
|
+
*/
|
|
899
1152
|
inline bool test(element_t const& elem) const noexcept {
|
|
900
1153
|
std::size_t index = hasher_(elem) & (capacity_ - 1);
|
|
901
1154
|
while (slots_[index] != default_free_value<element_t>()) {
|
|
@@ -908,7 +1161,7 @@ class growing_hash_set_gt {
|
|
|
908
1161
|
}
|
|
909
1162
|
|
|
910
1163
|
/**
|
|
911
|
-
*
|
|
1164
|
+
* @brief Inserts an element into the hash-set.
|
|
912
1165
|
* @return Similar to `bitset_gt`, returns the previous value.
|
|
913
1166
|
*/
|
|
914
1167
|
inline bool set(element_t const& elem) noexcept {
|
|
@@ -925,6 +1178,10 @@ class growing_hash_set_gt {
|
|
|
925
1178
|
return false;
|
|
926
1179
|
}
|
|
927
1180
|
|
|
1181
|
+
/**
|
|
1182
|
+
* @brief Extends the capacity of the hash-set.
|
|
1183
|
+
* @return `true` if enough capacity is available, `false` if memory allocation failed.
|
|
1184
|
+
*/
|
|
928
1185
|
bool reserve(std::size_t new_capacity) noexcept {
|
|
929
1186
|
new_capacity = (new_capacity * 5u) / 3u;
|
|
930
1187
|
if (new_capacity <= capacity_)
|
|
@@ -1007,7 +1264,7 @@ class ring_gt {
|
|
|
1007
1264
|
size_t size() const noexcept {
|
|
1008
1265
|
if (empty_)
|
|
1009
1266
|
return 0;
|
|
1010
|
-
else if (head_
|
|
1267
|
+
else if (head_ > tail_)
|
|
1011
1268
|
return head_ - tail_;
|
|
1012
1269
|
else
|
|
1013
1270
|
return capacity_ - (tail_ - head_);
|
|
@@ -1052,7 +1309,9 @@ class ring_gt {
|
|
|
1052
1309
|
return true;
|
|
1053
1310
|
}
|
|
1054
1311
|
|
|
1055
|
-
void push(element_t const& value)
|
|
1312
|
+
void push(element_t const& value) usearch_noexcept_m {
|
|
1313
|
+
usearch_assert_m(capacity() > 0, "Ring buffer is not initialized");
|
|
1314
|
+
usearch_assert_m(size() < capacity(), "Ring buffer is full");
|
|
1056
1315
|
elements_[head_] = value;
|
|
1057
1316
|
head_ = (head_ + 1) % capacity_;
|
|
1058
1317
|
empty_ = false;
|
|
@@ -1060,7 +1319,7 @@ class ring_gt {
|
|
|
1060
1319
|
|
|
1061
1320
|
bool try_push(element_t const& value) noexcept {
|
|
1062
1321
|
if (head_ == tail_ && !empty_)
|
|
1063
|
-
return false; // elements_ is full
|
|
1322
|
+
return false; // `elements_` is full
|
|
1064
1323
|
|
|
1065
1324
|
return push(value);
|
|
1066
1325
|
return true;
|
|
@@ -1113,20 +1372,49 @@ struct index_config_t {
|
|
|
1113
1372
|
std::size_t connectivity_base = default_connectivity() * 2;
|
|
1114
1373
|
|
|
1115
1374
|
inline index_config_t() = default;
|
|
1116
|
-
inline index_config_t(std::size_t c) noexcept
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1375
|
+
inline index_config_t(std::size_t c, std::size_t cb = 0) noexcept : connectivity(c), connectivity_base(cb) {}
|
|
1376
|
+
|
|
1377
|
+
/**
|
|
1378
|
+
* @brief Validates the configuration settings, updating them in-place.
|
|
1379
|
+
* @return Error message, if any.
|
|
1380
|
+
*/
|
|
1381
|
+
inline error_t validate() noexcept {
|
|
1382
|
+
if (connectivity == 0)
|
|
1383
|
+
connectivity = default_connectivity();
|
|
1384
|
+
if (connectivity_base == 0)
|
|
1385
|
+
connectivity_base = connectivity * 2;
|
|
1386
|
+
if (connectivity < 2)
|
|
1387
|
+
return "Connectivity must be at least 2, otherwise the index degenerates into ropes";
|
|
1388
|
+
if (connectivity_base < connectivity)
|
|
1389
|
+
return "Base layer should be at least as connected as the rest of the graph";
|
|
1390
|
+
return {};
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
/**
|
|
1394
|
+
* @brief Immutable function to check if the configuration is valid.
|
|
1395
|
+
* @return `true` if the configuration is valid.
|
|
1396
|
+
*/
|
|
1397
|
+
inline bool is_valid() const noexcept { return connectivity >= 2 && connectivity_base >= connectivity; }
|
|
1120
1398
|
};
|
|
1121
1399
|
|
|
1400
|
+
/**
|
|
1401
|
+
* @brief Growth settings for the index container.
|
|
1402
|
+
* Includes the upper bound for `::members` capacity,
|
|
1403
|
+
* and the number of read/write threads expected to work with the index.
|
|
1404
|
+
*/
|
|
1122
1405
|
struct index_limits_t {
|
|
1406
|
+
/// @brief Maximum number of entries in the index.
|
|
1123
1407
|
std::size_t members = 0;
|
|
1408
|
+
/// @brief Max number of threads simultaneously updating entries.
|
|
1124
1409
|
std::size_t threads_add = std::thread::hardware_concurrency();
|
|
1410
|
+
/// @brief Max number of threads simultaneously searching entries.
|
|
1125
1411
|
std::size_t threads_search = std::thread::hardware_concurrency();
|
|
1126
1412
|
|
|
1127
1413
|
inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
|
|
1128
1414
|
inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
1415
|
+
/// @brief Returns the upper limit for the number of threads.
|
|
1129
1416
|
inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
1417
|
+
/// @brief Returns the concurrency-level of the index - the minimum of thread counts.
|
|
1130
1418
|
inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
1131
1419
|
};
|
|
1132
1420
|
|
|
@@ -1404,8 +1692,10 @@ class input_file_t {
|
|
|
1404
1692
|
serialization_result_t read(void* begin, std::size_t length) noexcept {
|
|
1405
1693
|
serialization_result_t result;
|
|
1406
1694
|
std::size_t read = std::fread(begin, length, 1, file_);
|
|
1407
|
-
if (length && !read)
|
|
1408
|
-
|
|
1695
|
+
if (length && !read) {
|
|
1696
|
+
bool reached_eof = std::feof(file_);
|
|
1697
|
+
return result.failed(reached_eof ? "End of file reached!" : std::strerror(errno));
|
|
1698
|
+
}
|
|
1409
1699
|
return result;
|
|
1410
1700
|
}
|
|
1411
1701
|
void close() noexcept {
|
|
@@ -1563,6 +1853,14 @@ class memory_mapped_file_t {
|
|
|
1563
1853
|
}
|
|
1564
1854
|
};
|
|
1565
1855
|
|
|
1856
|
+
/**
|
|
1857
|
+
* @brief Metadata header for the serialized index.
|
|
1858
|
+
*
|
|
1859
|
+
* This structure is very minimalistic by design. It contains no information
|
|
1860
|
+
* about the capacity of the index, so you'll have to `reserve` after loading.
|
|
1861
|
+
* It also contains no info on the metric or key types, so you'll have to store
|
|
1862
|
+
* that information elsewhere, like we do in `index_dense_head_t`.
|
|
1863
|
+
*/
|
|
1566
1864
|
struct index_serialized_header_t {
|
|
1567
1865
|
std::uint64_t size = 0;
|
|
1568
1866
|
std::uint64_t connectivity = 0;
|
|
@@ -1608,8 +1906,8 @@ template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m)
|
|
|
1608
1906
|
* be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
|
|
1609
1907
|
*
|
|
1610
1908
|
* Unlike most implementations, this one is generic anc can be used for any search,
|
|
1611
|
-
* not just within equi-dimensional vectors. Examples range from
|
|
1612
|
-
* positions.
|
|
1909
|
+
* not just within equi-dimensional vectors. Examples range from Texts to similar Chess
|
|
1910
|
+
* positions, Geo-Spatial Search, and even Graphs.
|
|
1613
1911
|
*
|
|
1614
1912
|
* @tparam key_at
|
|
1615
1913
|
* The type of primary objects stored in the index.
|
|
@@ -1695,6 +1993,7 @@ class index_gt {
|
|
|
1695
1993
|
using dynamic_allocator_t = dynamic_allocator_at;
|
|
1696
1994
|
using tape_allocator_t = tape_allocator_at;
|
|
1697
1995
|
static_assert(sizeof(vector_key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
|
|
1996
|
+
static_assert(std::is_signed<distance_t>::value, "Distance must be a signed type, as we use the unary minus.");
|
|
1698
1997
|
|
|
1699
1998
|
using member_cref_t = member_cref_gt<vector_key_t>;
|
|
1700
1999
|
using member_ref_t = member_ref_gt<vector_key_t>;
|
|
@@ -1705,10 +2004,17 @@ class index_gt {
|
|
|
1705
2004
|
|
|
1706
2005
|
friend class index_gt;
|
|
1707
2006
|
member_iterator_gt() noexcept {}
|
|
1708
|
-
member_iterator_gt(index_t* index,
|
|
2007
|
+
member_iterator_gt(index_t* index, compressed_slot_t slot) noexcept : index_(index), slot_(slot) {}
|
|
2008
|
+
|
|
2009
|
+
template <int> ref_t call_key(std::true_type) const noexcept {
|
|
2010
|
+
return ref_t{index_->node_at_(slot_).ckey(), slot_};
|
|
2011
|
+
}
|
|
2012
|
+
template <int> ref_t call_key(std::false_type) const noexcept {
|
|
2013
|
+
return ref_t{index_->node_at_(slot_).key(), slot_};
|
|
2014
|
+
}
|
|
1709
2015
|
|
|
1710
2016
|
index_t* index_{};
|
|
1711
|
-
|
|
2017
|
+
compressed_slot_t slot_{};
|
|
1712
2018
|
|
|
1713
2019
|
public:
|
|
1714
2020
|
using iterator_category = std::random_access_iterator_tag;
|
|
@@ -1717,22 +2023,21 @@ class index_gt {
|
|
|
1717
2023
|
using pointer = void;
|
|
1718
2024
|
using reference = ref_t;
|
|
1719
2025
|
|
|
1720
|
-
reference operator*() const noexcept { return
|
|
1721
|
-
vector_key_t key() const noexcept { return index_->node_at_(slot_).
|
|
2026
|
+
reference operator*() const noexcept { return call_key<0>(std::is_const<index_t>()); }
|
|
2027
|
+
vector_key_t key() const noexcept { return index_->node_at_(slot_).ckey(); }
|
|
1722
2028
|
|
|
1723
|
-
friend inline
|
|
2029
|
+
friend inline compressed_slot_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
|
|
1724
2030
|
friend inline vector_key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
|
|
1725
2031
|
|
|
1726
|
-
member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
|
|
1727
|
-
member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
|
|
1728
|
-
member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
|
|
1729
|
-
member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
|
|
1730
|
-
|
|
1731
2032
|
// clang-format off
|
|
1732
|
-
member_iterator_gt
|
|
1733
|
-
member_iterator_gt
|
|
1734
|
-
member_iterator_gt
|
|
1735
|
-
member_iterator_gt
|
|
2033
|
+
member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + 1)); }
|
|
2034
|
+
member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - 1)); }
|
|
2035
|
+
member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + d)); }
|
|
2036
|
+
member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - d)); }
|
|
2037
|
+
member_iterator_gt& operator++() noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + 1); return *this; }
|
|
2038
|
+
member_iterator_gt& operator--() noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - 1); return *this; }
|
|
2039
|
+
member_iterator_gt& operator+=(difference_type d) noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + d); return *this; }
|
|
2040
|
+
member_iterator_gt& operator-=(difference_type d) noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - d); return *this; }
|
|
1736
2041
|
bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
|
|
1737
2042
|
bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
|
|
1738
2043
|
// clang-format on
|
|
@@ -1823,8 +2128,10 @@ class index_gt {
|
|
|
1823
2128
|
node_t& operator=(node_t const&) = default;
|
|
1824
2129
|
|
|
1825
2130
|
misaligned_ref_gt<vector_key_t const> ckey() const noexcept { return {tape_}; }
|
|
1826
|
-
misaligned_ref_gt<vector_key_t>
|
|
1827
|
-
misaligned_ref_gt<
|
|
2131
|
+
misaligned_ref_gt<vector_key_t const> ckey() noexcept { return {tape_}; }
|
|
2132
|
+
misaligned_ref_gt<vector_key_t const> key() const noexcept { return {tape_}; }
|
|
2133
|
+
misaligned_ref_gt<vector_key_t> key() noexcept { return {tape_}; }
|
|
2134
|
+
misaligned_ref_gt<level_t> level() noexcept { return {tape_ + sizeof(vector_key_t)}; }
|
|
1828
2135
|
|
|
1829
2136
|
void key(vector_key_t v) noexcept { return misaligned_store<vector_key_t>(tape_, v); }
|
|
1830
2137
|
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(vector_key_t), v); }
|
|
@@ -1842,16 +2149,22 @@ class index_gt {
|
|
|
1842
2149
|
class neighbors_ref_t {
|
|
1843
2150
|
byte_t* tape_;
|
|
1844
2151
|
|
|
1845
|
-
static constexpr std::size_t shift(std::size_t i = 0) {
|
|
2152
|
+
static constexpr std::size_t shift(std::size_t i = 0) noexcept {
|
|
1846
2153
|
return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
|
|
1847
2154
|
}
|
|
1848
2155
|
|
|
1849
2156
|
public:
|
|
2157
|
+
using iterator = misaligned_ptr_gt<compressed_slot_t>;
|
|
2158
|
+
using const_iterator = misaligned_ptr_gt<compressed_slot_t const>;
|
|
2159
|
+
using value_type = compressed_slot_t;
|
|
2160
|
+
|
|
1850
2161
|
neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1851
2162
|
misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
|
|
1852
2163
|
misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
|
|
1853
2164
|
misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
|
|
1854
2165
|
misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
|
|
2166
|
+
misaligned_ptr_gt<compressed_slot_t const> cbegin() noexcept { return tape_ + shift(); }
|
|
2167
|
+
misaligned_ptr_gt<compressed_slot_t const> cend() noexcept { return cbegin() + size(); }
|
|
1855
2168
|
compressed_slot_t operator[](std::size_t i) const noexcept {
|
|
1856
2169
|
return misaligned_load<compressed_slot_t>(tape_ + shift(i));
|
|
1857
2170
|
}
|
|
@@ -1859,7 +2172,7 @@ class index_gt {
|
|
|
1859
2172
|
void clear() noexcept {
|
|
1860
2173
|
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1861
2174
|
std::memset(tape_, 0, shift(n));
|
|
1862
|
-
|
|
2175
|
+
misaligned_store<neighbors_count_t>(tape_, 0);
|
|
1863
2176
|
}
|
|
1864
2177
|
void push_back(compressed_slot_t slot) noexcept {
|
|
1865
2178
|
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
@@ -1879,29 +2192,55 @@ class index_gt {
|
|
|
1879
2192
|
visits_hash_set_t visits{};
|
|
1880
2193
|
std::default_random_engine level_generator{};
|
|
1881
2194
|
std::size_t iteration_cycles{};
|
|
1882
|
-
std::size_t
|
|
2195
|
+
std::size_t computed_distances{};
|
|
2196
|
+
std::size_t computed_distances_in_refines{};
|
|
2197
|
+
std::size_t computed_distances_in_reverse_refines{};
|
|
1883
2198
|
|
|
2199
|
+
/// @brief Heterogeneous distance calculation.
|
|
1884
2200
|
template <typename value_at, typename metric_at, typename entry_at> //
|
|
1885
2201
|
inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1886
2202
|
static_assert( //
|
|
1887
2203
|
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1888
2204
|
"Unexpected type");
|
|
1889
2205
|
|
|
1890
|
-
|
|
2206
|
+
computed_distances++;
|
|
1891
2207
|
return metric(first, second);
|
|
1892
2208
|
}
|
|
1893
2209
|
|
|
2210
|
+
/// @brief Homogeneous distance calculation.
|
|
1894
2211
|
template <typename metric_at, typename entry_at> //
|
|
1895
2212
|
inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1896
2213
|
static_assert( //
|
|
1897
2214
|
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1898
2215
|
"Unexpected type");
|
|
1899
2216
|
|
|
1900
|
-
|
|
2217
|
+
computed_distances++;
|
|
1901
2218
|
return metric(first, second);
|
|
1902
2219
|
}
|
|
2220
|
+
|
|
2221
|
+
/// @brief Heterogeneous batch distance calculation.
|
|
2222
|
+
template <typename value_at, typename metric_at, typename entries_at, typename candidate_allowed_at,
|
|
2223
|
+
typename transform_at,
|
|
2224
|
+
typename callback_at> //
|
|
2225
|
+
inline void measure_batch(value_at const& first, entries_at const& second_entries, metric_at&& metric,
|
|
2226
|
+
candidate_allowed_at&& candidate_allowed, transform_at&& transform,
|
|
2227
|
+
callback_at&& callback) noexcept {
|
|
2228
|
+
|
|
2229
|
+
using entry_t = typename std::remove_reference<decltype(second_entries[0])>::type;
|
|
2230
|
+
metric.batch(first, second_entries, candidate_allowed, transform,
|
|
2231
|
+
[&](entry_t const& entry, distance_t distance) {
|
|
2232
|
+
callback(entry, distance);
|
|
2233
|
+
computed_distances++;
|
|
2234
|
+
});
|
|
2235
|
+
}
|
|
1903
2236
|
};
|
|
1904
2237
|
|
|
2238
|
+
/// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
|
|
2239
|
+
mutable std::atomic<std::size_t> nodes_capacity_{};
|
|
2240
|
+
|
|
2241
|
+
/// @brief Number of "slots" already storing non-null nodes.
|
|
2242
|
+
mutable std::atomic<std::size_t> nodes_count_{};
|
|
2243
|
+
|
|
1905
2244
|
index_config_t config_{};
|
|
1906
2245
|
index_limits_t limits_{};
|
|
1907
2246
|
|
|
@@ -1911,12 +2250,6 @@ class index_gt {
|
|
|
1911
2250
|
precomputed_constants_t pre_{};
|
|
1912
2251
|
memory_mapped_file_t viewed_file_{};
|
|
1913
2252
|
|
|
1914
|
-
/// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
|
|
1915
|
-
usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
|
|
1916
|
-
|
|
1917
|
-
/// @brief Number of "slots" already storing non-null nodes.
|
|
1918
|
-
usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
|
|
1919
|
-
|
|
1920
2253
|
/// @brief Controls access to `max_level_` and `entry_slot_`.
|
|
1921
2254
|
/// If any thread is updating those values, no other threads can `add()` or `search()`.
|
|
1922
2255
|
std::mutex global_mutex_{};
|
|
@@ -1929,7 +2262,7 @@ class index_gt {
|
|
|
1929
2262
|
|
|
1930
2263
|
using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
|
|
1931
2264
|
|
|
1932
|
-
/// @brief C-style array of `node_t` smart-pointers.
|
|
2265
|
+
/// @brief C-style array of `node_t` smart-pointers. Use `compressed_slot_t` for indexing.
|
|
1933
2266
|
buffer_gt<node_t, nodes_allocator_t> nodes_{};
|
|
1934
2267
|
|
|
1935
2268
|
/// @brief Mutex, that limits concurrent access to `nodes_`.
|
|
@@ -1948,17 +2281,33 @@ class index_gt {
|
|
|
1948
2281
|
index_config_t const& config() const noexcept { return config_; }
|
|
1949
2282
|
index_limits_t const& limits() const noexcept { return limits_; }
|
|
1950
2283
|
bool is_immutable() const noexcept { return bool(viewed_file_); }
|
|
2284
|
+
explicit operator bool() const noexcept { return config_.is_valid(); }
|
|
1951
2285
|
|
|
1952
2286
|
/**
|
|
2287
|
+
* @brief Default index constructor, suitable only for stateless allocators.
|
|
2288
|
+
* @warning Consider `index_gt::make` instead, or explicitly convert to `bool` to check if the index is valid.
|
|
1953
2289
|
* @section Exceptions
|
|
1954
|
-
* Doesn't throw, unless the ::
|
|
2290
|
+
* Doesn't throw, unless the ::dynamic_allocator's and ::tape_allocator's throw on move-construction.
|
|
1955
2291
|
*/
|
|
1956
2292
|
explicit index_gt( //
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
2293
|
+
dynamic_allocator_t dynamic_allocator = {}, tape_allocator_t tape_allocator = {}) noexcept(false)
|
|
2294
|
+
: nodes_capacity_(0u), nodes_count_(0u), config_(), limits_(0, 0),
|
|
2295
|
+
dynamic_allocator_(std::move(dynamic_allocator)), tape_allocator_(std::move(tape_allocator)),
|
|
2296
|
+
pre_(precompute_({})), max_level_(-1), entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
|
|
2297
|
+
|
|
2298
|
+
/**
|
|
2299
|
+
* @brief Default index constructor, suitable only for stateless allocators.
|
|
2300
|
+
* @warning Consider `index_gt::make` instead, or explicitly convert to `bool` to check if the index is valid.
|
|
2301
|
+
* @section Exceptions
|
|
2302
|
+
* Doesn't throw, unless the ::dynamic_allocator's and ::tape_allocator's throw on move-construction.
|
|
2303
|
+
*/
|
|
2304
|
+
explicit index_gt(index_config_t config, dynamic_allocator_t dynamic_allocator = {},
|
|
2305
|
+
tape_allocator_t tape_allocator = {}) noexcept(false)
|
|
2306
|
+
: index_gt(dynamic_allocator, tape_allocator) {
|
|
2307
|
+
config.validate();
|
|
2308
|
+
config_ = config;
|
|
2309
|
+
pre_ = precompute_(config);
|
|
2310
|
+
}
|
|
1962
2311
|
|
|
1963
2312
|
/**
|
|
1964
2313
|
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
@@ -1974,17 +2323,56 @@ class index_gt {
|
|
|
1974
2323
|
return *this;
|
|
1975
2324
|
}
|
|
1976
2325
|
|
|
1977
|
-
struct
|
|
1978
|
-
error_t error;
|
|
2326
|
+
struct state_result_t {
|
|
1979
2327
|
index_gt index;
|
|
2328
|
+
error_t error;
|
|
1980
2329
|
|
|
1981
2330
|
explicit operator bool() const noexcept { return !error; }
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
2331
|
+
state_result_t failed(error_t message) noexcept { return {std::move(index), std::move(message)}; }
|
|
2332
|
+
operator index_gt&&() && {
|
|
2333
|
+
if (error)
|
|
2334
|
+
__usearch_raise_runtime_error(error.what());
|
|
2335
|
+
return std::move(index);
|
|
1985
2336
|
}
|
|
1986
2337
|
};
|
|
2338
|
+
using copy_result_t = state_result_t;
|
|
1987
2339
|
|
|
2340
|
+
/**
|
|
2341
|
+
* @brief The recommended way to initialize the index, as unlike the constructor,
|
|
2342
|
+
* it can fail with an error message, without raising an exception.
|
|
2343
|
+
*
|
|
2344
|
+
* @param[in] config The configuration specs of the index.
|
|
2345
|
+
* @param[in] dynamic_allocator The allocator for temporary buffers and thread contexts, like priority queues.
|
|
2346
|
+
* @param[in] tape_allocator The allocator for the primary allocations of nodes and vectors.
|
|
2347
|
+
*/
|
|
2348
|
+
static state_result_t make( //
|
|
2349
|
+
index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
|
|
2350
|
+
tape_allocator_t tape_allocator = {}) noexcept {
|
|
2351
|
+
|
|
2352
|
+
state_result_t result;
|
|
2353
|
+
result.error = config.validate();
|
|
2354
|
+
if (result.error)
|
|
2355
|
+
return result;
|
|
2356
|
+
|
|
2357
|
+
index_gt index;
|
|
2358
|
+
index.config_ = std::move(config);
|
|
2359
|
+
index.dynamic_allocator_ = std::move(dynamic_allocator);
|
|
2360
|
+
index.tape_allocator_ = std::move(tape_allocator);
|
|
2361
|
+
index.pre_ = precompute_(index.config_);
|
|
2362
|
+
index.nodes_count_ = 0u;
|
|
2363
|
+
index.max_level_ = -1;
|
|
2364
|
+
index.entry_slot_ = 0u;
|
|
2365
|
+
|
|
2366
|
+
result.index = std::move(index);
|
|
2367
|
+
return result;
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
/**
|
|
2371
|
+
* @brief The recommended way to copy the index, as unlike the copy-constructor,
|
|
2372
|
+
* it can fail with an error message, without raising an exception.
|
|
2373
|
+
*
|
|
2374
|
+
* @param[in] config The configuration specs for the copy-operation. Currently unused.
|
|
2375
|
+
*/
|
|
1988
2376
|
copy_result_t copy(index_copy_config_t config = {}) const noexcept {
|
|
1989
2377
|
copy_result_t result;
|
|
1990
2378
|
index_gt& other = result.index;
|
|
@@ -2006,22 +2394,24 @@ class index_gt {
|
|
|
2006
2394
|
return result;
|
|
2007
2395
|
}
|
|
2008
2396
|
|
|
2009
|
-
member_citerator_t cbegin() const noexcept { return {this,
|
|
2010
|
-
member_citerator_t cend() const noexcept { return {this, size()}; }
|
|
2011
|
-
member_citerator_t begin() const noexcept { return {this,
|
|
2012
|
-
member_citerator_t end() const noexcept { return {this, size()}; }
|
|
2013
|
-
member_iterator_t begin() noexcept { return {this,
|
|
2014
|
-
member_iterator_t end() noexcept { return {this, size()}; }
|
|
2397
|
+
member_citerator_t cbegin() const noexcept { return {this, static_cast<compressed_slot_t>(0u)}; }
|
|
2398
|
+
member_citerator_t cend() const noexcept { return {this, static_cast<compressed_slot_t>(size())}; }
|
|
2399
|
+
member_citerator_t begin() const noexcept { return {this, static_cast<compressed_slot_t>(0u)}; }
|
|
2400
|
+
member_citerator_t end() const noexcept { return {this, static_cast<compressed_slot_t>(size())}; }
|
|
2401
|
+
member_iterator_t begin() noexcept { return {this, static_cast<compressed_slot_t>(0u)}; }
|
|
2402
|
+
member_iterator_t end() noexcept { return {this, static_cast<compressed_slot_t>(size())}; }
|
|
2015
2403
|
|
|
2016
|
-
member_ref_t at(
|
|
2017
|
-
member_cref_t at(
|
|
2018
|
-
member_iterator_t iterator_at(
|
|
2019
|
-
member_citerator_t citerator_at(
|
|
2404
|
+
member_ref_t at(compressed_slot_t slot) noexcept { return {nodes_[slot].key(), slot}; }
|
|
2405
|
+
member_cref_t at(compressed_slot_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
|
|
2406
|
+
member_iterator_t iterator_at(compressed_slot_t slot) noexcept { return {this, slot}; }
|
|
2407
|
+
member_citerator_t citerator_at(compressed_slot_t slot) const noexcept { return {this, slot}; }
|
|
2020
2408
|
|
|
2021
2409
|
dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
|
|
2022
2410
|
tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
|
|
2023
2411
|
|
|
2412
|
+
#if defined(USEARCH_USE_PRAGMA_REGION)
|
|
2024
2413
|
#pragma region Adjusting Configuration
|
|
2414
|
+
#endif
|
|
2025
2415
|
|
|
2026
2416
|
/**
|
|
2027
2417
|
* @brief Erases all the vectors from the index.
|
|
@@ -2089,13 +2479,20 @@ class index_gt {
|
|
|
2089
2479
|
* @brief Increases the `capacity()` of the index to allow adding more vectors.
|
|
2090
2480
|
* @return `true` on success, `false` on memory allocation errors.
|
|
2091
2481
|
*/
|
|
2092
|
-
bool
|
|
2482
|
+
bool try_reserve(index_limits_t limits) usearch_noexcept_m {
|
|
2093
2483
|
|
|
2094
2484
|
if (limits.threads_add <= limits_.threads_add //
|
|
2095
2485
|
&& limits.threads_search <= limits_.threads_search //
|
|
2096
2486
|
&& limits.members <= limits_.members)
|
|
2097
2487
|
return true;
|
|
2098
2488
|
|
|
2489
|
+
// In some cases, we don't want to update the number of members,
|
|
2490
|
+
// just want to make sure that future reserves use the new thread limits.
|
|
2491
|
+
if (!limits.members && !size()) {
|
|
2492
|
+
limits_ = limits;
|
|
2493
|
+
return true;
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2099
2496
|
nodes_mutexes_t new_mutexes(limits.members);
|
|
2100
2497
|
buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
|
|
2101
2498
|
buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
|
|
@@ -2114,16 +2511,27 @@ class index_gt {
|
|
|
2114
2511
|
return true;
|
|
2115
2512
|
}
|
|
2116
2513
|
|
|
2514
|
+
/**
|
|
2515
|
+
* @brief Increases the `capacity()` of the index to allow adding more vectors.
|
|
2516
|
+
* @warning Unlike STL, won't throw exceptions on memory allocations, so check the return value.
|
|
2517
|
+
* @return `true` on success, `false` on memory allocation errors.
|
|
2518
|
+
*/
|
|
2519
|
+
bool reserve(index_limits_t limits) usearch_noexcept_m { return try_reserve(limits); }
|
|
2520
|
+
|
|
2521
|
+
#if defined(USEARCH_USE_PRAGMA_REGION)
|
|
2117
2522
|
#pragma endregion
|
|
2118
2523
|
|
|
2119
2524
|
#pragma region Construction and Search
|
|
2525
|
+
#endif
|
|
2120
2526
|
|
|
2121
2527
|
struct add_result_t {
|
|
2122
2528
|
error_t error{};
|
|
2123
2529
|
std::size_t new_size{};
|
|
2124
2530
|
std::size_t visited_members{};
|
|
2125
2531
|
std::size_t computed_distances{};
|
|
2126
|
-
std::size_t
|
|
2532
|
+
std::size_t computed_distances_in_refines{};
|
|
2533
|
+
std::size_t computed_distances_in_reverse_refines{};
|
|
2534
|
+
compressed_slot_t slot{};
|
|
2127
2535
|
|
|
2128
2536
|
explicit operator bool() const noexcept { return !error; }
|
|
2129
2537
|
add_result_t failed(error_t message) noexcept {
|
|
@@ -2168,15 +2576,15 @@ class index_gt {
|
|
|
2168
2576
|
top_candidates_t const* top_{};
|
|
2169
2577
|
|
|
2170
2578
|
friend class index_gt;
|
|
2171
|
-
inline search_result_t(index_gt const& index, top_candidates_t
|
|
2172
|
-
: nodes_(index.nodes_), top_(
|
|
2579
|
+
inline search_result_t(index_gt const& index, top_candidates_t const* top) noexcept
|
|
2580
|
+
: nodes_(index.nodes_), top_(top) {}
|
|
2173
2581
|
|
|
2174
2582
|
public:
|
|
2175
|
-
/**
|
|
2583
|
+
/** @brief Number of search results found. */
|
|
2176
2584
|
std::size_t count{};
|
|
2177
|
-
/**
|
|
2585
|
+
/** @brief Number of graph nodes traversed. */
|
|
2178
2586
|
std::size_t visited_members{};
|
|
2179
|
-
/**
|
|
2587
|
+
/** @brief Number of times the distances were computed. */
|
|
2180
2588
|
std::size_t computed_distances{};
|
|
2181
2589
|
error_t error{};
|
|
2182
2590
|
|
|
@@ -2310,57 +2718,76 @@ class index_gt {
|
|
|
2310
2718
|
|
|
2311
2719
|
// Determining how much memory to allocate for the node depends on the target level
|
|
2312
2720
|
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
2313
|
-
level_t max_level_copy = max_level_;
|
|
2314
|
-
|
|
2315
|
-
level_t
|
|
2721
|
+
level_t max_level_copy = max_level_; // Copy under lock
|
|
2722
|
+
compressed_slot_t entry_slot_copy = static_cast<compressed_slot_t>(entry_slot_); // Copy under lock
|
|
2723
|
+
level_t new_target_level = choose_random_level_(context.level_generator);
|
|
2316
2724
|
|
|
2317
2725
|
// Make sure we are not overflowing
|
|
2318
2726
|
std::size_t capacity = nodes_capacity_.load();
|
|
2319
|
-
std::size_t
|
|
2320
|
-
if (
|
|
2727
|
+
std::size_t old_size = nodes_count_.fetch_add(1);
|
|
2728
|
+
if (old_size >= capacity) {
|
|
2321
2729
|
nodes_count_.fetch_sub(1);
|
|
2322
2730
|
return result.failed("Reserve capacity ahead of insertions!");
|
|
2323
2731
|
}
|
|
2324
2732
|
|
|
2325
2733
|
// Allocate the neighbors
|
|
2326
|
-
node_t
|
|
2327
|
-
if (!
|
|
2734
|
+
node_t new_node = node_make_(key, new_target_level);
|
|
2735
|
+
if (!new_node) {
|
|
2328
2736
|
nodes_count_.fetch_sub(1);
|
|
2329
2737
|
return result.failed("Out of memory!");
|
|
2330
2738
|
}
|
|
2331
|
-
if (
|
|
2739
|
+
if (new_target_level <= max_level_copy)
|
|
2332
2740
|
new_level_lock.unlock();
|
|
2333
2741
|
|
|
2334
|
-
nodes_[
|
|
2335
|
-
result.new_size =
|
|
2336
|
-
result.slot =
|
|
2337
|
-
callback(at(
|
|
2338
|
-
node_lock_t new_lock = node_lock_(new_slot);
|
|
2742
|
+
nodes_[old_size] = new_node;
|
|
2743
|
+
result.new_size = old_size + 1;
|
|
2744
|
+
compressed_slot_t new_slot = result.slot = static_cast<compressed_slot_t>(old_size);
|
|
2745
|
+
callback(at(result.slot));
|
|
2339
2746
|
|
|
2340
2747
|
// Do nothing for the first element
|
|
2341
|
-
if (!
|
|
2342
|
-
entry_slot_ =
|
|
2343
|
-
max_level_ =
|
|
2748
|
+
if (!old_size) {
|
|
2749
|
+
entry_slot_ = result.slot;
|
|
2750
|
+
max_level_ = new_target_level;
|
|
2344
2751
|
return result;
|
|
2345
2752
|
}
|
|
2346
2753
|
|
|
2347
2754
|
// Pull stats
|
|
2348
|
-
result.computed_distances = context.
|
|
2755
|
+
result.computed_distances = context.computed_distances;
|
|
2756
|
+
result.computed_distances_in_refines = context.computed_distances_in_refines;
|
|
2757
|
+
result.computed_distances_in_reverse_refines = context.computed_distances_in_reverse_refines;
|
|
2349
2758
|
result.visited_members = context.iteration_cycles;
|
|
2350
2759
|
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2760
|
+
// Go down the level, tracking only the closest match
|
|
2761
|
+
compressed_slot_t closest_slot = search_for_one_( //
|
|
2762
|
+
value, metric, prefetch, //
|
|
2763
|
+
entry_slot_copy, max_level_copy, new_target_level, context);
|
|
2764
|
+
|
|
2765
|
+
// From `new_target_level` down - perform proper extensive search
|
|
2766
|
+
for (level_t level = (std::min)(new_target_level, max_level_copy); level >= 0; --level) {
|
|
2767
|
+
// TODO: Handle out of memory conditions
|
|
2768
|
+
search_to_insert_(value, metric, prefetch, closest_slot, level, config.expansion, context);
|
|
2769
|
+
candidates_view_t closest_view;
|
|
2770
|
+
{
|
|
2771
|
+
node_lock_t new_lock = node_lock_(new_slot);
|
|
2772
|
+
neighbors_(new_node, level).clear();
|
|
2773
|
+
closest_view = form_links_to_closest_(metric, new_slot, level, context);
|
|
2774
|
+
closest_slot = closest_view[0].slot;
|
|
2775
|
+
}
|
|
2776
|
+
form_reverse_links_(metric, new_slot, closest_view, value, level, context);
|
|
2777
|
+
}
|
|
2355
2778
|
|
|
2356
2779
|
// Normalize stats
|
|
2357
|
-
result.computed_distances = context.
|
|
2780
|
+
result.computed_distances = context.computed_distances - result.computed_distances;
|
|
2781
|
+
result.computed_distances_in_refines =
|
|
2782
|
+
context.computed_distances_in_refines - result.computed_distances_in_refines;
|
|
2783
|
+
result.computed_distances_in_reverse_refines =
|
|
2784
|
+
context.computed_distances_in_reverse_refines - result.computed_distances_in_reverse_refines;
|
|
2358
2785
|
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2359
2786
|
|
|
2360
2787
|
// Updating the entry point if needed
|
|
2361
|
-
if (
|
|
2788
|
+
if (new_target_level > max_level_copy) {
|
|
2362
2789
|
entry_slot_ = new_slot;
|
|
2363
|
-
max_level_ =
|
|
2790
|
+
max_level_ = new_target_level;
|
|
2364
2791
|
}
|
|
2365
2792
|
return result;
|
|
2366
2793
|
}
|
|
@@ -2368,6 +2795,10 @@ class index_gt {
|
|
|
2368
2795
|
/**
|
|
2369
2796
|
* @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
|
|
2370
2797
|
*
|
|
2798
|
+
* ! It's assumed that different threads aren't updating the same entry at the same time.
|
|
2799
|
+
* ! The state won't be corrupted, but no transactional guarantees are provided and the
|
|
2800
|
+
* ! resulting value & neighbors list may be inconsistent.
|
|
2801
|
+
*
|
|
2371
2802
|
* @tparam metric_at
|
|
2372
2803
|
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2373
2804
|
* It should be callable into distinctly different scenarios:
|
|
@@ -2399,9 +2830,13 @@ class index_gt {
|
|
|
2399
2830
|
callback_at&& callback = callback_at{}, //
|
|
2400
2831
|
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2401
2832
|
|
|
2833
|
+
// Someone is gonna fuzz this, so let's make sure we cover the basics
|
|
2834
|
+
if (!config.expansion)
|
|
2835
|
+
config.expansion = default_expansion_add();
|
|
2836
|
+
|
|
2402
2837
|
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
2403
2838
|
add_result_t result;
|
|
2404
|
-
|
|
2839
|
+
compressed_slot_t updated_slot = iterator.slot_;
|
|
2405
2840
|
|
|
2406
2841
|
// Make sure we have enough local memory to perform this request
|
|
2407
2842
|
context_t& context = contexts_[config.thread];
|
|
@@ -2419,30 +2854,60 @@ class index_gt {
|
|
|
2419
2854
|
if (!next.reserve(config.expansion))
|
|
2420
2855
|
return result.failed("Out of memory!");
|
|
2421
2856
|
|
|
2422
|
-
|
|
2423
|
-
|
|
2857
|
+
node_t updated_node = node_at_(updated_slot);
|
|
2858
|
+
level_t updated_node_level = updated_node.level();
|
|
2424
2859
|
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2860
|
+
// Copy entry coordinates under locks
|
|
2861
|
+
level_t max_level_copy;
|
|
2862
|
+
compressed_slot_t entry_slot_copy;
|
|
2863
|
+
{
|
|
2864
|
+
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
2865
|
+
max_level_copy = max_level_; // Copy under lock
|
|
2866
|
+
entry_slot_copy = static_cast<compressed_slot_t>(entry_slot_); // Copy under lock
|
|
2867
|
+
}
|
|
2429
2868
|
|
|
2430
2869
|
// Pull stats
|
|
2431
|
-
result.computed_distances = context.
|
|
2870
|
+
result.computed_distances = context.computed_distances;
|
|
2432
2871
|
result.visited_members = context.iteration_cycles;
|
|
2433
2872
|
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2873
|
+
// If we are updating the entry node itself, it won't contain any neighbors,
|
|
2874
|
+
// so we should traverse a level down to find the closest match.
|
|
2875
|
+
if (updated_node_level == max_level_copy)
|
|
2876
|
+
updated_node_level--;
|
|
2877
|
+
|
|
2878
|
+
// Go down the level, tracking only the closest match;
|
|
2879
|
+
// It may even be equal to the `updated_slot`
|
|
2880
|
+
compressed_slot_t closest_slot = search_for_one_( //
|
|
2881
|
+
value, metric, prefetch, //
|
|
2882
|
+
entry_slot_copy, max_level_copy, updated_node_level, context);
|
|
2883
|
+
|
|
2884
|
+
// From `updated_node_level` down - perform proper extensive search
|
|
2885
|
+
for (level_t level = (std::min)(updated_node_level, max_level_copy); level >= 0; --level) {
|
|
2886
|
+
if (!search_to_update_(value, metric, prefetch, closest_slot, updated_slot, level, config.expansion,
|
|
2887
|
+
context))
|
|
2888
|
+
return result.failed("Out of memory!");
|
|
2889
|
+
|
|
2890
|
+
candidates_view_t closest_view;
|
|
2891
|
+
{
|
|
2892
|
+
node_lock_t updated_lock = node_lock_(updated_slot);
|
|
2893
|
+
// TODO: Go through existing neighbors removing reverse links
|
|
2894
|
+
// for (compressed_slot_t slot : neighbors_(updated_node, level))
|
|
2895
|
+
// remove_link_(slot, updated_slot, level);
|
|
2896
|
+
neighbors_(updated_node, level).clear();
|
|
2897
|
+
closest_view = form_links_to_closest_(metric, updated_slot, level, context);
|
|
2898
|
+
if (closest_view.size())
|
|
2899
|
+
closest_slot = closest_view[0].slot;
|
|
2900
|
+
}
|
|
2901
|
+
form_reverse_links_(metric, updated_slot, closest_view, value, level, context);
|
|
2902
|
+
}
|
|
2903
|
+
updated_node.key(key);
|
|
2439
2904
|
|
|
2440
2905
|
// Normalize stats
|
|
2441
|
-
result.computed_distances = context.
|
|
2906
|
+
result.computed_distances = context.computed_distances - result.computed_distances;
|
|
2442
2907
|
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2443
|
-
result.slot =
|
|
2908
|
+
result.slot = updated_slot;
|
|
2444
2909
|
|
|
2445
|
-
callback(at(
|
|
2910
|
+
callback(at(updated_slot));
|
|
2446
2911
|
return result;
|
|
2447
2912
|
}
|
|
2448
2913
|
|
|
@@ -2467,16 +2932,28 @@ class index_gt {
|
|
|
2467
2932
|
metric_at&& metric, //
|
|
2468
2933
|
index_search_config_t config = {}, //
|
|
2469
2934
|
predicate_at&& predicate = predicate_at{}, //
|
|
2470
|
-
prefetch_at&& prefetch = prefetch_at{}) const
|
|
2935
|
+
prefetch_at&& prefetch = prefetch_at{}) const usearch_noexcept_m {
|
|
2471
2936
|
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2937
|
+
// Someone is gonna fuzz this, so let's make sure we cover the basics
|
|
2938
|
+
if (!wanted)
|
|
2939
|
+
return search_result_t{};
|
|
2940
|
+
|
|
2941
|
+
// Expansion factor set to zero is equivalent to the default value
|
|
2942
|
+
if (!config.expansion)
|
|
2943
|
+
config.expansion = default_expansion_search();
|
|
2944
|
+
|
|
2945
|
+
// Using references is cleaner, but would result in UBSan false positives
|
|
2946
|
+
context_t* context_ptr = contexts_.data() + config.thread;
|
|
2947
|
+
top_candidates_t* top_ptr = context_ptr ? &context_ptr->top_candidates : nullptr;
|
|
2948
|
+
search_result_t result{*this, top_ptr};
|
|
2949
|
+
if (!nodes_count_.load(std::memory_order_relaxed))
|
|
2476
2950
|
return result;
|
|
2477
2951
|
|
|
2952
|
+
usearch_assert_m(contexts_.size() > config.thread, "Thread index out of bounds");
|
|
2953
|
+
context_t& context = *context_ptr;
|
|
2954
|
+
top_candidates_t& top = *top_ptr;
|
|
2478
2955
|
// Go down the level, tracking only the closest match
|
|
2479
|
-
result.computed_distances = context.
|
|
2956
|
+
result.computed_distances = context.computed_distances;
|
|
2480
2957
|
result.visited_members = context.iteration_cycles;
|
|
2481
2958
|
|
|
2482
2959
|
if (config.exact) {
|
|
@@ -2486,12 +2963,14 @@ class index_gt {
|
|
|
2486
2963
|
} else {
|
|
2487
2964
|
next_candidates_t& next = context.next_candidates;
|
|
2488
2965
|
std::size_t expansion = (std::max)(config.expansion, wanted);
|
|
2966
|
+
usearch_assert_m(expansion > 0, "Expansion factor can't be a zero!");
|
|
2489
2967
|
if (!next.reserve(expansion))
|
|
2490
2968
|
return result.failed("Out of memory!");
|
|
2491
2969
|
if (!top.reserve(expansion))
|
|
2492
2970
|
return result.failed("Out of memory!");
|
|
2493
2971
|
|
|
2494
|
-
|
|
2972
|
+
compressed_slot_t closest_slot = search_for_one_(
|
|
2973
|
+
query, metric, prefetch, static_cast<compressed_slot_t>(entry_slot_), max_level_, 0, context);
|
|
2495
2974
|
|
|
2496
2975
|
// For bottom layer we need a more optimized procedure
|
|
2497
2976
|
if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
|
|
@@ -2502,7 +2981,7 @@ class index_gt {
|
|
|
2502
2981
|
top.shrink(wanted);
|
|
2503
2982
|
|
|
2504
2983
|
// Normalize stats
|
|
2505
|
-
result.computed_distances = context.
|
|
2984
|
+
result.computed_distances = context.computed_distances - result.computed_distances;
|
|
2506
2985
|
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2507
2986
|
result.count = top.size();
|
|
2508
2987
|
return result;
|
|
@@ -2537,7 +3016,7 @@ class index_gt {
|
|
|
2537
3016
|
return result.failed("No clusters to identify");
|
|
2538
3017
|
|
|
2539
3018
|
// Go down the level, tracking only the closest match
|
|
2540
|
-
result.computed_distances = context.
|
|
3019
|
+
result.computed_distances = context.computed_distances;
|
|
2541
3020
|
result.visited_members = context.iteration_cycles;
|
|
2542
3021
|
|
|
2543
3022
|
next_candidates_t& next = context.next_candidates;
|
|
@@ -2545,21 +3024,24 @@ class index_gt {
|
|
|
2545
3024
|
if (!next.reserve(expansion))
|
|
2546
3025
|
return result.failed("Out of memory!");
|
|
2547
3026
|
|
|
2548
|
-
result.cluster.member =
|
|
2549
|
-
|
|
3027
|
+
result.cluster.member =
|
|
3028
|
+
at(search_for_one_(query, metric, prefetch, static_cast<compressed_slot_t>(entry_slot_), max_level_,
|
|
3029
|
+
static_cast<level_t>(level <= 0 ? 0 : level - 1), context));
|
|
2550
3030
|
result.cluster.distance = context.measure(query, result.cluster.member, metric);
|
|
2551
3031
|
|
|
2552
3032
|
// Normalize stats
|
|
2553
|
-
result.computed_distances = context.
|
|
3033
|
+
result.computed_distances = context.computed_distances - result.computed_distances;
|
|
2554
3034
|
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2555
3035
|
|
|
2556
3036
|
(void)predicate;
|
|
2557
3037
|
return result;
|
|
2558
3038
|
}
|
|
2559
3039
|
|
|
3040
|
+
#if defined(USEARCH_USE_PRAGMA_REGION)
|
|
2560
3041
|
#pragma endregion
|
|
2561
3042
|
|
|
2562
3043
|
#pragma region Metadata
|
|
3044
|
+
#endif
|
|
2563
3045
|
|
|
2564
3046
|
struct stats_t {
|
|
2565
3047
|
std::size_t nodes{};
|
|
@@ -2568,6 +3050,9 @@ class index_gt {
|
|
|
2568
3050
|
std::size_t allocated_bytes{};
|
|
2569
3051
|
};
|
|
2570
3052
|
|
|
3053
|
+
/**
|
|
3054
|
+
* @brief Aggregates stats on the number of nodes, edges, and memory usage across all levels.
|
|
3055
|
+
*/
|
|
2571
3056
|
stats_t stats() const noexcept {
|
|
2572
3057
|
stats_t result{};
|
|
2573
3058
|
|
|
@@ -2586,10 +3071,17 @@ class index_gt {
|
|
|
2586
3071
|
return result;
|
|
2587
3072
|
}
|
|
2588
3073
|
|
|
3074
|
+
/**
|
|
3075
|
+
* @brief Aggregates stats on the number of nodes, edges, and memory usage up to a specific level.
|
|
3076
|
+
*
|
|
3077
|
+
* The `level` parameter is zero-based, where `0` is the base level.
|
|
3078
|
+
* For example, `level=1` will include the base level and the first level of connections.
|
|
3079
|
+
*/
|
|
2589
3080
|
stats_t stats(std::size_t level) const noexcept {
|
|
2590
3081
|
stats_t result{};
|
|
2591
|
-
|
|
2592
3082
|
std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
|
|
3083
|
+
std::size_t max_edges_per_node = !level ? config_.connectivity_base : config_.connectivity;
|
|
3084
|
+
|
|
2593
3085
|
for (std::size_t i = 0; i != size(); ++i) {
|
|
2594
3086
|
node_t node = node_at_(i);
|
|
2595
3087
|
if (static_cast<std::size_t>(node.level()) < level)
|
|
@@ -2600,11 +3092,17 @@ class index_gt {
|
|
|
2600
3092
|
result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
|
|
2601
3093
|
}
|
|
2602
3094
|
|
|
2603
|
-
std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
|
|
2604
3095
|
result.max_edges = result.nodes * max_edges_per_node;
|
|
2605
3096
|
return result;
|
|
2606
3097
|
}
|
|
2607
3098
|
|
|
3099
|
+
/**
|
|
3100
|
+
* @brief Aggregates stats on the number of nodes, edges, and memory usage up to a specific level,
|
|
3101
|
+
* simultaneously exporting the stats for each level into the `stats_per_level` C-style array.
|
|
3102
|
+
*
|
|
3103
|
+
* The `max_level` parameter is zero-based, where `0` is the base level.
|
|
3104
|
+
* For example, `max_level=1` will include the base level and the first level of connections.
|
|
3105
|
+
*/
|
|
2608
3106
|
stats_t stats(stats_t* stats_per_level, std::size_t max_level) const noexcept {
|
|
2609
3107
|
|
|
2610
3108
|
std::size_t head_bytes = node_head_bytes_();
|
|
@@ -2663,9 +3161,11 @@ class index_gt {
|
|
|
2663
3161
|
|
|
2664
3162
|
std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
|
|
2665
3163
|
|
|
3164
|
+
#if defined(USEARCH_USE_PRAGMA_REGION)
|
|
2666
3165
|
#pragma endregion
|
|
2667
3166
|
|
|
2668
3167
|
#pragma region Serialization
|
|
3168
|
+
#endif
|
|
2669
3169
|
|
|
2670
3170
|
/**
|
|
2671
3171
|
* @brief Estimate the binary length (in bytes) of the serialized index.
|
|
@@ -2732,6 +3232,7 @@ class index_gt {
|
|
|
2732
3232
|
serialization_result_t result;
|
|
2733
3233
|
|
|
2734
3234
|
// Remove previously stored objects
|
|
3235
|
+
index_limits_t old_limits = limits_;
|
|
2735
3236
|
reset();
|
|
2736
3237
|
|
|
2737
3238
|
// Pull basic metadata
|
|
@@ -2756,9 +3257,15 @@ class index_gt {
|
|
|
2756
3257
|
// Submit metadata
|
|
2757
3258
|
config_.connectivity = header.connectivity;
|
|
2758
3259
|
config_.connectivity_base = header.connectivity_base;
|
|
3260
|
+
error_t error = config_.validate();
|
|
3261
|
+
if (error)
|
|
3262
|
+
return result.failed(std::move(error));
|
|
3263
|
+
|
|
2759
3264
|
pre_ = precompute_(config_);
|
|
2760
3265
|
index_limits_t limits;
|
|
2761
3266
|
limits.members = header.size;
|
|
3267
|
+
limits.threads_add = (std::max<std::size_t>)(1, old_limits.threads_add);
|
|
3268
|
+
limits.threads_search = (std::max<std::size_t>)(1, old_limits.threads_search);
|
|
2762
3269
|
if (!reserve(limits)) {
|
|
2763
3270
|
reset();
|
|
2764
3271
|
return result.failed("Out of memory");
|
|
@@ -2808,8 +3315,12 @@ class index_gt {
|
|
|
2808
3315
|
},
|
|
2809
3316
|
std::forward<progress_at>(progress));
|
|
2810
3317
|
|
|
2811
|
-
if (!stream_result)
|
|
3318
|
+
if (!stream_result) {
|
|
3319
|
+
// Drop generic messages like "end of file reached" in favor
|
|
3320
|
+
// of more specific messages from the stream
|
|
3321
|
+
io_result.error.release();
|
|
2812
3322
|
return stream_result;
|
|
3323
|
+
}
|
|
2813
3324
|
return io_result;
|
|
2814
3325
|
}
|
|
2815
3326
|
|
|
@@ -2857,8 +3368,12 @@ class index_gt {
|
|
|
2857
3368
|
},
|
|
2858
3369
|
std::forward<progress_at>(progress));
|
|
2859
3370
|
|
|
2860
|
-
if (!stream_result)
|
|
3371
|
+
if (!stream_result) {
|
|
3372
|
+
// Drop generic messages like "end of file reached" in favor
|
|
3373
|
+
// of more specific messages from the stream
|
|
3374
|
+
io_result.error.release();
|
|
2861
3375
|
return stream_result;
|
|
3376
|
+
}
|
|
2862
3377
|
return io_result;
|
|
2863
3378
|
}
|
|
2864
3379
|
|
|
@@ -2897,6 +3412,7 @@ class index_gt {
|
|
|
2897
3412
|
progress_at&& progress = {}) noexcept {
|
|
2898
3413
|
|
|
2899
3414
|
// Remove previously stored objects
|
|
3415
|
+
index_limits_t old_limits = limits_;
|
|
2900
3416
|
reset();
|
|
2901
3417
|
|
|
2902
3418
|
serialization_result_t result = file.open_if_not();
|
|
@@ -2923,6 +3439,10 @@ class index_gt {
|
|
|
2923
3439
|
|
|
2924
3440
|
config_.connectivity = header.connectivity;
|
|
2925
3441
|
config_.connectivity_base = header.connectivity_base;
|
|
3442
|
+
error_t error = config_.validate();
|
|
3443
|
+
if (error)
|
|
3444
|
+
return result.failed(std::move(error));
|
|
3445
|
+
|
|
2926
3446
|
pre_ = precompute_(config_);
|
|
2927
3447
|
misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
|
|
2928
3448
|
offsets[0u] = offset + sizeof(header) + sizeof(level_t) * header.size;
|
|
@@ -2938,6 +3458,8 @@ class index_gt {
|
|
|
2938
3458
|
// Submit metadata and reserve memory
|
|
2939
3459
|
index_limits_t limits;
|
|
2940
3460
|
limits.members = header.size;
|
|
3461
|
+
limits.threads_add = (std::max<std::size_t>)(1, old_limits.threads_add);
|
|
3462
|
+
limits.threads_search = (std::max<std::size_t>)(1, old_limits.threads_search);
|
|
2941
3463
|
if (!reserve(limits)) {
|
|
2942
3464
|
reset();
|
|
2943
3465
|
return result.failed("Out of memory");
|
|
@@ -2956,23 +3478,21 @@ class index_gt {
|
|
|
2956
3478
|
return {};
|
|
2957
3479
|
}
|
|
2958
3480
|
|
|
3481
|
+
#if defined(USEARCH_USE_PRAGMA_REGION)
|
|
2959
3482
|
#pragma endregion
|
|
3483
|
+
#endif
|
|
2960
3484
|
|
|
2961
3485
|
/**
|
|
2962
3486
|
* @brief Performs compaction on the whole HNSW index, purging some entries
|
|
2963
3487
|
* and links to them, while also generating a more efficient mapping,
|
|
2964
3488
|
* putting the more frequently used entries closer together.
|
|
2965
3489
|
*
|
|
2966
|
-
*
|
|
2967
|
-
*
|
|
2968
|
-
*
|
|
2969
|
-
* of the graph, while keeping their outgoing links, in case the node
|
|
2970
|
-
* is structurally relevant and has a crucial role in the index.
|
|
2971
|
-
* It won't reclaim the memory.
|
|
2972
|
-
*
|
|
2973
|
-
* @param[in] allow_member Predicate to mark nodes for isolation.
|
|
3490
|
+
* @param[in] values A []-subscriptable object, providing access to the values.
|
|
3491
|
+
* @param[in] metric Callable object measuring distance between any ::values and present objects.
|
|
3492
|
+
* @param[in] slot_transition Callable object to inform changes in slot assignments.
|
|
2974
3493
|
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
2975
3494
|
* @param[in] progress Callback to report the execution progress.
|
|
3495
|
+
* @param[in] prefetch Callable object to prefetch data into the cache.
|
|
2976
3496
|
*/
|
|
2977
3497
|
template <typename values_at, typename metric_at, //
|
|
2978
3498
|
typename slot_transition_at = dummy_key_to_key_mapping_t, //
|
|
@@ -3006,16 +3526,14 @@ class index_gt {
|
|
|
3006
3526
|
std::size_t const total = 3 * slots_and_levels.size();
|
|
3007
3527
|
|
|
3008
3528
|
// For every bottom level node, determine its parent cluster
|
|
3009
|
-
executor.dynamic(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t
|
|
3529
|
+
executor.dynamic(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot_as_uint) {
|
|
3010
3530
|
context_t& context = contexts_[thread_idx];
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
static_cast<compressed_slot_t>(cluster), //
|
|
3018
|
-
node_at_(old_slot).level()};
|
|
3531
|
+
compressed_slot_t old_slot = static_cast<compressed_slot_t>(old_slot_as_uint);
|
|
3532
|
+
compressed_slot_t cluster = search_for_one_( //
|
|
3533
|
+
values[citerator_at(old_slot)], //
|
|
3534
|
+
metric, prefetch, //
|
|
3535
|
+
static_cast<compressed_slot_t>(entry_slot_), max_level_, 0, context);
|
|
3536
|
+
slots_and_levels[old_slot] = {old_slot, cluster, node_at_(old_slot).level()};
|
|
3019
3537
|
++processed;
|
|
3020
3538
|
if (thread_idx == 0)
|
|
3021
3539
|
do_tasks = progress(processed.load(), total);
|
|
@@ -3198,90 +3716,90 @@ class index_gt {
|
|
|
3198
3716
|
return {nodes_mutexes_, slot};
|
|
3199
3717
|
}
|
|
3200
3718
|
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
// Go down the level, tracking only the closest match
|
|
3208
|
-
std::size_t closest_slot = search_for_one_( //
|
|
3209
|
-
value, metric, prefetch, //
|
|
3210
|
-
entry_slot, max_level, target_level, context);
|
|
3211
|
-
|
|
3212
|
-
// From `target_level` down perform proper extensive search
|
|
3213
|
-
for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
|
|
3214
|
-
// TODO: Handle out of memory conditions
|
|
3215
|
-
search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
|
|
3216
|
-
closest_slot = connect_new_node_(metric, node_slot, level, context);
|
|
3217
|
-
reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
|
|
3719
|
+
struct node_conditional_lock_t {
|
|
3720
|
+
nodes_mutexes_t& mutexes;
|
|
3721
|
+
std::size_t slot;
|
|
3722
|
+
inline ~node_conditional_lock_t() noexcept {
|
|
3723
|
+
if (slot != std::numeric_limits<std::size_t>::max())
|
|
3724
|
+
mutexes.atomic_reset(slot);
|
|
3218
3725
|
}
|
|
3726
|
+
};
|
|
3727
|
+
|
|
3728
|
+
inline node_conditional_lock_t node_try_conditional_lock_(std::size_t slot, bool condition,
|
|
3729
|
+
bool& failed_to_acquire) const noexcept {
|
|
3730
|
+
failed_to_acquire = condition ? nodes_mutexes_.atomic_set(slot) : false;
|
|
3731
|
+
return {nodes_mutexes_, failed_to_acquire ? std::numeric_limits<std::size_t>::max() : slot};
|
|
3219
3732
|
}
|
|
3220
3733
|
|
|
3221
|
-
template <typename metric_at>
|
|
3222
|
-
|
|
3734
|
+
template <typename metric_at, bool require_non_empty_ak = false>
|
|
3735
|
+
candidates_view_t form_links_to_closest_( //
|
|
3223
3736
|
metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
|
|
3224
3737
|
|
|
3225
3738
|
node_t new_node = node_at_(new_slot);
|
|
3226
3739
|
top_candidates_t& top = context.top_candidates;
|
|
3740
|
+
usearch_assert_m(top.size() || !require_non_empty_ak, "No candidates found");
|
|
3741
|
+
candidates_view_t top_view =
|
|
3742
|
+
refine_(metric, config_.connectivity, top, context, context.computed_distances_in_refines);
|
|
3743
|
+
usearch_assert_m(top_view.size() || !require_non_empty_ak, "This would lead to isolated nodes");
|
|
3227
3744
|
|
|
3228
3745
|
// Outgoing links from `new_slot`:
|
|
3229
3746
|
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
3236
|
-
usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
|
|
3237
|
-
new_neighbors.push_back(top_view[idx].slot);
|
|
3238
|
-
}
|
|
3747
|
+
usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
|
|
3748
|
+
for (std::size_t idx = 0; idx != top_view.size(); idx++) {
|
|
3749
|
+
usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
3750
|
+
usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
|
|
3751
|
+
new_neighbors.push_back(top_view[idx].slot);
|
|
3239
3752
|
}
|
|
3240
3753
|
|
|
3241
|
-
return
|
|
3754
|
+
return top_view;
|
|
3242
3755
|
}
|
|
3243
3756
|
|
|
3244
3757
|
template <typename value_at, typename metric_at>
|
|
3245
|
-
void
|
|
3246
|
-
metric_at&& metric,
|
|
3247
|
-
context_t& context) usearch_noexcept_m {
|
|
3758
|
+
void form_reverse_links_( //
|
|
3759
|
+
metric_at&& metric, compressed_slot_t new_slot, candidates_view_t new_neighbors, value_at&& value,
|
|
3760
|
+
level_t level, context_t& context) usearch_noexcept_m {
|
|
3248
3761
|
|
|
3249
|
-
node_t new_node = node_at_(new_slot);
|
|
3250
3762
|
top_candidates_t& top = context.top_candidates;
|
|
3251
|
-
|
|
3763
|
+
std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
|
|
3252
3764
|
|
|
3253
3765
|
// Reverse links from the neighbors:
|
|
3254
|
-
|
|
3255
|
-
|
|
3766
|
+
for (auto new_neighbor : new_neighbors) {
|
|
3767
|
+
compressed_slot_t close_slot = new_neighbor.slot;
|
|
3256
3768
|
if (close_slot == new_slot)
|
|
3257
3769
|
continue;
|
|
3258
3770
|
node_lock_t close_lock = node_lock_(close_slot);
|
|
3259
3771
|
node_t close_node = node_at_(close_slot);
|
|
3260
|
-
|
|
3261
3772
|
neighbors_ref_t close_header = neighbors_(close_node, level);
|
|
3262
|
-
|
|
3773
|
+
|
|
3774
|
+
// The node may have no neighbors only in one case, when it's the first one in the index,
|
|
3775
|
+
// but that is problematic to track in multi-threaded environments, where the order of insertion
|
|
3776
|
+
// is not guaranteed.
|
|
3777
|
+
// usearch_assert_m(close_header.size() || new_slot == 1, "Possible corruption - isolated node");
|
|
3778
|
+
usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption - overflow");
|
|
3263
3779
|
usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
|
|
3264
3780
|
usearch_assert_m(level <= close_node.level(), "Linking to missing level");
|
|
3265
3781
|
|
|
3266
3782
|
// If `new_slot` is already present in the neighboring connections of `close_slot`
|
|
3267
3783
|
// then no need to modify any connections or run the heuristics.
|
|
3268
3784
|
if (close_header.size() < connectivity_max) {
|
|
3269
|
-
close_header.push_back(
|
|
3785
|
+
close_header.push_back(new_slot);
|
|
3270
3786
|
continue;
|
|
3271
3787
|
}
|
|
3272
3788
|
|
|
3273
3789
|
// To fit a new connection we need to drop an existing one.
|
|
3274
3790
|
top.clear();
|
|
3275
|
-
usearch_assert_m((top.
|
|
3276
|
-
|
|
3277
|
-
|
|
3791
|
+
usearch_assert_m((top.capacity() >= (close_header.size() + 1)),
|
|
3792
|
+
"The memory must have been reserved in `add`");
|
|
3793
|
+
top.insert_reserved({context.measure(value, citerator_at(close_slot), metric), new_slot});
|
|
3278
3794
|
for (compressed_slot_t successor_slot : close_header)
|
|
3279
3795
|
top.insert_reserved(
|
|
3280
3796
|
{context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
|
|
3281
3797
|
|
|
3282
3798
|
// Export the results:
|
|
3283
3799
|
close_header.clear();
|
|
3284
|
-
candidates_view_t top_view =
|
|
3800
|
+
candidates_view_t top_view =
|
|
3801
|
+
refine_(metric, connectivity_max, top, context, context.computed_distances_in_reverse_refines);
|
|
3802
|
+
usearch_assert_m(top_view.size(), "This would lead to isolated nodes");
|
|
3285
3803
|
for (std::size_t idx = 0; idx != top_view.size(); idx++)
|
|
3286
3804
|
close_header.push_back(top_view[idx].slot);
|
|
3287
3805
|
}
|
|
@@ -3323,12 +3841,12 @@ class index_gt {
|
|
|
3323
3841
|
using pointer = misaligned_ptr_gt<element_t>;
|
|
3324
3842
|
using reference = misaligned_ref_gt<element_t>;
|
|
3325
3843
|
|
|
3326
|
-
|
|
3844
|
+
value_type operator*() const noexcept { return neighbors_[current_]; }
|
|
3327
3845
|
candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_hash_set_t& visits,
|
|
3328
3846
|
std::size_t progress) noexcept
|
|
3329
3847
|
: index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
|
|
3330
3848
|
candidates_iterator_t operator++(int) noexcept {
|
|
3331
|
-
return candidates_iterator_t(index_,
|
|
3849
|
+
return candidates_iterator_t(index_, neighbors_, visits_, current_ + 1).skip_missing();
|
|
3332
3850
|
}
|
|
3333
3851
|
candidates_iterator_t& operator++() noexcept {
|
|
3334
3852
|
++current_;
|
|
@@ -3338,7 +3856,7 @@ class index_gt {
|
|
|
3338
3856
|
bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
|
|
3339
3857
|
bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
|
|
3340
3858
|
|
|
3341
|
-
vector_key_t key() const noexcept { return index_
|
|
3859
|
+
vector_key_t key() const noexcept { return index_.node_at_(slot()).key(); }
|
|
3342
3860
|
compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
|
|
3343
3861
|
friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
|
|
3344
3862
|
friend inline vector_key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
|
|
@@ -3356,16 +3874,16 @@ class index_gt {
|
|
|
3356
3874
|
};
|
|
3357
3875
|
|
|
3358
3876
|
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
3359
|
-
|
|
3877
|
+
compressed_slot_t search_for_one_( //
|
|
3360
3878
|
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
3361
|
-
|
|
3879
|
+
compressed_slot_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
|
|
3362
3880
|
|
|
3363
3881
|
visits_hash_set_t& visits = context.visits;
|
|
3364
3882
|
visits.clear();
|
|
3365
3883
|
|
|
3366
3884
|
// Optional prefetching
|
|
3367
3885
|
if (!is_dummy<prefetch_at>())
|
|
3368
|
-
prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1)
|
|
3886
|
+
prefetch(citerator_at(closest_slot), citerator_at(closest_slot) + 1);
|
|
3369
3887
|
|
|
3370
3888
|
distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
|
|
3371
3889
|
for (level_t level = begin_level; level > end_level; --level) {
|
|
@@ -3390,6 +3908,7 @@ class index_gt {
|
|
|
3390
3908
|
changed = true;
|
|
3391
3909
|
}
|
|
3392
3910
|
}
|
|
3911
|
+
|
|
3393
3912
|
context.iteration_cycles++;
|
|
3394
3913
|
} while (changed);
|
|
3395
3914
|
}
|
|
@@ -3404,8 +3923,7 @@ class index_gt {
|
|
|
3404
3923
|
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
3405
3924
|
bool search_to_insert_( //
|
|
3406
3925
|
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
3407
|
-
|
|
3408
|
-
context_t& context) noexcept {
|
|
3926
|
+
compressed_slot_t start_slot, level_t level, std::size_t top_limit, context_t& context) noexcept {
|
|
3409
3927
|
|
|
3410
3928
|
visits_hash_set_t& visits = context.visits;
|
|
3411
3929
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
@@ -3414,18 +3932,21 @@ class index_gt {
|
|
|
3414
3932
|
visits.clear();
|
|
3415
3933
|
next.clear();
|
|
3416
3934
|
top.clear();
|
|
3935
|
+
|
|
3936
|
+
// At the very least we are going to explore the starting node and its neighbors
|
|
3417
3937
|
if (!visits.reserve(config_.connectivity_base + 1u))
|
|
3418
3938
|
return false;
|
|
3419
3939
|
|
|
3420
3940
|
// Optional prefetching
|
|
3421
3941
|
if (!is_dummy<prefetch_at>())
|
|
3422
|
-
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1)
|
|
3942
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot) + 1);
|
|
3423
3943
|
|
|
3424
3944
|
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
3425
|
-
next.insert_reserved({-radius,
|
|
3426
|
-
top.insert_reserved({radius,
|
|
3427
|
-
visits.set(
|
|
3945
|
+
next.insert_reserved({-radius, start_slot});
|
|
3946
|
+
top.insert_reserved({radius, start_slot});
|
|
3947
|
+
visits.set(start_slot);
|
|
3428
3948
|
|
|
3949
|
+
// The primary loop of the graph traversal
|
|
3429
3950
|
while (!next.empty()) {
|
|
3430
3951
|
|
|
3431
3952
|
candidate_t candidacy = next.top();
|
|
@@ -3436,8 +3957,6 @@ class index_gt {
|
|
|
3436
3957
|
context.iteration_cycles++;
|
|
3437
3958
|
|
|
3438
3959
|
compressed_slot_t candidate_slot = candidacy.slot;
|
|
3439
|
-
if (new_slot == candidate_slot)
|
|
3440
|
-
continue;
|
|
3441
3960
|
node_t candidate_ref = node_at_(candidate_slot);
|
|
3442
3961
|
node_lock_t candidate_lock = node_lock_(candidate_slot);
|
|
3443
3962
|
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
@@ -3456,6 +3975,8 @@ class index_gt {
|
|
|
3456
3975
|
if (visits.set(successor_slot))
|
|
3457
3976
|
continue;
|
|
3458
3977
|
|
|
3978
|
+
// We don't access the neighbors of the `successor_slot` node,
|
|
3979
|
+
// so we don't have to lock it.
|
|
3459
3980
|
// node_lock_t successor_lock = node_lock_(successor_slot);
|
|
3460
3981
|
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3461
3982
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
@@ -3470,6 +3991,95 @@ class index_gt {
|
|
|
3470
3991
|
return true;
|
|
3471
3992
|
}
|
|
3472
3993
|
|
|
3994
|
+
/**
|
|
3995
|
+
* @brief Traverses a layer of a graph, to find the best neighbors list for updated node.
|
|
3996
|
+
* Locks the nodes in the process, assuming other threads are updating neighbors lists.
|
|
3997
|
+
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3998
|
+
*/
|
|
3999
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
4000
|
+
bool search_to_update_( //
|
|
4001
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
4002
|
+
compressed_slot_t start_slot, compressed_slot_t updated_slot, level_t level, std::size_t top_limit,
|
|
4003
|
+
context_t& context) noexcept {
|
|
4004
|
+
|
|
4005
|
+
visits_hash_set_t& visits = context.visits;
|
|
4006
|
+
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
4007
|
+
top_candidates_t& top = context.top_candidates; // pop max, push
|
|
4008
|
+
|
|
4009
|
+
visits.clear();
|
|
4010
|
+
next.clear();
|
|
4011
|
+
top.clear();
|
|
4012
|
+
|
|
4013
|
+
// At the very least we are going to explore the starting node and its neighbors
|
|
4014
|
+
if (!visits.reserve(config_.connectivity_base + 1u))
|
|
4015
|
+
return false;
|
|
4016
|
+
|
|
4017
|
+
// Optional prefetching
|
|
4018
|
+
if (!is_dummy<prefetch_at>())
|
|
4019
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot) + 1);
|
|
4020
|
+
|
|
4021
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
4022
|
+
next.insert_reserved({-radius, start_slot});
|
|
4023
|
+
visits.set(start_slot);
|
|
4024
|
+
if (start_slot != updated_slot)
|
|
4025
|
+
top.insert_reserved({radius, start_slot});
|
|
4026
|
+
|
|
4027
|
+
// The primary loop of the graph traversal
|
|
4028
|
+
while (!next.empty()) {
|
|
4029
|
+
|
|
4030
|
+
candidate_t candidacy = next.top();
|
|
4031
|
+
if ((-candidacy.distance) > radius && top.size() == top_limit)
|
|
4032
|
+
break;
|
|
4033
|
+
|
|
4034
|
+
next.pop();
|
|
4035
|
+
context.iteration_cycles++;
|
|
4036
|
+
|
|
4037
|
+
compressed_slot_t candidate_slot = candidacy.slot;
|
|
4038
|
+
node_t candidate_ref = node_at_(candidate_slot);
|
|
4039
|
+
|
|
4040
|
+
// The trickiest part of update-heavy workloads is mitigating dead-locks
|
|
4041
|
+
// in connected nodes during traversal. A "good enough" solution would be
|
|
4042
|
+
// to skip concurrent access, assuming the other "close" node is gonna add
|
|
4043
|
+
// this one when forming reverse connections.
|
|
4044
|
+
bool failed_to_acquire = false;
|
|
4045
|
+
node_conditional_lock_t candidate_lock =
|
|
4046
|
+
node_try_conditional_lock_(candidate_slot, updated_slot != candidate_slot, failed_to_acquire);
|
|
4047
|
+
if (failed_to_acquire)
|
|
4048
|
+
continue;
|
|
4049
|
+
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
4050
|
+
|
|
4051
|
+
// Optional prefetching
|
|
4052
|
+
if (!is_dummy<prefetch_at>()) {
|
|
4053
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
4054
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
4055
|
+
}
|
|
4056
|
+
|
|
4057
|
+
// Assume the worst-case when reserving memory
|
|
4058
|
+
if (!visits.reserve(visits.size() + candidate_neighbors.size()))
|
|
4059
|
+
return false;
|
|
4060
|
+
|
|
4061
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
4062
|
+
if (visits.set(successor_slot))
|
|
4063
|
+
continue;
|
|
4064
|
+
|
|
4065
|
+
// We don't access the neighbors of the `successor_slot` node,
|
|
4066
|
+
// so we don't have to lock it.
|
|
4067
|
+
// node_conditional_lock_t successor_lock =
|
|
4068
|
+
// node_try_conditional_lock_(successor_slot, updated_slot != successor_slot);
|
|
4069
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
4070
|
+
if (top.size() < top_limit || successor_dist < radius) {
|
|
4071
|
+
// This can substantially grow our priority queue:
|
|
4072
|
+
next.insert({-successor_dist, successor_slot});
|
|
4073
|
+
// This will automatically evict poor matches:
|
|
4074
|
+
if (updated_slot != successor_slot)
|
|
4075
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
4076
|
+
radius = top.top().distance;
|
|
4077
|
+
}
|
|
4078
|
+
}
|
|
4079
|
+
}
|
|
4080
|
+
return true;
|
|
4081
|
+
}
|
|
4082
|
+
|
|
3473
4083
|
/**
|
|
3474
4084
|
* @brief Traverses the @b base layer of a graph, to find a close match.
|
|
3475
4085
|
* Doesn't lock any nodes, assuming read-only simultaneous access.
|
|
@@ -3478,7 +4088,7 @@ class index_gt {
|
|
|
3478
4088
|
template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
|
|
3479
4089
|
bool search_to_find_in_base_( //
|
|
3480
4090
|
value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
|
|
3481
|
-
|
|
4091
|
+
compressed_slot_t start_slot, std::size_t expansion, context_t& context) const usearch_noexcept_m {
|
|
3482
4092
|
|
|
3483
4093
|
visits_hash_set_t& visits = context.visits;
|
|
3484
4094
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
@@ -3493,20 +4103,24 @@ class index_gt {
|
|
|
3493
4103
|
|
|
3494
4104
|
// Optional prefetching
|
|
3495
4105
|
if (!is_dummy<prefetch_at>())
|
|
3496
|
-
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1)
|
|
4106
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot) + 1);
|
|
3497
4107
|
|
|
3498
4108
|
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
3499
|
-
next.
|
|
3500
|
-
|
|
4109
|
+
usearch_assert_m(next.capacity(), "The `max_heap_gt` must have been reserved in the search entry point");
|
|
4110
|
+
next.insert_reserved({-radius, start_slot});
|
|
4111
|
+
visits.set(start_slot);
|
|
3501
4112
|
|
|
3502
4113
|
// Don't populate the top list if the predicate is not satisfied
|
|
3503
|
-
if (is_dummy<predicate_at>() || predicate(member_cref_t{node_at_(start_slot).ckey(), start_slot}))
|
|
3504
|
-
top.
|
|
4114
|
+
if (is_dummy<predicate_at>() || predicate(member_cref_t{node_at_(start_slot).ckey(), start_slot})) {
|
|
4115
|
+
usearch_assert_m(top.capacity(),
|
|
4116
|
+
"The `sorted_buffer_gt` must have been reserved in the search entry point");
|
|
4117
|
+
top.insert_reserved({radius, start_slot});
|
|
4118
|
+
}
|
|
3505
4119
|
|
|
3506
4120
|
while (!next.empty()) {
|
|
3507
4121
|
|
|
3508
4122
|
candidate_t candidate = next.top();
|
|
3509
|
-
if ((-candidate.distance) > radius)
|
|
4123
|
+
if ((-candidate.distance) > radius && top.size() == top_limit)
|
|
3510
4124
|
break;
|
|
3511
4125
|
|
|
3512
4126
|
next.pop();
|
|
@@ -3555,12 +4169,13 @@ class index_gt {
|
|
|
3555
4169
|
top.clear();
|
|
3556
4170
|
top.reserve(count);
|
|
3557
4171
|
for (std::size_t i = 0; i != size(); ++i) {
|
|
4172
|
+
auto slot = static_cast<compressed_slot_t>(i);
|
|
3558
4173
|
if (!is_dummy<predicate_at>())
|
|
3559
|
-
if (!predicate(at(
|
|
4174
|
+
if (!predicate(at(slot)))
|
|
3560
4175
|
continue;
|
|
3561
4176
|
|
|
3562
|
-
distance_t distance = context.measure(query, citerator_at(
|
|
3563
|
-
top.insert(candidate_t{distance,
|
|
4177
|
+
distance_t distance = context.measure(query, citerator_at(slot), metric);
|
|
4178
|
+
top.insert(candidate_t{distance, slot}, count);
|
|
3564
4179
|
}
|
|
3565
4180
|
}
|
|
3566
4181
|
|
|
@@ -3570,22 +4185,27 @@ class index_gt {
|
|
|
3570
4185
|
* to keep only the neighbors, that are from each other.
|
|
3571
4186
|
*/
|
|
3572
4187
|
template <typename metric_at>
|
|
3573
|
-
candidates_view_t refine_(
|
|
3574
|
-
metric_at&& metric,
|
|
3575
|
-
std::size_t needed, top_candidates_t& top, context_t& context
|
|
4188
|
+
candidates_view_t refine_( //
|
|
4189
|
+
metric_at&& metric, //
|
|
4190
|
+
std::size_t needed, top_candidates_t& top, context_t& context, //
|
|
4191
|
+
std::size_t& refines_counter) const noexcept {
|
|
3576
4192
|
|
|
3577
|
-
|
|
4193
|
+
// Avoid expensive computation, if the set is already small
|
|
3578
4194
|
candidate_t* top_data = top.data();
|
|
3579
4195
|
std::size_t const top_count = top.size();
|
|
3580
4196
|
if (top_count < needed)
|
|
3581
4197
|
return {top_data, top_count};
|
|
3582
4198
|
|
|
4199
|
+
// Sort before processing
|
|
4200
|
+
top.sort_ascending();
|
|
4201
|
+
|
|
3583
4202
|
std::size_t submitted_count = 1;
|
|
3584
4203
|
std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
|
|
3585
4204
|
while (submitted_count < needed && consumed_count < top_count) {
|
|
3586
4205
|
candidate_t candidate = top_data[consumed_count];
|
|
3587
4206
|
bool good = true;
|
|
3588
|
-
|
|
4207
|
+
std::size_t idx = 0;
|
|
4208
|
+
for (; idx < submitted_count; idx++) {
|
|
3589
4209
|
candidate_t submitted = top_data[idx];
|
|
3590
4210
|
distance_t inter_result_dist = context.measure( //
|
|
3591
4211
|
citerator_at(candidate.slot), //
|
|
@@ -3596,6 +4216,7 @@ class index_gt {
|
|
|
3596
4216
|
break;
|
|
3597
4217
|
}
|
|
3598
4218
|
}
|
|
4219
|
+
refines_counter += idx;
|
|
3599
4220
|
|
|
3600
4221
|
if (good) {
|
|
3601
4222
|
top_data[submitted_count] = top_data[consumed_count];
|
|
@@ -3777,11 +4398,12 @@ static join_result_t join( //
|
|
|
3777
4398
|
bool woman_is_free = husband_slot == missing_slot;
|
|
3778
4399
|
if (woman_is_free) {
|
|
3779
4400
|
// Engagement
|
|
3780
|
-
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
4401
|
+
man_to_woman_slots[free_man_slot] = static_cast<compressed_slot_t>(woman.slot);
|
|
3781
4402
|
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3782
4403
|
engagements++;
|
|
3783
4404
|
} else {
|
|
3784
|
-
distance_t distance_from_husband =
|
|
4405
|
+
distance_t distance_from_husband =
|
|
4406
|
+
women_metric(women_values[static_cast<compressed_slot_t>(woman.slot)], men_values[husband_slot]);
|
|
3785
4407
|
distance_t distance_from_candidate = match.distance;
|
|
3786
4408
|
if (distance_from_husband > distance_from_candidate) {
|
|
3787
4409
|
// Break-up
|
|
@@ -3791,7 +4413,7 @@ static join_result_t join( //
|
|
|
3791
4413
|
men_locks.atomic_reset(husband_slot);
|
|
3792
4414
|
|
|
3793
4415
|
// New Engagement
|
|
3794
|
-
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
4416
|
+
man_to_woman_slots[free_man_slot] = static_cast<compressed_slot_t>(woman.slot);
|
|
3795
4417
|
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3796
4418
|
engagements++;
|
|
3797
4419
|
|
|
@@ -3816,7 +4438,7 @@ static join_result_t join( //
|
|
|
3816
4438
|
for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
|
|
3817
4439
|
compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
|
|
3818
4440
|
if (woman_slot != missing_slot) {
|
|
3819
|
-
man_key_t man = men.at(man_slot).key;
|
|
4441
|
+
man_key_t man = men.at(static_cast<compressed_slot_t>(man_slot)).key;
|
|
3820
4442
|
woman_key_t woman = women.at(woman_slot).key;
|
|
3821
4443
|
man_to_woman[man] = woman;
|
|
3822
4444
|
woman_to_man[woman] = man;
|