usearch 2.10.3 → 2.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.gyp +101 -0
- package/c/CMakeLists.txt +27 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +362 -0
- package/c/test.c +382 -0
- package/c/usearch.h +356 -0
- package/cmake/clipp.cmake +7 -0
- package/cmake/config.cmake.in +15 -0
- package/cmake/emscripten.cmake +8 -0
- package/cmake/jemalloc.cmake +62 -0
- package/cmake/pkg-config.pc.in +4 -0
- package/cmake/usearchConfigVersion.cmake.in +20 -0
- package/cpp/CMakeLists.txt +14 -0
- package/cpp/README.md +161 -0
- package/cpp/bench.cpp +616 -0
- package/cpp/test.cpp +324 -0
- package/fp16/CMakeLists.txt +176 -0
- package/fp16/LICENSE +11 -0
- package/fp16/README.md +20 -0
- package/fp16/bench/alt-element.cc +89 -0
- package/fp16/bench/from-alt-array.cc +175 -0
- package/fp16/bench/from-ieee-array.cc +399 -0
- package/fp16/bench/ieee-element.cc +272 -0
- package/fp16/bench/to-alt-array.cc +91 -0
- package/fp16/bench/to-ieee-array.cc +303 -0
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +15 -0
- package/fp16/cmake/DownloadGoogleTest.cmake +15 -0
- package/fp16/cmake/DownloadPSimd.cmake +15 -0
- package/fp16/configure.py +65 -0
- package/fp16/confu.yaml +8 -0
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +50 -0
- package/fp16/include/fp16/avx2.py +53 -0
- package/fp16/include/fp16/bitcasts.h +92 -0
- package/fp16/include/fp16/fp16.h +451 -0
- package/fp16/include/fp16/psimd.h +131 -0
- package/fp16/include/fp16.h +11 -0
- package/fp16/jni/Android.mk +6 -0
- package/fp16/jni/Application.mk +4 -0
- package/fp16/test/alt-from-fp32-value.cc +498 -0
- package/fp16/test/alt-to-fp32-bits.cc +263 -0
- package/fp16/test/alt-to-fp32-psimd.cc +145 -0
- package/fp16/test/alt-to-fp32-value.cc +348 -0
- package/fp16/test/alt-to-fp32x2-psimd.cc +245 -0
- package/fp16/test/bitcasts.cc +56 -0
- package/fp16/test/ieee-from-fp32-value.cc +523 -0
- package/fp16/test/ieee-to-fp32-bits.cc +333 -0
- package/fp16/test/ieee-to-fp32-psimd.cc +312 -0
- package/fp16/test/ieee-to-fp32-value.cc +420 -0
- package/fp16/test/ieee-to-fp32x2-psimd.cc +541 -0
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +96 -0
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +96 -0
- package/fp16/test/peachpy/stubs.py +38 -0
- package/fp16/test/tables.cc +530 -0
- package/fp16/test/tables.h +13 -0
- package/fp16/third-party/THHalf.h +92 -0
- package/fp16/third-party/eigen-half.h +170 -0
- package/fp16/third-party/float16-compressor.h +80 -0
- package/fp16/third-party/half.hpp +2916 -0
- package/fp16/third-party/npy-halffloat.h +328 -0
- package/include/usearch/index.hpp +3838 -0
- package/include/usearch/index_dense.hpp +2004 -0
- package/include/usearch/index_plugins.hpp +2255 -0
- package/javascript/dist-package-cjs.json +3 -0
- package/javascript/dist-package-esm.json +3 -0
- package/javascript/lib.cpp +350 -0
- package/javascript/node-gyp-build.d.ts +1 -0
- package/javascript/tsconfig-base.json +25 -0
- package/javascript/tsconfig-cjs.json +8 -0
- package/javascript/tsconfig-esm.json +8 -0
- package/javascript/usearch.test.js +84 -0
- package/javascript/usearch.ts +640 -0
- package/package.json +10 -3
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/prebuilds/win32-ia32/usearch.node +0 -0
- package/prebuilds/win32-x64/usearch.node +0 -0
- package/simsimd/.releaserc +87 -0
- package/simsimd/CONTRIBUTING.md +114 -0
- package/simsimd/Cargo.lock +709 -0
- package/simsimd/Cargo.toml +44 -0
- package/simsimd/LICENSE +201 -0
- package/simsimd/README.md +708 -0
- package/simsimd/binding.gyp +36 -0
- package/simsimd/build.rs +51 -0
- package/simsimd/c/lib.c +110 -0
- package/simsimd/include/simsimd/binary.h +258 -0
- package/simsimd/include/simsimd/dot.h +1237 -0
- package/simsimd/include/simsimd/geospatial.h +37 -0
- package/simsimd/include/simsimd/probability.h +563 -0
- package/simsimd/include/simsimd/simsimd.h +1129 -0
- package/simsimd/include/simsimd/spatial.h +1019 -0
- package/simsimd/include/simsimd/types.h +309 -0
- package/simsimd/javascript/bench.js +127 -0
- package/simsimd/javascript/dist-package-cjs.json +3 -0
- package/simsimd/javascript/dist-package-esm.json +3 -0
- package/simsimd/javascript/fallback.ts +203 -0
- package/simsimd/javascript/lib.c +107 -0
- package/simsimd/javascript/node-gyp-build.d.ts +1 -0
- package/simsimd/javascript/simsimd.ts +135 -0
- package/simsimd/javascript/test.mjs +179 -0
- package/simsimd/javascript/tsconfig-base.json +37 -0
- package/simsimd/javascript/tsconfig-cjs.json +8 -0
- package/simsimd/javascript/tsconfig-esm.json +8 -0
- package/simsimd/package-lock.json +1709 -0
- package/simsimd/package.json +61 -0
- package/simsimd/rust/benches/cosine.rs +27 -0
- package/simsimd/rust/benches/native.rs +30 -0
- package/simsimd/rust/benches/sqeuclidean.rs +27 -0
- package/simsimd/rust/lib.rs +778 -0
|
@@ -0,0 +1,3838 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file index.hpp
|
|
3
|
+
* @author Ash Vardanian
|
|
4
|
+
* @brief Single-header Vector Search.
|
|
5
|
+
* @date 2023-04-26
|
|
6
|
+
*
|
|
7
|
+
* @copyright Copyright (c) 2023
|
|
8
|
+
*/
|
|
9
|
+
#ifndef UNUM_USEARCH_HPP
|
|
10
|
+
#define UNUM_USEARCH_HPP
|
|
11
|
+
|
|
12
|
+
#define USEARCH_VERSION_MAJOR 2
|
|
13
|
+
#define USEARCH_VERSION_MINOR 10
|
|
14
|
+
#define USEARCH_VERSION_PATCH 5
|
|
15
|
+
|
|
16
|
+
// Inferring C++ version
|
|
17
|
+
// https://stackoverflow.com/a/61552074
|
|
18
|
+
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
19
|
+
#define USEARCH_DEFINED_CPP17
|
|
20
|
+
#endif
|
|
21
|
+
|
|
22
|
+
// Inferring target OS: Windows, MacOS, or Linux
|
|
23
|
+
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
|
|
24
|
+
#define USEARCH_DEFINED_WINDOWS
|
|
25
|
+
#elif defined(__APPLE__) && defined(__MACH__)
|
|
26
|
+
#define USEARCH_DEFINED_APPLE
|
|
27
|
+
#elif defined(__linux__)
|
|
28
|
+
#define USEARCH_DEFINED_LINUX
|
|
29
|
+
#endif
|
|
30
|
+
|
|
31
|
+
// Inferring the compiler: Clang vs GCC
|
|
32
|
+
#if defined(__clang__)
|
|
33
|
+
#define USEARCH_DEFINED_CLANG
|
|
34
|
+
#elif defined(__GNUC__)
|
|
35
|
+
#define USEARCH_DEFINED_GCC
|
|
36
|
+
#endif
|
|
37
|
+
|
|
38
|
+
// Inferring hardware architecture: x86 vs Arm
|
|
39
|
+
#if defined(__x86_64__)
|
|
40
|
+
#define USEARCH_DEFINED_X86
|
|
41
|
+
#elif defined(__aarch64__)
|
|
42
|
+
#define USEARCH_DEFINED_ARM
|
|
43
|
+
#endif
|
|
44
|
+
|
|
45
|
+
// Inferring hardware bitness: 32 vs 64
|
|
46
|
+
// https://stackoverflow.com/a/5273354
|
|
47
|
+
#if INTPTR_MAX == INT64_MAX
|
|
48
|
+
#define USEARCH_64BIT_ENV
|
|
49
|
+
#elif INTPTR_MAX == INT32_MAX
|
|
50
|
+
#define USEARCH_32BIT_ENV
|
|
51
|
+
#else
|
|
52
|
+
#error Unknown pointer size or missing size macros!
|
|
53
|
+
#endif
|
|
54
|
+
|
|
55
|
+
#if !defined(USEARCH_USE_OPENMP)
|
|
56
|
+
#define USEARCH_USE_OPENMP 0
|
|
57
|
+
#endif
|
|
58
|
+
|
|
59
|
+
// OS-specific includes
|
|
60
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
61
|
+
#define _USE_MATH_DEFINES
|
|
62
|
+
#define NOMINMAX
|
|
63
|
+
#include <Windows.h>
|
|
64
|
+
#include <sys/stat.h> // `fstat` for file size
|
|
65
|
+
#undef NOMINMAX
|
|
66
|
+
#undef _USE_MATH_DEFINES
|
|
67
|
+
#else
|
|
68
|
+
#include <fcntl.h> // `fallocate`
|
|
69
|
+
#include <stdlib.h> // `posix_memalign`
|
|
70
|
+
#include <sys/mman.h> // `mmap`
|
|
71
|
+
#include <sys/stat.h> // `fstat` for file size
|
|
72
|
+
#include <unistd.h> // `open`, `close`
|
|
73
|
+
#endif
|
|
74
|
+
|
|
75
|
+
// STL includes
|
|
76
|
+
#include <algorithm> // `std::sort_heap`
|
|
77
|
+
#include <atomic> // `std::atomic`
|
|
78
|
+
#include <bitset> // `std::bitset`
|
|
79
|
+
#include <climits> // `CHAR_BIT`
|
|
80
|
+
#include <cmath> // `std::sqrt`
|
|
81
|
+
#include <cstring> // `std::memset`
|
|
82
|
+
#include <iterator> // `std::reverse_iterator`
|
|
83
|
+
#include <mutex> // `std::unique_lock` - replacement candidate
|
|
84
|
+
#include <random> // `std::default_random_engine` - replacement candidate
|
|
85
|
+
#include <stdexcept> // `std::runtime_exception`
|
|
86
|
+
#include <thread> // `std::thread`
|
|
87
|
+
#include <utility> // `std::pair`
|
|
88
|
+
|
|
89
|
+
// Prefetching
|
|
90
|
+
#if defined(USEARCH_DEFINED_GCC)
|
|
91
|
+
// https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
|
|
92
|
+
// Zero means we are only going to read from that memory.
|
|
93
|
+
// Three means high temporal locality and suggests to keep
|
|
94
|
+
// the data in all layers of cache.
|
|
95
|
+
#define prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
|
|
96
|
+
#elif defined(USEARCH_DEFINED_X86)
|
|
97
|
+
#define prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
|
|
98
|
+
#else
|
|
99
|
+
#define prefetch_m(ptr)
|
|
100
|
+
#endif
|
|
101
|
+
|
|
102
|
+
// Alignment
|
|
103
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
104
|
+
#define usearch_pack_m
|
|
105
|
+
#define usearch_align_m __declspec(align(64))
|
|
106
|
+
#else
|
|
107
|
+
#define usearch_pack_m __attribute__((packed))
|
|
108
|
+
#define usearch_align_m __attribute__((aligned(64)))
|
|
109
|
+
#endif
|
|
110
|
+
|
|
111
|
+
// Debugging
|
|
112
|
+
#if defined(NDEBUG)
|
|
113
|
+
#define usearch_assert_m(must_be_true, message)
|
|
114
|
+
#define usearch_noexcept_m noexcept
|
|
115
|
+
#else
|
|
116
|
+
#define usearch_assert_m(must_be_true, message) \
|
|
117
|
+
if (!(must_be_true)) { \
|
|
118
|
+
throw std::runtime_error(message); \
|
|
119
|
+
}
|
|
120
|
+
#define usearch_noexcept_m
|
|
121
|
+
#endif
|
|
122
|
+
|
|
123
|
+
namespace unum {
|
|
124
|
+
namespace usearch {
|
|
125
|
+
|
|
126
|
+
using byte_t = char;
|
|
127
|
+
|
|
128
|
+
template <std::size_t multiple_ak> std::size_t divide_round_up(std::size_t num) noexcept {
|
|
129
|
+
return (num + multiple_ak - 1) / multiple_ak;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
inline std::size_t divide_round_up(std::size_t num, std::size_t denominator) noexcept {
|
|
133
|
+
return (num + denominator - 1) / denominator;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
inline std::size_t ceil2(std::size_t v) noexcept {
|
|
137
|
+
v--;
|
|
138
|
+
v |= v >> 1;
|
|
139
|
+
v |= v >> 2;
|
|
140
|
+
v |= v >> 4;
|
|
141
|
+
v |= v >> 8;
|
|
142
|
+
v |= v >> 16;
|
|
143
|
+
#ifdef USEARCH_64BIT_ENV
|
|
144
|
+
v |= v >> 32;
|
|
145
|
+
#endif
|
|
146
|
+
v++;
|
|
147
|
+
return v;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/// @brief Simply dereferencing misaligned pointers can be dangerous.
|
|
151
|
+
template <typename at> void misaligned_store(void* ptr, at v) noexcept {
|
|
152
|
+
static_assert(!std::is_reference<at>::value, "Can't store a reference");
|
|
153
|
+
std::memcpy(ptr, &v, sizeof(at));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/// @brief Simply dereferencing misaligned pointers can be dangerous.
|
|
157
|
+
template <typename at> at misaligned_load(void* ptr) noexcept {
|
|
158
|
+
static_assert(!std::is_reference<at>::value, "Can't load a reference");
|
|
159
|
+
at v;
|
|
160
|
+
std::memcpy(&v, ptr, sizeof(at));
|
|
161
|
+
return v;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/// @brief The `std::exchange` alternative for C++11.
|
|
165
|
+
template <typename at, typename other_at = at> at exchange(at& obj, other_at&& new_value) {
|
|
166
|
+
at old_value = std::move(obj);
|
|
167
|
+
obj = std::forward<other_at>(new_value);
|
|
168
|
+
return old_value;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/// @brief The `std::destroy_at` alternative for C++11.
|
|
172
|
+
template <typename at, typename sfinae_at = at>
|
|
173
|
+
typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
|
|
174
|
+
template <typename at, typename sfinae_at = at>
|
|
175
|
+
typename std::enable_if<!std::is_pod<sfinae_at>::value>::type destroy_at(at* obj) {
|
|
176
|
+
obj->~sfinae_at();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/// @brief The `std::construct_at` alternative for C++11.
|
|
180
|
+
template <typename at, typename sfinae_at = at>
|
|
181
|
+
typename std::enable_if<std::is_pod<sfinae_at>::value>::type construct_at(at*) {}
|
|
182
|
+
template <typename at, typename sfinae_at = at>
|
|
183
|
+
typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* obj) {
|
|
184
|
+
new (obj) at();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* @brief A reference to a misaligned memory location with a specific type.
|
|
189
|
+
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
190
|
+
* indivisible by `sizeof(at)`.
|
|
191
|
+
*/
|
|
192
|
+
template <typename at> class misaligned_ref_gt {
|
|
193
|
+
using element_t = at;
|
|
194
|
+
using mutable_t = typename std::remove_const<element_t>::type;
|
|
195
|
+
byte_t* ptr_;
|
|
196
|
+
|
|
197
|
+
public:
|
|
198
|
+
misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
199
|
+
operator mutable_t() const noexcept { return misaligned_load<mutable_t>(ptr_); }
|
|
200
|
+
misaligned_ref_gt& operator=(mutable_t const& v) noexcept {
|
|
201
|
+
misaligned_store<mutable_t>(ptr_, v);
|
|
202
|
+
return *this;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
void reset(byte_t* ptr) noexcept { ptr_ = ptr; }
|
|
206
|
+
byte_t* ptr() const noexcept { return ptr_; }
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* @brief A pointer to a misaligned memory location with a specific type.
|
|
211
|
+
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
212
|
+
* indivisible by `sizeof(at)`.
|
|
213
|
+
*/
|
|
214
|
+
template <typename at> class misaligned_ptr_gt {
|
|
215
|
+
using element_t = at;
|
|
216
|
+
using mutable_t = typename std::remove_const<element_t>::type;
|
|
217
|
+
byte_t* ptr_;
|
|
218
|
+
|
|
219
|
+
public:
|
|
220
|
+
using iterator_category = std::random_access_iterator_tag;
|
|
221
|
+
using value_type = element_t;
|
|
222
|
+
using difference_type = std::ptrdiff_t;
|
|
223
|
+
using pointer = misaligned_ptr_gt<element_t>;
|
|
224
|
+
using reference = misaligned_ref_gt<element_t>;
|
|
225
|
+
|
|
226
|
+
reference operator*() const noexcept { return {ptr_}; }
|
|
227
|
+
reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
|
|
228
|
+
value_type operator[](std::size_t i) const noexcept {
|
|
229
|
+
return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
233
|
+
misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(element_t)); }
|
|
234
|
+
misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(element_t)); }
|
|
235
|
+
misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(element_t)); }
|
|
236
|
+
misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(element_t)); }
|
|
237
|
+
|
|
238
|
+
// clang-format off
|
|
239
|
+
misaligned_ptr_gt& operator++() noexcept { ptr_ += sizeof(element_t); return *this; }
|
|
240
|
+
misaligned_ptr_gt& operator--() noexcept { ptr_ -= sizeof(element_t); return *this; }
|
|
241
|
+
misaligned_ptr_gt& operator+=(difference_type d) noexcept { ptr_ += d * sizeof(element_t); return *this; }
|
|
242
|
+
misaligned_ptr_gt& operator-=(difference_type d) noexcept { ptr_ -= d * sizeof(element_t); return *this; }
|
|
243
|
+
// clang-format on
|
|
244
|
+
|
|
245
|
+
bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
|
|
246
|
+
bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* @brief Non-owning memory range view, similar to `std::span`, but for C++11.
|
|
251
|
+
*/
|
|
252
|
+
template <typename scalar_at> class span_gt {
|
|
253
|
+
scalar_at* data_;
|
|
254
|
+
std::size_t size_;
|
|
255
|
+
|
|
256
|
+
public:
|
|
257
|
+
span_gt() noexcept : data_(nullptr), size_(0u) {}
|
|
258
|
+
span_gt(scalar_at* begin, scalar_at* end) noexcept : data_(begin), size_(end - begin) {}
|
|
259
|
+
span_gt(scalar_at* begin, std::size_t size) noexcept : data_(begin), size_(size) {}
|
|
260
|
+
scalar_at* data() const noexcept { return data_; }
|
|
261
|
+
std::size_t size() const noexcept { return size_; }
|
|
262
|
+
scalar_at* begin() const noexcept { return data_; }
|
|
263
|
+
scalar_at* end() const noexcept { return data_ + size_; }
|
|
264
|
+
operator scalar_at*() const noexcept { return data(); }
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* @brief Similar to `std::vector`, but doesn't support dynamic resizing.
|
|
269
|
+
* On the bright side, this can't throw exceptions.
|
|
270
|
+
*/
|
|
271
|
+
template <typename scalar_at, typename allocator_at = std::allocator<scalar_at>> class buffer_gt {
|
|
272
|
+
scalar_at* data_;
|
|
273
|
+
std::size_t size_;
|
|
274
|
+
|
|
275
|
+
public:
|
|
276
|
+
buffer_gt() noexcept : data_(nullptr), size_(0u) {}
|
|
277
|
+
buffer_gt(std::size_t size) noexcept : data_(allocator_at{}.allocate(size)), size_(data_ ? size : 0u) {
|
|
278
|
+
if (!std::is_trivially_default_constructible<scalar_at>::value)
|
|
279
|
+
for (std::size_t i = 0; i != size_; ++i)
|
|
280
|
+
construct_at(data_ + i);
|
|
281
|
+
}
|
|
282
|
+
~buffer_gt() noexcept {
|
|
283
|
+
if (!std::is_trivially_destructible<scalar_at>::value)
|
|
284
|
+
for (std::size_t i = 0; i != size_; ++i)
|
|
285
|
+
destroy_at(data_ + i);
|
|
286
|
+
allocator_at{}.deallocate(data_, size_);
|
|
287
|
+
data_ = nullptr;
|
|
288
|
+
size_ = 0;
|
|
289
|
+
}
|
|
290
|
+
scalar_at* data() const noexcept { return data_; }
|
|
291
|
+
std::size_t size() const noexcept { return size_; }
|
|
292
|
+
scalar_at* begin() const noexcept { return data_; }
|
|
293
|
+
scalar_at* end() const noexcept { return data_ + size_; }
|
|
294
|
+
operator scalar_at*() const noexcept { return data(); }
|
|
295
|
+
scalar_at& operator[](std::size_t i) noexcept { return data_[i]; }
|
|
296
|
+
scalar_at const& operator[](std::size_t i) const noexcept { return data_[i]; }
|
|
297
|
+
explicit operator bool() const noexcept { return data_; }
|
|
298
|
+
scalar_at* release() noexcept {
|
|
299
|
+
size_ = 0;
|
|
300
|
+
return exchange(data_, nullptr);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
buffer_gt(buffer_gt const&) = delete;
|
|
304
|
+
buffer_gt& operator=(buffer_gt const&) = delete;
|
|
305
|
+
|
|
306
|
+
buffer_gt(buffer_gt&& other) noexcept : data_(exchange(other.data_, nullptr)), size_(exchange(other.size_, 0)) {}
|
|
307
|
+
buffer_gt& operator=(buffer_gt&& other) noexcept {
|
|
308
|
+
std::swap(data_, other.data_);
|
|
309
|
+
std::swap(size_, other.size_);
|
|
310
|
+
return *this;
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* @brief A lightweight error class for handling error messages,
|
|
316
|
+
* which are expected to be allocated in static memory.
|
|
317
|
+
*/
|
|
318
|
+
class error_t {
|
|
319
|
+
char const* message_{};
|
|
320
|
+
|
|
321
|
+
public:
|
|
322
|
+
error_t(char const* message = nullptr) noexcept : message_(message) {}
|
|
323
|
+
error_t& operator=(char const* message) noexcept {
|
|
324
|
+
message_ = message;
|
|
325
|
+
return *this;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
error_t(error_t const&) = delete;
|
|
329
|
+
error_t& operator=(error_t const&) = delete;
|
|
330
|
+
error_t(error_t&& other) noexcept : message_(exchange(other.message_, nullptr)) {}
|
|
331
|
+
error_t& operator=(error_t&& other) noexcept {
|
|
332
|
+
std::swap(message_, other.message_);
|
|
333
|
+
return *this;
|
|
334
|
+
}
|
|
335
|
+
explicit operator bool() const noexcept { return message_ != nullptr; }
|
|
336
|
+
char const* what() const noexcept { return message_; }
|
|
337
|
+
char const* release() noexcept { return exchange(message_, nullptr); }
|
|
338
|
+
|
|
339
|
+
#if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
|
|
340
|
+
~error_t() noexcept(false) {
|
|
341
|
+
#if defined(USEARCH_DEFINED_CPP17)
|
|
342
|
+
if (message_ && std::uncaught_exceptions() == 0)
|
|
343
|
+
#else
|
|
344
|
+
if (message_ && std::uncaught_exception() == 0)
|
|
345
|
+
#endif
|
|
346
|
+
raise();
|
|
347
|
+
}
|
|
348
|
+
void raise() noexcept(false) {
|
|
349
|
+
if (message_)
|
|
350
|
+
throw std::runtime_error(exchange(message_, nullptr));
|
|
351
|
+
}
|
|
352
|
+
#else
|
|
353
|
+
~error_t() noexcept { raise(); }
|
|
354
|
+
void raise() noexcept {
|
|
355
|
+
if (message_)
|
|
356
|
+
std::terminate();
|
|
357
|
+
}
|
|
358
|
+
#endif
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* @brief Similar to `std::expected` in C++23, wraps a statement evaluation result,
|
|
363
|
+
* or an error. It's used to avoid raising exception, and gracefully propagate
|
|
364
|
+
* the error.
|
|
365
|
+
*
|
|
366
|
+
* @tparam result_at The type of the expected result.
|
|
367
|
+
*/
|
|
368
|
+
template <typename result_at> struct expected_gt {
|
|
369
|
+
result_at result;
|
|
370
|
+
error_t error;
|
|
371
|
+
|
|
372
|
+
operator result_at&() & {
|
|
373
|
+
error.raise();
|
|
374
|
+
return result;
|
|
375
|
+
}
|
|
376
|
+
operator result_at&&() && {
|
|
377
|
+
error.raise();
|
|
378
|
+
return std::move(result);
|
|
379
|
+
}
|
|
380
|
+
result_at const& operator*() const noexcept { return result; }
|
|
381
|
+
explicit operator bool() const noexcept { return !error; }
|
|
382
|
+
expected_gt failed(error_t message) noexcept {
|
|
383
|
+
error = std::move(message);
|
|
384
|
+
return std::move(*this);
|
|
385
|
+
}
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* @brief Light-weight bitset implementation to sync nodes updates during graph mutations.
|
|
390
|
+
* Extends basic functionality with @b atomic operations.
|
|
391
|
+
*/
|
|
392
|
+
template <typename allocator_at = std::allocator<byte_t>> class bitset_gt {
|
|
393
|
+
using allocator_t = allocator_at;
|
|
394
|
+
using byte_t = typename allocator_t::value_type;
|
|
395
|
+
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
396
|
+
|
|
397
|
+
using compressed_slot_t = unsigned long;
|
|
398
|
+
|
|
399
|
+
static constexpr std::size_t bits_per_slot() { return sizeof(compressed_slot_t) * CHAR_BIT; }
|
|
400
|
+
static constexpr compressed_slot_t bits_mask() { return sizeof(compressed_slot_t) * CHAR_BIT - 1; }
|
|
401
|
+
static constexpr std::size_t slots(std::size_t bits) { return divide_round_up<bits_per_slot()>(bits); }
|
|
402
|
+
|
|
403
|
+
compressed_slot_t* slots_{};
|
|
404
|
+
/// @brief Number of slots.
|
|
405
|
+
std::size_t count_{};
|
|
406
|
+
|
|
407
|
+
public:
|
|
408
|
+
bitset_gt() noexcept {}
|
|
409
|
+
~bitset_gt() noexcept { reset(); }
|
|
410
|
+
|
|
411
|
+
explicit operator bool() const noexcept { return slots_; }
|
|
412
|
+
void clear() noexcept {
|
|
413
|
+
if (slots_)
|
|
414
|
+
std::memset(slots_, 0, count_ * sizeof(compressed_slot_t));
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
void reset() noexcept {
|
|
418
|
+
if (slots_)
|
|
419
|
+
allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(compressed_slot_t));
|
|
420
|
+
slots_ = nullptr;
|
|
421
|
+
count_ = 0;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
bitset_gt(std::size_t capacity) noexcept
|
|
425
|
+
: slots_((compressed_slot_t*)allocator_t{}.allocate(slots(capacity) * sizeof(compressed_slot_t))),
|
|
426
|
+
count_(slots_ ? slots(capacity) : 0u) {
|
|
427
|
+
clear();
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
bitset_gt(bitset_gt&& other) noexcept {
|
|
431
|
+
slots_ = exchange(other.slots_, nullptr);
|
|
432
|
+
count_ = exchange(other.count_, 0);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
bitset_gt& operator=(bitset_gt&& other) noexcept {
|
|
436
|
+
std::swap(slots_, other.slots_);
|
|
437
|
+
std::swap(count_, other.count_);
|
|
438
|
+
return *this;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
bitset_gt(bitset_gt const&) = delete;
|
|
442
|
+
bitset_gt& operator=(bitset_gt const&) = delete;
|
|
443
|
+
|
|
444
|
+
inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
|
|
445
|
+
inline bool set(std::size_t i) noexcept {
|
|
446
|
+
compressed_slot_t& slot = slots_[i / bits_per_slot()];
|
|
447
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
448
|
+
bool value = slot & mask;
|
|
449
|
+
slot |= mask;
|
|
450
|
+
return value;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
454
|
+
|
|
455
|
+
inline bool atomic_set(std::size_t i) noexcept {
|
|
456
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
457
|
+
return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
inline void atomic_reset(std::size_t i) noexcept {
|
|
461
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
462
|
+
InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
#else
|
|
466
|
+
|
|
467
|
+
inline bool atomic_set(std::size_t i) noexcept {
|
|
468
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
469
|
+
return __atomic_fetch_or(&slots_[i / bits_per_slot()], mask, __ATOMIC_ACQUIRE) & mask;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
inline void atomic_reset(std::size_t i) noexcept {
|
|
473
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
474
|
+
__atomic_fetch_and(&slots_[i / bits_per_slot()], ~mask, __ATOMIC_RELEASE);
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
#endif
|
|
478
|
+
|
|
479
|
+
class lock_t {
|
|
480
|
+
bitset_gt& bitset_;
|
|
481
|
+
std::size_t bit_offset_;
|
|
482
|
+
|
|
483
|
+
public:
|
|
484
|
+
inline ~lock_t() noexcept { bitset_.atomic_reset(bit_offset_); }
|
|
485
|
+
inline lock_t(bitset_gt& bitset, std::size_t bit_offset) noexcept : bitset_(bitset), bit_offset_(bit_offset) {
|
|
486
|
+
while (bitset_.atomic_set(bit_offset_))
|
|
487
|
+
;
|
|
488
|
+
}
|
|
489
|
+
};
|
|
490
|
+
|
|
491
|
+
inline lock_t lock(std::size_t i) noexcept { return {*this, i}; }
|
|
492
|
+
};
|
|
493
|
+
|
|
494
|
+
using bitset_t = bitset_gt<>;
|
|
495
|
+
|
|
496
|
+
/**
|
|
497
|
+
* @brief Similar to `std::priority_queue`, but allows raw access to underlying
|
|
498
|
+
* memory, in case you want to shuffle it or sort. Good for collections
|
|
499
|
+
* from 100s to 10'000s elements.
|
|
500
|
+
*/
|
|
501
|
+
template <typename element_at, //
|
|
502
|
+
typename comparator_at = std::less<void>, // <void> is needed before C++14.
|
|
503
|
+
typename allocator_at = std::allocator<element_at>> //
|
|
504
|
+
class max_heap_gt {
|
|
505
|
+
public:
|
|
506
|
+
using element_t = element_at;
|
|
507
|
+
using comparator_t = comparator_at;
|
|
508
|
+
using allocator_t = allocator_at;
|
|
509
|
+
|
|
510
|
+
using value_type = element_t;
|
|
511
|
+
|
|
512
|
+
static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
|
|
513
|
+
static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
|
|
514
|
+
|
|
515
|
+
private:
|
|
516
|
+
element_t* elements_;
|
|
517
|
+
std::size_t size_;
|
|
518
|
+
std::size_t capacity_;
|
|
519
|
+
|
|
520
|
+
public:
|
|
521
|
+
max_heap_gt() noexcept : elements_(nullptr), size_(0), capacity_(0) {}
|
|
522
|
+
|
|
523
|
+
max_heap_gt(max_heap_gt&& other) noexcept
|
|
524
|
+
: elements_(exchange(other.elements_, nullptr)), size_(exchange(other.size_, 0)),
|
|
525
|
+
capacity_(exchange(other.capacity_, 0)) {}
|
|
526
|
+
|
|
527
|
+
max_heap_gt& operator=(max_heap_gt&& other) noexcept {
|
|
528
|
+
std::swap(elements_, other.elements_);
|
|
529
|
+
std::swap(size_, other.size_);
|
|
530
|
+
std::swap(capacity_, other.capacity_);
|
|
531
|
+
return *this;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
max_heap_gt(max_heap_gt const&) = delete;
|
|
535
|
+
max_heap_gt& operator=(max_heap_gt const&) = delete;
|
|
536
|
+
|
|
537
|
+
~max_heap_gt() noexcept { reset(); }
|
|
538
|
+
|
|
539
|
+
void reset() noexcept {
|
|
540
|
+
if (elements_)
|
|
541
|
+
allocator_t{}.deallocate(elements_, capacity_);
|
|
542
|
+
elements_ = nullptr;
|
|
543
|
+
capacity_ = 0;
|
|
544
|
+
size_ = 0;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
inline bool empty() const noexcept { return !size_; }
|
|
548
|
+
inline std::size_t size() const noexcept { return size_; }
|
|
549
|
+
inline std::size_t capacity() const noexcept { return capacity_; }
|
|
550
|
+
|
|
551
|
+
/// @brief Selects the largest element in the heap.
|
|
552
|
+
/// @return Reference to the stored element.
|
|
553
|
+
inline element_t const& top() const noexcept { return elements_[0]; }
|
|
554
|
+
inline void clear() noexcept { size_ = 0; }
|
|
555
|
+
|
|
556
|
+
bool reserve(std::size_t new_capacity) noexcept {
|
|
557
|
+
if (new_capacity < capacity_)
|
|
558
|
+
return true;
|
|
559
|
+
|
|
560
|
+
new_capacity = ceil2(new_capacity);
|
|
561
|
+
new_capacity = (std::max<std::size_t>)(new_capacity, (std::max<std::size_t>)(capacity_ * 2u, 16u));
|
|
562
|
+
auto allocator = allocator_t{};
|
|
563
|
+
auto new_elements = allocator.allocate(new_capacity);
|
|
564
|
+
if (!new_elements)
|
|
565
|
+
return false;
|
|
566
|
+
|
|
567
|
+
if (elements_) {
|
|
568
|
+
std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
|
|
569
|
+
allocator.deallocate(elements_, capacity_);
|
|
570
|
+
}
|
|
571
|
+
elements_ = new_elements;
|
|
572
|
+
capacity_ = new_capacity;
|
|
573
|
+
return new_elements;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
bool insert(element_t&& element) noexcept {
|
|
577
|
+
if (!reserve(size_ + 1))
|
|
578
|
+
return false;
|
|
579
|
+
|
|
580
|
+
insert_reserved(std::move(element));
|
|
581
|
+
return true;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
inline void insert_reserved(element_t&& element) noexcept {
|
|
585
|
+
new (&elements_[size_]) element_t(element);
|
|
586
|
+
size_++;
|
|
587
|
+
shift_up(size_ - 1);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
inline element_t pop() noexcept {
|
|
591
|
+
element_t result = top();
|
|
592
|
+
std::swap(elements_[0], elements_[size_ - 1]);
|
|
593
|
+
size_--;
|
|
594
|
+
elements_[size_].~element_t();
|
|
595
|
+
shift_down(0);
|
|
596
|
+
return result;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/** @brief Invalidates the "max-heap" property, transforming into ascending range. */
|
|
600
|
+
inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
|
|
601
|
+
inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
602
|
+
|
|
603
|
+
inline element_t* data() noexcept { return elements_; }
|
|
604
|
+
inline element_t const* data() const noexcept { return elements_; }
|
|
605
|
+
|
|
606
|
+
private:
|
|
607
|
+
inline std::size_t parent_idx(std::size_t i) const noexcept { return (i - 1u) / 2u; }
|
|
608
|
+
inline std::size_t left_child_idx(std::size_t i) const noexcept { return (i * 2u) + 1u; }
|
|
609
|
+
inline std::size_t right_child_idx(std::size_t i) const noexcept { return (i * 2u) + 2u; }
|
|
610
|
+
static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
|
|
611
|
+
|
|
612
|
+
void shift_up(std::size_t i) noexcept {
|
|
613
|
+
for (; i && less(elements_[parent_idx(i)], elements_[i]); i = parent_idx(i))
|
|
614
|
+
std::swap(elements_[parent_idx(i)], elements_[i]);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
void shift_down(std::size_t i) noexcept {
|
|
618
|
+
std::size_t max_idx = i;
|
|
619
|
+
|
|
620
|
+
std::size_t left = left_child_idx(i);
|
|
621
|
+
if (left < size_ && less(elements_[max_idx], elements_[left]))
|
|
622
|
+
max_idx = left;
|
|
623
|
+
|
|
624
|
+
std::size_t right = right_child_idx(i);
|
|
625
|
+
if (right < size_ && less(elements_[max_idx], elements_[right]))
|
|
626
|
+
max_idx = right;
|
|
627
|
+
|
|
628
|
+
if (i != max_idx) {
|
|
629
|
+
std::swap(elements_[i], elements_[max_idx]);
|
|
630
|
+
shift_down(max_idx);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
};
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* @brief Similar to `std::priority_queue`, but allows raw access to underlying
|
|
637
|
+
* memory and always keeps the data sorted. Ideal for small collections
|
|
638
|
+
* under 128 elements.
|
|
639
|
+
*/
|
|
640
|
+
template <typename element_at, //
|
|
641
|
+
typename comparator_at = std::less<void>, // <void> is needed before C++14.
|
|
642
|
+
typename allocator_at = std::allocator<element_at>> //
|
|
643
|
+
class sorted_buffer_gt {
|
|
644
|
+
public:
|
|
645
|
+
using element_t = element_at;
|
|
646
|
+
using comparator_t = comparator_at;
|
|
647
|
+
using allocator_t = allocator_at;
|
|
648
|
+
|
|
649
|
+
static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
|
|
650
|
+
static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
|
|
651
|
+
|
|
652
|
+
using value_type = element_t;
|
|
653
|
+
|
|
654
|
+
private:
|
|
655
|
+
element_t* elements_;
|
|
656
|
+
std::size_t size_;
|
|
657
|
+
std::size_t capacity_;
|
|
658
|
+
|
|
659
|
+
public:
|
|
660
|
+
sorted_buffer_gt() noexcept : elements_(nullptr), size_(0), capacity_(0) {}
|
|
661
|
+
|
|
662
|
+
sorted_buffer_gt(sorted_buffer_gt&& other) noexcept
|
|
663
|
+
: elements_(exchange(other.elements_, nullptr)), size_(exchange(other.size_, 0)),
|
|
664
|
+
capacity_(exchange(other.capacity_, 0)) {}
|
|
665
|
+
|
|
666
|
+
sorted_buffer_gt& operator=(sorted_buffer_gt&& other) noexcept {
|
|
667
|
+
std::swap(elements_, other.elements_);
|
|
668
|
+
std::swap(size_, other.size_);
|
|
669
|
+
std::swap(capacity_, other.capacity_);
|
|
670
|
+
return *this;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
sorted_buffer_gt(sorted_buffer_gt const&) = delete;
|
|
674
|
+
sorted_buffer_gt& operator=(sorted_buffer_gt const&) = delete;
|
|
675
|
+
|
|
676
|
+
~sorted_buffer_gt() noexcept { reset(); }
|
|
677
|
+
|
|
678
|
+
void reset() noexcept {
|
|
679
|
+
if (elements_)
|
|
680
|
+
allocator_t{}.deallocate(elements_, capacity_);
|
|
681
|
+
elements_ = nullptr;
|
|
682
|
+
capacity_ = 0;
|
|
683
|
+
size_ = 0;
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
inline bool empty() const noexcept { return !size_; }
|
|
687
|
+
inline std::size_t size() const noexcept { return size_; }
|
|
688
|
+
inline std::size_t capacity() const noexcept { return capacity_; }
|
|
689
|
+
inline element_t const& top() const noexcept { return elements_[size_ - 1]; }
|
|
690
|
+
inline void clear() noexcept { size_ = 0; }
|
|
691
|
+
|
|
692
|
+
bool reserve(std::size_t new_capacity) noexcept {
|
|
693
|
+
if (new_capacity < capacity_)
|
|
694
|
+
return true;
|
|
695
|
+
|
|
696
|
+
new_capacity = ceil2(new_capacity);
|
|
697
|
+
new_capacity = (std::max<std::size_t>)(new_capacity, (std::max<std::size_t>)(capacity_ * 2u, 16u));
|
|
698
|
+
auto allocator = allocator_t{};
|
|
699
|
+
auto new_elements = allocator.allocate(new_capacity);
|
|
700
|
+
if (!new_elements)
|
|
701
|
+
return false;
|
|
702
|
+
|
|
703
|
+
if (size_)
|
|
704
|
+
std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
|
|
705
|
+
if (elements_)
|
|
706
|
+
allocator.deallocate(elements_, capacity_);
|
|
707
|
+
|
|
708
|
+
elements_ = new_elements;
|
|
709
|
+
capacity_ = new_capacity;
|
|
710
|
+
return true;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
inline void insert_reserved(element_t&& element) noexcept {
|
|
714
|
+
std::size_t slot = size_ ? std::lower_bound(elements_, elements_ + size_, element, &less) - elements_ : 0;
|
|
715
|
+
std::size_t to_move = size_ - slot;
|
|
716
|
+
element_t* source = elements_ + size_ - 1;
|
|
717
|
+
for (; to_move; --to_move, --source)
|
|
718
|
+
source[1] = source[0];
|
|
719
|
+
elements_[slot] = element;
|
|
720
|
+
size_++;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* @return `true` if the entry was added, `false` if it wasn't relevant enough.
|
|
725
|
+
*/
|
|
726
|
+
inline bool insert(element_t&& element, std::size_t limit) noexcept {
|
|
727
|
+
std::size_t slot = size_ ? std::lower_bound(elements_, elements_ + size_, element, &less) - elements_ : 0;
|
|
728
|
+
if (slot == limit)
|
|
729
|
+
return false;
|
|
730
|
+
std::size_t to_move = size_ - slot - (size_ == limit);
|
|
731
|
+
element_t* source = elements_ + size_ - 1 - (size_ == limit);
|
|
732
|
+
for (; to_move; --to_move, --source)
|
|
733
|
+
source[1] = source[0];
|
|
734
|
+
elements_[slot] = element;
|
|
735
|
+
size_ += size_ != limit;
|
|
736
|
+
return true;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
inline element_t pop() noexcept {
|
|
740
|
+
size_--;
|
|
741
|
+
element_t result = elements_[size_];
|
|
742
|
+
elements_[size_].~element_t();
|
|
743
|
+
return result;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
void sort_ascending() noexcept {}
|
|
747
|
+
inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
748
|
+
|
|
749
|
+
inline element_t* data() noexcept { return elements_; }
|
|
750
|
+
inline element_t const* data() const noexcept { return elements_; }
|
|
751
|
+
|
|
752
|
+
private:
|
|
753
|
+
static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
|
|
754
|
+
};
|
|
755
|
+
|
|
756
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
757
|
+
#pragma pack(push, 1) // Pack struct elements on 1-byte alignment
|
|
758
|
+
#endif
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* @brief Five-byte integer type to address node clouds with over 4B entries.
|
|
762
|
+
*
|
|
763
|
+
* @note Avoid usage in 32bit environment
|
|
764
|
+
*/
|
|
765
|
+
class usearch_pack_m uint40_t {
|
|
766
|
+
unsigned char octets[5];
|
|
767
|
+
|
|
768
|
+
inline uint40_t& broadcast(unsigned char c) {
|
|
769
|
+
std::memset(octets, c, 5);
|
|
770
|
+
return *this;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
public:
|
|
774
|
+
inline uint40_t() noexcept { broadcast(0); }
|
|
775
|
+
inline uint40_t(std::uint32_t n) noexcept { std::memcpy(&octets[1], &n, 4); }
|
|
776
|
+
|
|
777
|
+
#ifdef USEARCH_64BIT_ENV
|
|
778
|
+
inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
|
|
779
|
+
#endif
|
|
780
|
+
|
|
781
|
+
uint40_t(uint40_t&&) = default;
|
|
782
|
+
uint40_t(uint40_t const&) = default;
|
|
783
|
+
uint40_t& operator=(uint40_t&&) = default;
|
|
784
|
+
uint40_t& operator=(uint40_t const&) = default;
|
|
785
|
+
|
|
786
|
+
#if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
|
|
787
|
+
inline uint40_t(std::size_t n) noexcept {
|
|
788
|
+
#ifdef USEARCH_64BIT_ENV
|
|
789
|
+
std::memcpy(octets, &n, 5);
|
|
790
|
+
#else
|
|
791
|
+
std::memcpy(octets, &n, 4);
|
|
792
|
+
#endif
|
|
793
|
+
}
|
|
794
|
+
#endif
|
|
795
|
+
|
|
796
|
+
inline operator std::size_t() const noexcept {
|
|
797
|
+
std::size_t result = 0;
|
|
798
|
+
#ifdef USEARCH_64BIT_ENV
|
|
799
|
+
std::memcpy(&result, octets, 5);
|
|
800
|
+
#else
|
|
801
|
+
std::memcpy(&result, octets + 1, 4);
|
|
802
|
+
#endif
|
|
803
|
+
return result;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
|
|
807
|
+
inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
|
|
808
|
+
};
|
|
809
|
+
|
|
810
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
811
|
+
#pragma pack(pop) // Reset alignment to default
|
|
812
|
+
#endif
|
|
813
|
+
|
|
814
|
+
static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
|
|
815
|
+
|
|
816
|
+
// clang-format off
|
|
817
|
+
template <typename key_at, typename std::enable_if<std::is_integral<key_at>::value>::type* = nullptr> key_at default_free_value() { return std::numeric_limits<key_at>::max(); }
|
|
818
|
+
template <typename key_at, typename std::enable_if<std::is_same<key_at, uint40_t>::value>::type* = nullptr> uint40_t default_free_value() { return uint40_t::max(); }
|
|
819
|
+
template <typename key_at, typename std::enable_if<!std::is_integral<key_at>::value && !std::is_same<key_at, uint40_t>::value>::type* = nullptr> key_at default_free_value() { return key_at(); }
|
|
820
|
+
// clang-format on
|
|
821
|
+
|
|
822
|
+
template <typename element_at> struct hash_gt {
|
|
823
|
+
std::size_t operator()(element_at const& element) const noexcept { return std::hash<element_at>{}(element); }
|
|
824
|
+
};
|
|
825
|
+
|
|
826
|
+
template <> struct hash_gt<uint40_t> {
|
|
827
|
+
std::size_t operator()(uint40_t const& element) const noexcept { return std::hash<std::size_t>{}(element); }
|
|
828
|
+
};
|
|
829
|
+
|
|
830
|
+
/**
|
|
831
|
+
* @brief Minimalistic hash-set implementation to track visited nodes during graph traversal.
|
|
832
|
+
*
|
|
833
|
+
* It doesn't support deletion of separate objects, but supports `clear`-ing all at once.
|
|
834
|
+
* It expects `reserve` to be called ahead of all insertions, so no resizes are needed.
|
|
835
|
+
* It also assumes `0xFF...FF` slots to be unused, to simplify the design.
|
|
836
|
+
* It uses linear probing, the number of slots is always a power of two, and it uses linear-probing
|
|
837
|
+
* in case of bucket collisions.
|
|
838
|
+
*/
|
|
839
|
+
template <typename element_at, typename hasher_at = hash_gt<element_at>, typename allocator_at = std::allocator<byte_t>>
|
|
840
|
+
class growing_hash_set_gt {
|
|
841
|
+
|
|
842
|
+
using element_t = element_at;
|
|
843
|
+
using hasher_t = hasher_at;
|
|
844
|
+
|
|
845
|
+
using allocator_t = allocator_at;
|
|
846
|
+
using byte_t = typename allocator_t::value_type;
|
|
847
|
+
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
848
|
+
|
|
849
|
+
element_t* slots_{};
|
|
850
|
+
/// @brief Number of slots.
|
|
851
|
+
std::size_t capacity_{};
|
|
852
|
+
/// @brief Number of populated.
|
|
853
|
+
std::size_t count_{};
|
|
854
|
+
hasher_t hasher_{};
|
|
855
|
+
|
|
856
|
+
public:
|
|
857
|
+
growing_hash_set_gt() noexcept {}
|
|
858
|
+
~growing_hash_set_gt() noexcept { reset(); }
|
|
859
|
+
|
|
860
|
+
explicit operator bool() const noexcept { return slots_; }
|
|
861
|
+
std::size_t size() const noexcept { return count_; }
|
|
862
|
+
|
|
863
|
+
void clear() noexcept {
|
|
864
|
+
if (slots_)
|
|
865
|
+
std::memset((void*)slots_, 0xFF, capacity_ * sizeof(element_t));
|
|
866
|
+
count_ = 0;
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
void reset() noexcept {
|
|
870
|
+
if (slots_)
|
|
871
|
+
allocator_t{}.deallocate((byte_t*)slots_, capacity_ * sizeof(element_t));
|
|
872
|
+
slots_ = nullptr;
|
|
873
|
+
capacity_ = 0;
|
|
874
|
+
count_ = 0;
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
growing_hash_set_gt(std::size_t capacity) noexcept
|
|
878
|
+
: slots_((element_t*)allocator_t{}.allocate(ceil2(capacity) * sizeof(element_t))),
|
|
879
|
+
capacity_(slots_ ? ceil2(capacity) : 0u), count_(0u) {
|
|
880
|
+
clear();
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
growing_hash_set_gt(growing_hash_set_gt&& other) noexcept {
|
|
884
|
+
slots_ = exchange(other.slots_, nullptr);
|
|
885
|
+
capacity_ = exchange(other.capacity_, 0);
|
|
886
|
+
count_ = exchange(other.count_, 0);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
growing_hash_set_gt& operator=(growing_hash_set_gt&& other) noexcept {
|
|
890
|
+
std::swap(slots_, other.slots_);
|
|
891
|
+
std::swap(capacity_, other.capacity_);
|
|
892
|
+
std::swap(count_, other.count_);
|
|
893
|
+
return *this;
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
growing_hash_set_gt(growing_hash_set_gt const&) = delete;
|
|
897
|
+
growing_hash_set_gt& operator=(growing_hash_set_gt const&) = delete;
|
|
898
|
+
|
|
899
|
+
inline bool test(element_t const& elem) const noexcept {
|
|
900
|
+
std::size_t index = hasher_(elem) & (capacity_ - 1);
|
|
901
|
+
while (slots_[index] != default_free_value<element_t>()) {
|
|
902
|
+
if (slots_[index] == elem)
|
|
903
|
+
return true;
|
|
904
|
+
|
|
905
|
+
index = (index + 1) & (capacity_ - 1);
|
|
906
|
+
}
|
|
907
|
+
return false;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
*
|
|
912
|
+
* @return Similar to `bitset_gt`, returns the previous value.
|
|
913
|
+
*/
|
|
914
|
+
inline bool set(element_t const& elem) noexcept {
|
|
915
|
+
std::size_t index = hasher_(elem) & (capacity_ - 1);
|
|
916
|
+
while (slots_[index] != default_free_value<element_t>()) {
|
|
917
|
+
// Already exists
|
|
918
|
+
if (slots_[index] == elem)
|
|
919
|
+
return true;
|
|
920
|
+
|
|
921
|
+
index = (index + 1) & (capacity_ - 1);
|
|
922
|
+
}
|
|
923
|
+
slots_[index] = elem;
|
|
924
|
+
++count_;
|
|
925
|
+
return false;
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
bool reserve(std::size_t new_capacity) noexcept {
|
|
929
|
+
new_capacity = (new_capacity * 5u) / 3u;
|
|
930
|
+
if (new_capacity <= capacity_)
|
|
931
|
+
return true;
|
|
932
|
+
|
|
933
|
+
new_capacity = ceil2(new_capacity);
|
|
934
|
+
element_t* new_slots = (element_t*)allocator_t{}.allocate(new_capacity * sizeof(element_t));
|
|
935
|
+
if (!new_slots)
|
|
936
|
+
return false;
|
|
937
|
+
|
|
938
|
+
std::memset((void*)new_slots, 0xFF, new_capacity * sizeof(element_t));
|
|
939
|
+
std::size_t new_count = count_;
|
|
940
|
+
if (count_) {
|
|
941
|
+
for (std::size_t old_index = 0; old_index != capacity_; ++old_index) {
|
|
942
|
+
if (slots_[old_index] == default_free_value<element_t>())
|
|
943
|
+
continue;
|
|
944
|
+
|
|
945
|
+
std::size_t new_index = hasher_(slots_[old_index]) & (new_capacity - 1);
|
|
946
|
+
while (new_slots[new_index] != default_free_value<element_t>())
|
|
947
|
+
new_index = (new_index + 1) & (new_capacity - 1);
|
|
948
|
+
new_slots[new_index] = slots_[old_index];
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
reset();
|
|
953
|
+
slots_ = new_slots;
|
|
954
|
+
capacity_ = new_capacity;
|
|
955
|
+
count_ = new_count;
|
|
956
|
+
return true;
|
|
957
|
+
}
|
|
958
|
+
};
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* @brief Basic single-threaded @b ring class, used for all kinds of task queues.
|
|
962
|
+
*/
|
|
963
|
+
template <typename element_at, typename allocator_at = std::allocator<element_at>> //
|
|
964
|
+
class ring_gt {
|
|
965
|
+
public:
|
|
966
|
+
using element_t = element_at;
|
|
967
|
+
using allocator_t = allocator_at;
|
|
968
|
+
|
|
969
|
+
static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
|
|
970
|
+
static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
|
|
971
|
+
|
|
972
|
+
using value_type = element_t;
|
|
973
|
+
|
|
974
|
+
private:
|
|
975
|
+
element_t* elements_{};
|
|
976
|
+
std::size_t capacity_{};
|
|
977
|
+
std::size_t head_{};
|
|
978
|
+
std::size_t tail_{};
|
|
979
|
+
bool empty_{true};
|
|
980
|
+
allocator_t allocator_{};
|
|
981
|
+
|
|
982
|
+
public:
|
|
983
|
+
explicit ring_gt(allocator_t const& alloc = allocator_t()) noexcept : allocator_(alloc) {}
|
|
984
|
+
|
|
985
|
+
ring_gt(ring_gt const&) = delete;
|
|
986
|
+
ring_gt& operator=(ring_gt const&) = delete;
|
|
987
|
+
|
|
988
|
+
ring_gt(ring_gt&& other) noexcept { swap(other); }
|
|
989
|
+
ring_gt& operator=(ring_gt&& other) noexcept {
|
|
990
|
+
swap(other);
|
|
991
|
+
return *this;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
void swap(ring_gt& other) noexcept {
|
|
995
|
+
std::swap(elements_, other.elements_);
|
|
996
|
+
std::swap(capacity_, other.capacity_);
|
|
997
|
+
std::swap(head_, other.head_);
|
|
998
|
+
std::swap(tail_, other.tail_);
|
|
999
|
+
std::swap(empty_, other.empty_);
|
|
1000
|
+
std::swap(allocator_, other.allocator_);
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
~ring_gt() noexcept { reset(); }
|
|
1004
|
+
|
|
1005
|
+
bool empty() const noexcept { return empty_; }
|
|
1006
|
+
size_t capacity() const noexcept { return capacity_; }
|
|
1007
|
+
size_t size() const noexcept {
|
|
1008
|
+
if (empty_)
|
|
1009
|
+
return 0;
|
|
1010
|
+
else if (head_ >= tail_)
|
|
1011
|
+
return head_ - tail_;
|
|
1012
|
+
else
|
|
1013
|
+
return capacity_ - (tail_ - head_);
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
void clear() noexcept {
|
|
1017
|
+
head_ = 0;
|
|
1018
|
+
tail_ = 0;
|
|
1019
|
+
empty_ = true;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
void reset() noexcept {
|
|
1023
|
+
if (elements_)
|
|
1024
|
+
allocator_.deallocate(elements_, capacity_);
|
|
1025
|
+
elements_ = nullptr;
|
|
1026
|
+
capacity_ = 0;
|
|
1027
|
+
head_ = 0;
|
|
1028
|
+
tail_ = 0;
|
|
1029
|
+
empty_ = true;
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
bool reserve(std::size_t n) noexcept {
|
|
1033
|
+
if (n < size())
|
|
1034
|
+
return false; // prevent data loss
|
|
1035
|
+
if (n <= capacity())
|
|
1036
|
+
return true;
|
|
1037
|
+
n = (std::max<std::size_t>)(ceil2(n), 64u);
|
|
1038
|
+
element_t* elements = allocator_.allocate(n);
|
|
1039
|
+
if (!elements)
|
|
1040
|
+
return false;
|
|
1041
|
+
|
|
1042
|
+
std::size_t i = 0;
|
|
1043
|
+
while (try_pop(elements[i]))
|
|
1044
|
+
i++;
|
|
1045
|
+
|
|
1046
|
+
reset();
|
|
1047
|
+
elements_ = elements;
|
|
1048
|
+
capacity_ = n;
|
|
1049
|
+
head_ = i;
|
|
1050
|
+
tail_ = 0;
|
|
1051
|
+
empty_ = (i == 0);
|
|
1052
|
+
return true;
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
void push(element_t const& value) noexcept {
|
|
1056
|
+
elements_[head_] = value;
|
|
1057
|
+
head_ = (head_ + 1) % capacity_;
|
|
1058
|
+
empty_ = false;
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
bool try_push(element_t const& value) noexcept {
|
|
1062
|
+
if (head_ == tail_ && !empty_)
|
|
1063
|
+
return false; // elements_ is full
|
|
1064
|
+
|
|
1065
|
+
return push(value);
|
|
1066
|
+
return true;
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
bool try_pop(element_t& value) noexcept {
|
|
1070
|
+
if (empty_)
|
|
1071
|
+
return false;
|
|
1072
|
+
|
|
1073
|
+
value = std::move(elements_[tail_]);
|
|
1074
|
+
tail_ = (tail_ + 1) % capacity_;
|
|
1075
|
+
empty_ = head_ == tail_;
|
|
1076
|
+
return true;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
element_t const& operator[](std::size_t i) const noexcept { return elements_[(tail_ + i) % capacity_]; }
|
|
1080
|
+
};
|
|
1081
|
+
|
|
1082
|
+
/// @brief Number of neighbors per graph node.
|
|
1083
|
+
/// Defaults to 32 in FAISS and 16 in hnswlib.
|
|
1084
|
+
/// > It is called `M` in the paper.
|
|
1085
|
+
constexpr std::size_t default_connectivity() { return 16; }
|
|
1086
|
+
|
|
1087
|
+
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
1088
|
+
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
1089
|
+
/// > It is called `efConstruction` in the paper.
|
|
1090
|
+
constexpr std::size_t default_expansion_add() { return 128; }
|
|
1091
|
+
|
|
1092
|
+
/// @brief Hyper-parameter controlling the quality of search.
|
|
1093
|
+
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1094
|
+
/// > It is called `ef` in the paper.
|
|
1095
|
+
constexpr std::size_t default_expansion_search() { return 64; }
|
|
1096
|
+
|
|
1097
|
+
constexpr std::size_t default_allocator_entry_bytes() { return 64; }
|
|
1098
|
+
|
|
1099
|
+
/**
|
|
1100
|
+
* @brief Configuration settings for the index construction.
|
|
1101
|
+
* Includes the main `::connectivity` parameter (`M` in the paper)
|
|
1102
|
+
* and two expansion factors - for construction and search.
|
|
1103
|
+
*/
|
|
1104
|
+
struct index_config_t {
|
|
1105
|
+
/// @brief Number of neighbors per graph node.
|
|
1106
|
+
/// Defaults to 32 in FAISS and 16 in hnswlib.
|
|
1107
|
+
/// > It is called `M` in the paper.
|
|
1108
|
+
std::size_t connectivity = default_connectivity();
|
|
1109
|
+
|
|
1110
|
+
/// @brief Number of neighbors per graph node in base level graph.
|
|
1111
|
+
/// Defaults to double of the other levels, so 64 in FAISS and 32 in hnswlib.
|
|
1112
|
+
/// > It is called `M0` in the paper.
|
|
1113
|
+
std::size_t connectivity_base = default_connectivity() * 2;
|
|
1114
|
+
|
|
1115
|
+
inline index_config_t() = default;
|
|
1116
|
+
inline index_config_t(std::size_t c) noexcept
|
|
1117
|
+
: connectivity(c ? c : default_connectivity()), connectivity_base(c ? c * 2 : default_connectivity() * 2) {}
|
|
1118
|
+
inline index_config_t(std::size_t c, std::size_t cb) noexcept
|
|
1119
|
+
: connectivity(c), connectivity_base((std::max)(c, cb)) {}
|
|
1120
|
+
};
|
|
1121
|
+
|
|
1122
|
+
struct index_limits_t {
|
|
1123
|
+
std::size_t members = 0;
|
|
1124
|
+
std::size_t threads_add = std::thread::hardware_concurrency();
|
|
1125
|
+
std::size_t threads_search = std::thread::hardware_concurrency();
|
|
1126
|
+
|
|
1127
|
+
inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
|
|
1128
|
+
inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
1129
|
+
inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
1130
|
+
inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
1131
|
+
};
|
|
1132
|
+
|
|
1133
|
+
struct index_update_config_t {
|
|
1134
|
+
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
1135
|
+
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
1136
|
+
/// > It is called `efConstruction` in the paper.
|
|
1137
|
+
std::size_t expansion = default_expansion_add();
|
|
1138
|
+
|
|
1139
|
+
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1140
|
+
std::size_t thread = 0;
|
|
1141
|
+
};
|
|
1142
|
+
|
|
1143
|
+
struct index_search_config_t {
|
|
1144
|
+
/// @brief Hyper-parameter controlling the quality of search.
|
|
1145
|
+
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1146
|
+
/// > It is called `ef` in the paper.
|
|
1147
|
+
std::size_t expansion = default_expansion_search();
|
|
1148
|
+
|
|
1149
|
+
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1150
|
+
std::size_t thread = 0;
|
|
1151
|
+
|
|
1152
|
+
/// @brief Brute-forces exhaustive search over all entries in the index.
|
|
1153
|
+
bool exact = false;
|
|
1154
|
+
};
|
|
1155
|
+
|
|
1156
|
+
struct index_cluster_config_t {
|
|
1157
|
+
/// @brief Hyper-parameter controlling the quality of search.
|
|
1158
|
+
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1159
|
+
/// > It is called `ef` in the paper.
|
|
1160
|
+
std::size_t expansion = default_expansion_search();
|
|
1161
|
+
|
|
1162
|
+
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1163
|
+
std::size_t thread = 0;
|
|
1164
|
+
};
|
|
1165
|
+
|
|
1166
|
+
struct index_copy_config_t {};
|
|
1167
|
+
|
|
1168
|
+
struct index_join_config_t {
|
|
1169
|
+
/// @brief Controls maximum number of proposals per man during stable marriage.
|
|
1170
|
+
std::size_t max_proposals = 0;
|
|
1171
|
+
|
|
1172
|
+
/// @brief Hyper-parameter controlling the quality of search.
|
|
1173
|
+
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1174
|
+
/// > It is called `ef` in the paper.
|
|
1175
|
+
std::size_t expansion = default_expansion_search();
|
|
1176
|
+
|
|
1177
|
+
/// @brief Brute-forces exhaustive search over all entries in the index.
|
|
1178
|
+
bool exact = false;
|
|
1179
|
+
};
|
|
1180
|
+
|
|
1181
|
+
/// @brief C++17 and newer version deprecate the `std::result_of`
|
|
1182
|
+
template <typename metric_at, typename... args_at>
|
|
1183
|
+
using return_type_gt =
|
|
1184
|
+
#if defined(USEARCH_DEFINED_CPP17)
|
|
1185
|
+
typename std::invoke_result<metric_at, args_at...>::type;
|
|
1186
|
+
#else
|
|
1187
|
+
typename std::result_of<metric_at(args_at...)>::type;
|
|
1188
|
+
#endif
|
|
1189
|
+
|
|
1190
|
+
/**
|
|
1191
|
+
* @brief An example of what a USearch-compatible ad-hoc filter would look like.
|
|
1192
|
+
*
|
|
1193
|
+
* A similar function object can be passed to search queries to further filter entries
|
|
1194
|
+
* on their auxiliary properties, such as some categorical keys stored in an external DBMS.
|
|
1195
|
+
*/
|
|
1196
|
+
struct dummy_predicate_t {
|
|
1197
|
+
template <typename member_at> constexpr bool operator()(member_at&&) const noexcept { return true; }
|
|
1198
|
+
};
|
|
1199
|
+
|
|
1200
|
+
/**
|
|
1201
|
+
* @brief An example of what a USearch-compatible ad-hoc operation on in-flight entries.
|
|
1202
|
+
*
|
|
1203
|
+
* This kind of callbacks is used when the engine is being updated and you want to patch
|
|
1204
|
+
* the entries, while their are still under locks - limiting concurrent access and providing
|
|
1205
|
+
* consistency.
|
|
1206
|
+
*/
|
|
1207
|
+
struct dummy_callback_t {
|
|
1208
|
+
template <typename member_at> void operator()(member_at&&) const noexcept {}
|
|
1209
|
+
};
|
|
1210
|
+
|
|
1211
|
+
/**
|
|
1212
|
+
* @brief An example of what a USearch-compatible progress-bar should look like.
|
|
1213
|
+
*
|
|
1214
|
+
* This is particularly helpful when handling long-running tasks, like serialization,
|
|
1215
|
+
* saving, and loading from disk, or index-level joins.
|
|
1216
|
+
* The reporter checks return value to continue or stop the process, `false` means need to stop.
|
|
1217
|
+
*/
|
|
1218
|
+
struct dummy_progress_t {
|
|
1219
|
+
inline bool operator()(std::size_t /*processed*/, std::size_t /*total*/) const noexcept { return true; }
|
|
1220
|
+
};
|
|
1221
|
+
|
|
1222
|
+
/**
|
|
1223
|
+
* @brief An example of what a USearch-compatible values prefetching mechanism should look like.
|
|
1224
|
+
*
|
|
1225
|
+
* USearch is designed to handle very large datasets, that may not fir into RAM. Fetching from
|
|
1226
|
+
* external memory is very expensive, so we've added a pre-fetching mechanism, that accepts
|
|
1227
|
+
* multiple objects at once, to cache in RAM ahead of the computation.
|
|
1228
|
+
* The received iterators support both `get_slot` and `get_key` operations.
|
|
1229
|
+
* An example usage may look like this:
|
|
1230
|
+
*
|
|
1231
|
+
* template <typename member_citerator_like_at>
|
|
1232
|
+
* inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {
|
|
1233
|
+
* for (; begin != end; ++begin)
|
|
1234
|
+
* io_uring_prefetch(offset_in_file(get_key(begin)));
|
|
1235
|
+
* }
|
|
1236
|
+
*/
|
|
1237
|
+
struct dummy_prefetch_t {
|
|
1238
|
+
template <typename member_citerator_like_at>
|
|
1239
|
+
inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {}
|
|
1240
|
+
};
|
|
1241
|
+
|
|
1242
|
+
/**
|
|
1243
|
+
* @brief An example of what a USearch-compatible executor (thread-pool) should look like.
|
|
1244
|
+
*
|
|
1245
|
+
* It's expected to have `parallel(callback)` API to schedule one task per thread;
|
|
1246
|
+
* an identical `fixed(count, callback)` and `dynamic(count, callback)` overloads that also accepts
|
|
1247
|
+
* the number of tasks, and somehow schedules them between threads; as well as `size()` to
|
|
1248
|
+
* determine the number of available threads.
|
|
1249
|
+
*/
|
|
1250
|
+
struct dummy_executor_t {
|
|
1251
|
+
dummy_executor_t() noexcept {}
|
|
1252
|
+
std::size_t size() const noexcept { return 1; }
|
|
1253
|
+
|
|
1254
|
+
template <typename thread_aware_function_at>
|
|
1255
|
+
void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept {
|
|
1256
|
+
for (std::size_t task_idx = 0; task_idx != tasks; ++task_idx)
|
|
1257
|
+
thread_aware_function(0, task_idx);
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
template <typename thread_aware_function_at>
|
|
1261
|
+
void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept {
|
|
1262
|
+
for (std::size_t task_idx = 0; task_idx != tasks; ++task_idx)
|
|
1263
|
+
if (!thread_aware_function(0, task_idx))
|
|
1264
|
+
break;
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
template <typename thread_aware_function_at>
|
|
1268
|
+
void parallel(thread_aware_function_at&& thread_aware_function) noexcept {
|
|
1269
|
+
thread_aware_function(0);
|
|
1270
|
+
}
|
|
1271
|
+
};
|
|
1272
|
+
|
|
1273
|
+
/**
|
|
1274
|
+
* @brief An example of what a USearch-compatible key-to-key mapping should look like.
|
|
1275
|
+
*
|
|
1276
|
+
* This is particularly helpful for "Semantic Joins", where we map entries of one collection
|
|
1277
|
+
* to entries of another. In asymmetric setups, where A -> B is needed, but B -> A is not,
|
|
1278
|
+
* this can be passed to minimize memory usage.
|
|
1279
|
+
*/
|
|
1280
|
+
struct dummy_key_to_key_mapping_t {
|
|
1281
|
+
struct member_ref_t {
|
|
1282
|
+
template <typename key_at> member_ref_t& operator=(key_at&&) noexcept { return *this; }
|
|
1283
|
+
};
|
|
1284
|
+
template <typename key_at> member_ref_t operator[](key_at&&) const noexcept { return {}; }
|
|
1285
|
+
};
|
|
1286
|
+
|
|
1287
|
+
/**
|
|
1288
|
+
* @brief Checks if the provided object has a dummy type, emulating an interface,
|
|
1289
|
+
* but performing no real computation.
|
|
1290
|
+
*/
|
|
1291
|
+
template <typename object_at> static constexpr bool is_dummy() {
|
|
1292
|
+
using object_t = typename std::remove_all_extents<object_at>::type;
|
|
1293
|
+
return std::is_same<typename std::decay<object_t>::type, dummy_predicate_t>::value || //
|
|
1294
|
+
std::is_same<typename std::decay<object_t>::type, dummy_callback_t>::value || //
|
|
1295
|
+
std::is_same<typename std::decay<object_t>::type, dummy_progress_t>::value || //
|
|
1296
|
+
std::is_same<typename std::decay<object_t>::type, dummy_prefetch_t>::value || //
|
|
1297
|
+
std::is_same<typename std::decay<object_t>::type, dummy_executor_t>::value || //
|
|
1298
|
+
std::is_same<typename std::decay<object_t>::type, dummy_key_to_key_mapping_t>::value;
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
template <typename, typename at> struct has_reset_gt {
|
|
1302
|
+
static_assert(std::integral_constant<at, false>::value, "Second template parameter needs to be of function type.");
|
|
1303
|
+
};
|
|
1304
|
+
|
|
1305
|
+
template <typename check_at, typename return_at, typename... args_at>
|
|
1306
|
+
struct has_reset_gt<check_at, return_at(args_at...)> {
|
|
1307
|
+
private:
|
|
1308
|
+
template <typename at>
|
|
1309
|
+
static constexpr auto check(at*) ->
|
|
1310
|
+
typename std::is_same<decltype(std::declval<at>().reset(std::declval<args_at>()...)), return_at>::type;
|
|
1311
|
+
template <typename> static constexpr std::false_type check(...);
|
|
1312
|
+
|
|
1313
|
+
typedef decltype(check<check_at>(0)) type;
|
|
1314
|
+
|
|
1315
|
+
public:
|
|
1316
|
+
static constexpr bool value = type::value;
|
|
1317
|
+
};
|
|
1318
|
+
|
|
1319
|
+
/**
|
|
1320
|
+
* @brief Checks if a certain class has a member function called `reset`.
|
|
1321
|
+
*/
|
|
1322
|
+
template <typename at> constexpr bool has_reset() { return has_reset_gt<at, void()>::value; }
|
|
1323
|
+
|
|
1324
|
+
struct serialization_result_t {
|
|
1325
|
+
error_t error;
|
|
1326
|
+
|
|
1327
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1328
|
+
serialization_result_t failed(error_t message) noexcept {
|
|
1329
|
+
error = std::move(message);
|
|
1330
|
+
return std::move(*this);
|
|
1331
|
+
}
|
|
1332
|
+
};
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* @brief Smart-pointer wrapping the LibC @b `FILE` for binary file @b outputs.
|
|
1336
|
+
*
|
|
1337
|
+
* This class raises no exceptions and corresponds errors through `serialization_result_t`.
|
|
1338
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1339
|
+
*/
|
|
1340
|
+
class output_file_t {
|
|
1341
|
+
char const* path_ = nullptr;
|
|
1342
|
+
std::FILE* file_ = nullptr;
|
|
1343
|
+
|
|
1344
|
+
public:
|
|
1345
|
+
output_file_t(char const* path) noexcept : path_(path) {}
|
|
1346
|
+
~output_file_t() noexcept { close(); }
|
|
1347
|
+
output_file_t(output_file_t&& other) noexcept
|
|
1348
|
+
: path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
|
|
1349
|
+
output_file_t& operator=(output_file_t&& other) noexcept {
|
|
1350
|
+
std::swap(path_, other.path_);
|
|
1351
|
+
std::swap(file_, other.file_);
|
|
1352
|
+
return *this;
|
|
1353
|
+
}
|
|
1354
|
+
serialization_result_t open_if_not() noexcept {
|
|
1355
|
+
serialization_result_t result;
|
|
1356
|
+
if (!file_)
|
|
1357
|
+
file_ = std::fopen(path_, "wb");
|
|
1358
|
+
if (!file_)
|
|
1359
|
+
return result.failed(std::strerror(errno));
|
|
1360
|
+
return result;
|
|
1361
|
+
}
|
|
1362
|
+
serialization_result_t write(void const* begin, std::size_t length) noexcept {
|
|
1363
|
+
serialization_result_t result;
|
|
1364
|
+
std::size_t written = std::fwrite(begin, length, 1, file_);
|
|
1365
|
+
if (length && !written)
|
|
1366
|
+
return result.failed(std::strerror(errno));
|
|
1367
|
+
return result;
|
|
1368
|
+
}
|
|
1369
|
+
void close() noexcept {
|
|
1370
|
+
if (file_)
|
|
1371
|
+
std::fclose(exchange(file_, nullptr));
|
|
1372
|
+
}
|
|
1373
|
+
};
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* @brief Smart-pointer wrapping the LibC @b `FILE` for binary files @b inputs.
|
|
1377
|
+
*
|
|
1378
|
+
* This class raises no exceptions and corresponds errors through `serialization_result_t`.
|
|
1379
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1380
|
+
*/
|
|
1381
|
+
class input_file_t {
|
|
1382
|
+
char const* path_ = nullptr;
|
|
1383
|
+
std::FILE* file_ = nullptr;
|
|
1384
|
+
|
|
1385
|
+
public:
|
|
1386
|
+
input_file_t(char const* path) noexcept : path_(path) {}
|
|
1387
|
+
~input_file_t() noexcept { close(); }
|
|
1388
|
+
input_file_t(input_file_t&& other) noexcept
|
|
1389
|
+
: path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
|
|
1390
|
+
input_file_t& operator=(input_file_t&& other) noexcept {
|
|
1391
|
+
std::swap(path_, other.path_);
|
|
1392
|
+
std::swap(file_, other.file_);
|
|
1393
|
+
return *this;
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
serialization_result_t open_if_not() noexcept {
|
|
1397
|
+
serialization_result_t result;
|
|
1398
|
+
if (!file_)
|
|
1399
|
+
file_ = std::fopen(path_, "rb");
|
|
1400
|
+
if (!file_)
|
|
1401
|
+
return result.failed(std::strerror(errno));
|
|
1402
|
+
return result;
|
|
1403
|
+
}
|
|
1404
|
+
serialization_result_t read(void* begin, std::size_t length) noexcept {
|
|
1405
|
+
serialization_result_t result;
|
|
1406
|
+
std::size_t read = std::fread(begin, length, 1, file_);
|
|
1407
|
+
if (length && !read)
|
|
1408
|
+
return result.failed(std::feof(file_) ? "End of file reached!" : std::strerror(errno));
|
|
1409
|
+
return result;
|
|
1410
|
+
}
|
|
1411
|
+
void close() noexcept {
|
|
1412
|
+
if (file_)
|
|
1413
|
+
std::fclose(exchange(file_, nullptr));
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
explicit operator bool() const noexcept { return file_; }
|
|
1417
|
+
bool seek_to(std::size_t progress) noexcept {
|
|
1418
|
+
return std::fseek(file_, static_cast<long>(progress), SEEK_SET) == 0;
|
|
1419
|
+
}
|
|
1420
|
+
bool seek_to_end() noexcept { return std::fseek(file_, 0L, SEEK_END) == 0; }
|
|
1421
|
+
bool infer_progress(std::size_t& progress) noexcept {
|
|
1422
|
+
long int result = std::ftell(file_);
|
|
1423
|
+
if (result == -1L)
|
|
1424
|
+
return false;
|
|
1425
|
+
progress = static_cast<std::size_t>(result);
|
|
1426
|
+
return true;
|
|
1427
|
+
}
|
|
1428
|
+
};
|
|
1429
|
+
|
|
1430
|
+
/**
|
|
1431
|
+
* @brief Represents a memory-mapped file or a pre-allocated anonymous memory region.
|
|
1432
|
+
*
|
|
1433
|
+
* This class provides a convenient way to memory-map a file and access its contents as a block of
|
|
1434
|
+
* memory. The class handles platform-specific memory-mapping operations on Windows, Linux, and MacOS.
|
|
1435
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1436
|
+
*/
|
|
1437
|
+
class memory_mapped_file_t {
|
|
1438
|
+
char const* path_{}; /**< The path to the file to be memory-mapped. */
|
|
1439
|
+
void* ptr_{}; /**< A pointer to the memory-mapping. */
|
|
1440
|
+
size_t length_{}; /**< The length of the memory-mapped file in bytes. */
|
|
1441
|
+
|
|
1442
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1443
|
+
HANDLE file_handle_{}; /**< The file handle on Windows. */
|
|
1444
|
+
HANDLE mapping_handle_{}; /**< The mapping handle on Windows. */
|
|
1445
|
+
#else
|
|
1446
|
+
int file_descriptor_{}; /**< The file descriptor on Linux and MacOS. */
|
|
1447
|
+
#endif
|
|
1448
|
+
|
|
1449
|
+
public:
|
|
1450
|
+
explicit operator bool() const noexcept { return ptr_ != nullptr; }
|
|
1451
|
+
byte_t* data() noexcept { return reinterpret_cast<byte_t*>(ptr_); }
|
|
1452
|
+
byte_t const* data() const noexcept { return reinterpret_cast<byte_t const*>(ptr_); }
|
|
1453
|
+
std::size_t size() const noexcept { return static_cast<std::size_t>(length_); }
|
|
1454
|
+
|
|
1455
|
+
memory_mapped_file_t() noexcept {}
|
|
1456
|
+
memory_mapped_file_t(char const* path) noexcept : path_(path) {}
|
|
1457
|
+
~memory_mapped_file_t() noexcept { close(); }
|
|
1458
|
+
memory_mapped_file_t(memory_mapped_file_t&& other) noexcept
|
|
1459
|
+
: path_(exchange(other.path_, nullptr)), ptr_(exchange(other.ptr_, nullptr)),
|
|
1460
|
+
length_(exchange(other.length_, 0)),
|
|
1461
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1462
|
+
file_handle_(exchange(other.file_handle_, nullptr)), mapping_handle_(exchange(other.mapping_handle_, nullptr))
|
|
1463
|
+
#else
|
|
1464
|
+
file_descriptor_(exchange(other.file_descriptor_, 0))
|
|
1465
|
+
#endif
|
|
1466
|
+
{
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
memory_mapped_file_t(byte_t* data, std::size_t length) noexcept : ptr_(data), length_(length) {}
|
|
1470
|
+
|
|
1471
|
+
memory_mapped_file_t& operator=(memory_mapped_file_t&& other) noexcept {
|
|
1472
|
+
std::swap(path_, other.path_);
|
|
1473
|
+
std::swap(ptr_, other.ptr_);
|
|
1474
|
+
std::swap(length_, other.length_);
|
|
1475
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1476
|
+
std::swap(file_handle_, other.file_handle_);
|
|
1477
|
+
std::swap(mapping_handle_, other.mapping_handle_);
|
|
1478
|
+
#else
|
|
1479
|
+
std::swap(file_descriptor_, other.file_descriptor_);
|
|
1480
|
+
#endif
|
|
1481
|
+
return *this;
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
serialization_result_t open_if_not() noexcept {
|
|
1485
|
+
serialization_result_t result;
|
|
1486
|
+
if (!path_ || ptr_)
|
|
1487
|
+
return result;
|
|
1488
|
+
|
|
1489
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1490
|
+
|
|
1491
|
+
HANDLE file_handle =
|
|
1492
|
+
CreateFile(path_, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
|
|
1493
|
+
if (file_handle == INVALID_HANDLE_VALUE)
|
|
1494
|
+
return result.failed("Opening file failed!");
|
|
1495
|
+
|
|
1496
|
+
std::size_t file_length = GetFileSize(file_handle, 0);
|
|
1497
|
+
HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
|
|
1498
|
+
if (mapping_handle == 0) {
|
|
1499
|
+
CloseHandle(file_handle);
|
|
1500
|
+
return result.failed("Mapping file failed!");
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
|
|
1504
|
+
if (file == 0) {
|
|
1505
|
+
CloseHandle(mapping_handle);
|
|
1506
|
+
CloseHandle(file_handle);
|
|
1507
|
+
return result.failed("View the map failed!");
|
|
1508
|
+
}
|
|
1509
|
+
file_handle_ = file_handle;
|
|
1510
|
+
mapping_handle_ = mapping_handle;
|
|
1511
|
+
ptr_ = file;
|
|
1512
|
+
length_ = file_length;
|
|
1513
|
+
#else
|
|
1514
|
+
|
|
1515
|
+
#if defined(USEARCH_DEFINED_LINUX)
|
|
1516
|
+
int descriptor = open(path_, O_RDONLY | O_NOATIME);
|
|
1517
|
+
#else
|
|
1518
|
+
int descriptor = open(path_, O_RDONLY);
|
|
1519
|
+
#endif
|
|
1520
|
+
if (descriptor < 0)
|
|
1521
|
+
return result.failed(std::strerror(errno));
|
|
1522
|
+
|
|
1523
|
+
// Estimate the file size
|
|
1524
|
+
struct stat file_stat;
|
|
1525
|
+
int fstat_status = fstat(descriptor, &file_stat);
|
|
1526
|
+
if (fstat_status < 0) {
|
|
1527
|
+
::close(descriptor);
|
|
1528
|
+
return result.failed(std::strerror(errno));
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
// Map the entire file
|
|
1532
|
+
byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_SHARED, descriptor, 0);
|
|
1533
|
+
if (file == MAP_FAILED) {
|
|
1534
|
+
::close(descriptor);
|
|
1535
|
+
return result.failed(std::strerror(errno));
|
|
1536
|
+
}
|
|
1537
|
+
file_descriptor_ = descriptor;
|
|
1538
|
+
ptr_ = file;
|
|
1539
|
+
length_ = file_stat.st_size;
|
|
1540
|
+
#endif // Platform specific code
|
|
1541
|
+
return result;
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
void close() noexcept {
|
|
1545
|
+
if (!path_) {
|
|
1546
|
+
ptr_ = nullptr;
|
|
1547
|
+
length_ = 0;
|
|
1548
|
+
return;
|
|
1549
|
+
}
|
|
1550
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1551
|
+
UnmapViewOfFile(ptr_);
|
|
1552
|
+
CloseHandle(mapping_handle_);
|
|
1553
|
+
CloseHandle(file_handle_);
|
|
1554
|
+
mapping_handle_ = nullptr;
|
|
1555
|
+
file_handle_ = nullptr;
|
|
1556
|
+
#else
|
|
1557
|
+
munmap(ptr_, length_);
|
|
1558
|
+
::close(file_descriptor_);
|
|
1559
|
+
file_descriptor_ = 0;
|
|
1560
|
+
#endif
|
|
1561
|
+
ptr_ = nullptr;
|
|
1562
|
+
length_ = 0;
|
|
1563
|
+
}
|
|
1564
|
+
};
|
|
1565
|
+
|
|
1566
|
+
struct index_serialized_header_t {
|
|
1567
|
+
std::uint64_t size = 0;
|
|
1568
|
+
std::uint64_t connectivity = 0;
|
|
1569
|
+
std::uint64_t connectivity_base = 0;
|
|
1570
|
+
std::uint64_t max_level = 0;
|
|
1571
|
+
std::uint64_t entry_slot = 0;
|
|
1572
|
+
};
|
|
1573
|
+
|
|
1574
|
+
using default_key_t = std::uint64_t;
|
|
1575
|
+
using default_slot_t = std::uint32_t;
|
|
1576
|
+
using default_distance_t = float;
|
|
1577
|
+
|
|
1578
|
+
template <typename key_at = default_key_t> struct member_gt {
|
|
1579
|
+
key_at key;
|
|
1580
|
+
std::size_t slot;
|
|
1581
|
+
};
|
|
1582
|
+
|
|
1583
|
+
template <typename key_at> inline std::size_t get_slot(member_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1584
|
+
template <typename key_at> inline key_at get_key(member_gt<key_at> const& m) noexcept { return m.key; }
|
|
1585
|
+
|
|
1586
|
+
template <typename key_at = default_key_t> struct member_cref_gt {
|
|
1587
|
+
misaligned_ref_gt<key_at const> key;
|
|
1588
|
+
std::size_t slot;
|
|
1589
|
+
};
|
|
1590
|
+
|
|
1591
|
+
template <typename key_at> inline std::size_t get_slot(member_cref_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1592
|
+
template <typename key_at> inline key_at get_key(member_cref_gt<key_at> const& m) noexcept { return m.key; }
|
|
1593
|
+
|
|
1594
|
+
template <typename key_at = default_key_t> struct member_ref_gt {
|
|
1595
|
+
misaligned_ref_gt<key_at> key;
|
|
1596
|
+
std::size_t slot;
|
|
1597
|
+
|
|
1598
|
+
inline operator member_cref_gt<key_at>() const noexcept { return {key.ptr(), slot}; }
|
|
1599
|
+
};
|
|
1600
|
+
|
|
1601
|
+
template <typename key_at> inline std::size_t get_slot(member_ref_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1602
|
+
template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m) noexcept { return m.key; }
|
|
1603
|
+
|
|
1604
|
+
/**
|
|
1605
|
+
* @brief Approximate Nearest Neighbors Search @b index-structure using the
|
|
1606
|
+
* Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
|
|
1607
|
+
* If classical containers store @b Key->Value mappings, this one can
|
|
1608
|
+
* be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
|
|
1609
|
+
*
|
|
1610
|
+
* Unlike most implementations, this one is generic anc can be used for any search,
|
|
1611
|
+
* not just within equi-dimensional vectors. Examples range from texts to similar Chess
|
|
1612
|
+
* positions.
|
|
1613
|
+
*
|
|
1614
|
+
* @tparam key_at
|
|
1615
|
+
* The type of primary objects stored in the index.
|
|
1616
|
+
* The values, to which those map, are not managed by the same index structure.
|
|
1617
|
+
*
|
|
1618
|
+
* @tparam compressed_slot_at
|
|
1619
|
+
* The smallest unsigned integer type to address indexed elements.
|
|
1620
|
+
* It is used internally to maximize space-efficiency and is generally
|
|
1621
|
+
* up-casted to @b `std::size_t` in public interfaces.
|
|
1622
|
+
* Can be a built-in @b `uint32_t`, `uint64_t`, or our custom @b `uint40_t`.
|
|
1623
|
+
* Which makes the most sense for 4B+ entry indexes.
|
|
1624
|
+
*
|
|
1625
|
+
* @tparam dynamic_allocator_at
|
|
1626
|
+
* Dynamic memory allocator for temporary buffers, visits indicators, and
|
|
1627
|
+
* priority queues, needed during construction and traversals of graphs.
|
|
1628
|
+
* The allocated buffers may be uninitialized.
|
|
1629
|
+
*
|
|
1630
|
+
* @tparam tape_allocator_at
|
|
1631
|
+
* Potentially different memory allocator for primary allocations of nodes and vectors.
|
|
1632
|
+
* It would never `deallocate` separate entries, and would only free all the space at once.
|
|
1633
|
+
* The allocated buffers may be uninitialized.
|
|
1634
|
+
*
|
|
1635
|
+
* @section Features
|
|
1636
|
+
*
|
|
1637
|
+
* - Thread-safe for concurrent construction, search, and updates.
|
|
1638
|
+
* - Doesn't allocate new threads, and reuses the ones its called from.
|
|
1639
|
+
* - Allows storing value externally, managing just the similarity index.
|
|
1640
|
+
* - Joins.
|
|
1641
|
+
|
|
1642
|
+
* @section Usage
|
|
1643
|
+
*
|
|
1644
|
+
* @subsection Exceptions
|
|
1645
|
+
*
|
|
1646
|
+
* None of the methods throw exceptions in the "Release" compilation mode.
|
|
1647
|
+
* It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
|
|
1648
|
+
* safe to copy.
|
|
1649
|
+
*
|
|
1650
|
+
* @subsection Serialization
|
|
1651
|
+
*
|
|
1652
|
+
* When serialized, doesn't include any additional metadata.
|
|
1653
|
+
* It is just the multi-level proximity-graph. You may want to store metadata about
|
|
1654
|
+
* the used metric and key types somewhere else.
|
|
1655
|
+
*
|
|
1656
|
+
* @section Implementation Details
|
|
1657
|
+
*
|
|
1658
|
+
* Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
|
|
1659
|
+
* Every added vector forms a node in one or more levels of the graph.
|
|
1660
|
+
* Every node is present in the base level. Every following level contains a smaller
|
|
1661
|
+
* fraction of nodes. During search, the operation starts with the smaller levels
|
|
1662
|
+
* and zooms-in on every following iteration of larger graph traversals.
|
|
1663
|
+
*
|
|
1664
|
+
* Just one memory allocation is performed regardless of the number of levels.
|
|
1665
|
+
* The adjacency lists across all levels are concatenated into that single buffer.
|
|
1666
|
+
* That buffer starts with a "head", that stores the metadata, such as the
|
|
1667
|
+
* tallest "level" of the graph that it belongs to, the external "key", and the
|
|
1668
|
+
* number of "dimensions" in the vector.
|
|
1669
|
+
*
|
|
1670
|
+
* @section Metrics, Predicates and Callbacks
|
|
1671
|
+
*
|
|
1672
|
+
*
|
|
1673
|
+
* @section Smart References and Iterators
|
|
1674
|
+
*
|
|
1675
|
+
* - `member_citerator_t` and `member_iterator_t` have only slots, no indirections.
|
|
1676
|
+
*
|
|
1677
|
+
* - `member_cref_t` and `member_ref_t` contains the `slot` and a reference
|
|
1678
|
+
* to the key. So it passes through 1 level of visited_members in `nodes_`.
|
|
1679
|
+
* Retrieving the key via `get_key` will cause fetching yet another cache line.
|
|
1680
|
+
*
|
|
1681
|
+
* - `member_gt` contains an already prefetched copy of the key.
|
|
1682
|
+
*
|
|
1683
|
+
*/
|
|
1684
|
+
template <typename distance_at = default_distance_t, //
|
|
1685
|
+
typename key_at = default_key_t, //
|
|
1686
|
+
typename compressed_slot_at = default_slot_t, //
|
|
1687
|
+
typename dynamic_allocator_at = std::allocator<byte_t>, //
|
|
1688
|
+
typename tape_allocator_at = dynamic_allocator_at> //
|
|
1689
|
+
class index_gt {
|
|
1690
|
+
public:
|
|
1691
|
+
using distance_t = distance_at;
|
|
1692
|
+
using vector_key_t = key_at;
|
|
1693
|
+
using key_t = vector_key_t;
|
|
1694
|
+
using compressed_slot_t = compressed_slot_at;
|
|
1695
|
+
using dynamic_allocator_t = dynamic_allocator_at;
|
|
1696
|
+
using tape_allocator_t = tape_allocator_at;
|
|
1697
|
+
static_assert(sizeof(vector_key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
|
|
1698
|
+
|
|
1699
|
+
using member_cref_t = member_cref_gt<vector_key_t>;
|
|
1700
|
+
using member_ref_t = member_ref_gt<vector_key_t>;
|
|
1701
|
+
|
|
1702
|
+
template <typename ref_at, typename index_at> class member_iterator_gt {
|
|
1703
|
+
using ref_t = ref_at;
|
|
1704
|
+
using index_t = index_at;
|
|
1705
|
+
|
|
1706
|
+
friend class index_gt;
|
|
1707
|
+
member_iterator_gt() noexcept {}
|
|
1708
|
+
member_iterator_gt(index_t* index, std::size_t slot) noexcept : index_(index), slot_(slot) {}
|
|
1709
|
+
|
|
1710
|
+
index_t* index_{};
|
|
1711
|
+
std::size_t slot_{};
|
|
1712
|
+
|
|
1713
|
+
public:
|
|
1714
|
+
using iterator_category = std::random_access_iterator_tag;
|
|
1715
|
+
using value_type = ref_t;
|
|
1716
|
+
using difference_type = std::ptrdiff_t;
|
|
1717
|
+
using pointer = void;
|
|
1718
|
+
using reference = ref_t;
|
|
1719
|
+
|
|
1720
|
+
reference operator*() const noexcept { return {index_->node_at_(slot_).key(), slot_}; }
|
|
1721
|
+
vector_key_t key() const noexcept { return index_->node_at_(slot_).key(); }
|
|
1722
|
+
|
|
1723
|
+
friend inline std::size_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
|
|
1724
|
+
friend inline vector_key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
|
|
1725
|
+
|
|
1726
|
+
member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
|
|
1727
|
+
member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
|
|
1728
|
+
member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
|
|
1729
|
+
member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
|
|
1730
|
+
|
|
1731
|
+
// clang-format off
|
|
1732
|
+
member_iterator_gt& operator++() noexcept { slot_ += 1; return *this; }
|
|
1733
|
+
member_iterator_gt& operator--() noexcept { slot_ -= 1; return *this; }
|
|
1734
|
+
member_iterator_gt& operator+=(difference_type d) noexcept { slot_ += d; return *this; }
|
|
1735
|
+
member_iterator_gt& operator-=(difference_type d) noexcept { slot_ -= d; return *this; }
|
|
1736
|
+
bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
|
|
1737
|
+
bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
|
|
1738
|
+
// clang-format on
|
|
1739
|
+
};
|
|
1740
|
+
|
|
1741
|
+
using member_iterator_t = member_iterator_gt<member_ref_t, index_gt>;
|
|
1742
|
+
using member_citerator_t = member_iterator_gt<member_cref_t, index_gt const>;
|
|
1743
|
+
|
|
1744
|
+
// STL compatibility:
|
|
1745
|
+
using value_type = vector_key_t;
|
|
1746
|
+
using allocator_type = dynamic_allocator_t;
|
|
1747
|
+
using size_type = std::size_t;
|
|
1748
|
+
using difference_type = std::ptrdiff_t;
|
|
1749
|
+
using reference = member_ref_t;
|
|
1750
|
+
using const_reference = member_cref_t;
|
|
1751
|
+
using pointer = void;
|
|
1752
|
+
using const_pointer = void;
|
|
1753
|
+
using iterator = member_iterator_t;
|
|
1754
|
+
using const_iterator = member_citerator_t;
|
|
1755
|
+
using reverse_iterator = std::reverse_iterator<member_iterator_t>;
|
|
1756
|
+
using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
|
|
1757
|
+
|
|
1758
|
+
using dynamic_allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
|
|
1759
|
+
using byte_t = typename dynamic_allocator_t::value_type;
|
|
1760
|
+
static_assert( //
|
|
1761
|
+
sizeof(byte_t) == 1, //
|
|
1762
|
+
"Primary allocator must allocate separate addressable bytes");
|
|
1763
|
+
|
|
1764
|
+
using tape_allocator_traits_t = std::allocator_traits<tape_allocator_t>;
|
|
1765
|
+
static_assert( //
|
|
1766
|
+
sizeof(typename tape_allocator_traits_t::value_type) == 1, //
|
|
1767
|
+
"Tape allocator must allocate separate addressable bytes");
|
|
1768
|
+
|
|
1769
|
+
private:
|
|
1770
|
+
/**
|
|
1771
|
+
* @brief Integer for the number of node neighbors at a specific level of the
|
|
1772
|
+
* multi-level graph. It's selected to be `std::uint32_t` to improve the
|
|
1773
|
+
* alignment in most common cases.
|
|
1774
|
+
*/
|
|
1775
|
+
using neighbors_count_t = std::uint32_t;
|
|
1776
|
+
using level_t = std::int16_t;
|
|
1777
|
+
|
|
1778
|
+
/**
|
|
1779
|
+
* @brief How many bytes of memory are needed to form the "head" of the node.
|
|
1780
|
+
*/
|
|
1781
|
+
static constexpr std::size_t node_head_bytes_() { return sizeof(vector_key_t) + sizeof(level_t); }
|
|
1782
|
+
|
|
1783
|
+
using nodes_mutexes_t = bitset_gt<dynamic_allocator_t>;
|
|
1784
|
+
|
|
1785
|
+
using visits_hash_set_t = growing_hash_set_gt<compressed_slot_t, hash_gt<compressed_slot_t>, dynamic_allocator_t>;
|
|
1786
|
+
|
|
1787
|
+
struct precomputed_constants_t {
|
|
1788
|
+
double inverse_log_connectivity{};
|
|
1789
|
+
std::size_t neighbors_bytes{};
|
|
1790
|
+
std::size_t neighbors_base_bytes{};
|
|
1791
|
+
};
|
|
1792
|
+
/// @brief A space-efficient internal data-structure used in graph traversal queues.
|
|
1793
|
+
struct candidate_t {
|
|
1794
|
+
distance_t distance;
|
|
1795
|
+
compressed_slot_t slot;
|
|
1796
|
+
inline bool operator<(candidate_t other) const noexcept { return distance < other.distance; }
|
|
1797
|
+
};
|
|
1798
|
+
|
|
1799
|
+
using candidates_view_t = span_gt<candidate_t const>;
|
|
1800
|
+
using candidates_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<candidate_t>;
|
|
1801
|
+
using top_candidates_t = sorted_buffer_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
|
|
1802
|
+
using next_candidates_t = max_heap_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
|
|
1803
|
+
|
|
1804
|
+
/**
|
|
1805
|
+
* @brief A loosely-structured handle for every node. One such node is created for every member.
|
|
1806
|
+
* To minimize memory usage and maximize the number of entries per cache-line, it only
|
|
1807
|
+
* stores to pointers. The internal tape starts with a `vector_key_t` @b key, then
|
|
1808
|
+
* a `level_t` for the number of graph @b levels in which this member appears,
|
|
1809
|
+
* then the { `neighbors_count_t`, `compressed_slot_t`, `compressed_slot_t` ... } sequences
|
|
1810
|
+
* for @b each-level.
|
|
1811
|
+
*/
|
|
1812
|
+
class node_t {
|
|
1813
|
+
byte_t* tape_{};
|
|
1814
|
+
|
|
1815
|
+
public:
|
|
1816
|
+
explicit node_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1817
|
+
byte_t* tape() const noexcept { return tape_; }
|
|
1818
|
+
byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
|
|
1819
|
+
explicit operator bool() const noexcept { return tape_; }
|
|
1820
|
+
|
|
1821
|
+
node_t() = default;
|
|
1822
|
+
node_t(node_t const&) = default;
|
|
1823
|
+
node_t& operator=(node_t const&) = default;
|
|
1824
|
+
|
|
1825
|
+
misaligned_ref_gt<vector_key_t const> ckey() const noexcept { return {tape_}; }
|
|
1826
|
+
misaligned_ref_gt<vector_key_t> key() const noexcept { return {tape_}; }
|
|
1827
|
+
misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(vector_key_t)}; }
|
|
1828
|
+
|
|
1829
|
+
void key(vector_key_t v) noexcept { return misaligned_store<vector_key_t>(tape_, v); }
|
|
1830
|
+
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(vector_key_t), v); }
|
|
1831
|
+
};
|
|
1832
|
+
|
|
1833
|
+
static_assert(std::is_trivially_copy_constructible<node_t>::value, "Nodes must be light!");
|
|
1834
|
+
static_assert(std::is_trivially_destructible<node_t>::value, "Nodes must be light!");
|
|
1835
|
+
|
|
1836
|
+
/**
|
|
1837
|
+
* @brief A slice of the node's tape, containing a the list of neighbors
|
|
1838
|
+
* for a node in a single graph level. It's pre-allocated to fit
|
|
1839
|
+
* as many neighbors "slots", as may be needed at the target level,
|
|
1840
|
+
* and starts with a single integer `neighbors_count_t` counter.
|
|
1841
|
+
*/
|
|
1842
|
+
class neighbors_ref_t {
|
|
1843
|
+
byte_t* tape_;
|
|
1844
|
+
|
|
1845
|
+
static constexpr std::size_t shift(std::size_t i = 0) {
|
|
1846
|
+
return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
public:
|
|
1850
|
+
neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1851
|
+
misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
|
|
1852
|
+
misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
|
|
1853
|
+
misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
|
|
1854
|
+
misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
|
|
1855
|
+
compressed_slot_t operator[](std::size_t i) const noexcept {
|
|
1856
|
+
return misaligned_load<compressed_slot_t>(tape_ + shift(i));
|
|
1857
|
+
}
|
|
1858
|
+
std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
|
|
1859
|
+
void clear() noexcept {
|
|
1860
|
+
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1861
|
+
std::memset(tape_, 0, shift(n));
|
|
1862
|
+
// misaligned_store<neighbors_count_t>(tape_, 0);
|
|
1863
|
+
}
|
|
1864
|
+
void push_back(compressed_slot_t slot) noexcept {
|
|
1865
|
+
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1866
|
+
misaligned_store<compressed_slot_t>(tape_ + shift(n), slot);
|
|
1867
|
+
misaligned_store<neighbors_count_t>(tape_, n + 1);
|
|
1868
|
+
}
|
|
1869
|
+
};
|
|
1870
|
+
|
|
1871
|
+
/**
|
|
1872
|
+
* @brief A package of all kinds of temporary data-structures, that the threads
|
|
1873
|
+
* would reuse to process requests. Similar to having all of those as
|
|
1874
|
+
* separate `thread_local` global variables.
|
|
1875
|
+
*/
|
|
1876
|
+
struct usearch_align_m context_t {
|
|
1877
|
+
top_candidates_t top_candidates{};
|
|
1878
|
+
next_candidates_t next_candidates{};
|
|
1879
|
+
visits_hash_set_t visits{};
|
|
1880
|
+
std::default_random_engine level_generator{};
|
|
1881
|
+
std::size_t iteration_cycles{};
|
|
1882
|
+
std::size_t computed_distances_count{};
|
|
1883
|
+
|
|
1884
|
+
template <typename value_at, typename metric_at, typename entry_at> //
|
|
1885
|
+
inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1886
|
+
static_assert( //
|
|
1887
|
+
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1888
|
+
"Unexpected type");
|
|
1889
|
+
|
|
1890
|
+
computed_distances_count++;
|
|
1891
|
+
return metric(first, second);
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
template <typename metric_at, typename entry_at> //
|
|
1895
|
+
inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1896
|
+
static_assert( //
|
|
1897
|
+
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1898
|
+
"Unexpected type");
|
|
1899
|
+
|
|
1900
|
+
computed_distances_count++;
|
|
1901
|
+
return metric(first, second);
|
|
1902
|
+
}
|
|
1903
|
+
};
|
|
1904
|
+
|
|
1905
|
+
index_config_t config_{};
|
|
1906
|
+
index_limits_t limits_{};
|
|
1907
|
+
|
|
1908
|
+
mutable dynamic_allocator_t dynamic_allocator_{};
|
|
1909
|
+
tape_allocator_t tape_allocator_{};
|
|
1910
|
+
|
|
1911
|
+
precomputed_constants_t pre_{};
|
|
1912
|
+
memory_mapped_file_t viewed_file_{};
|
|
1913
|
+
|
|
1914
|
+
/// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
|
|
1915
|
+
usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
|
|
1916
|
+
|
|
1917
|
+
/// @brief Number of "slots" already storing non-null nodes.
|
|
1918
|
+
usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
|
|
1919
|
+
|
|
1920
|
+
/// @brief Controls access to `max_level_` and `entry_slot_`.
|
|
1921
|
+
/// If any thread is updating those values, no other threads can `add()` or `search()`.
|
|
1922
|
+
std::mutex global_mutex_{};
|
|
1923
|
+
|
|
1924
|
+
/// @brief The level of the top-most graph in the index. Grows as the logarithm of size, starts from zero.
|
|
1925
|
+
level_t max_level_{};
|
|
1926
|
+
|
|
1927
|
+
/// @brief The slot in which the only node of the top-level graph is stored.
|
|
1928
|
+
std::size_t entry_slot_{};
|
|
1929
|
+
|
|
1930
|
+
using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
|
|
1931
|
+
|
|
1932
|
+
/// @brief C-style array of `node_t` smart-pointers.
|
|
1933
|
+
buffer_gt<node_t, nodes_allocator_t> nodes_{};
|
|
1934
|
+
|
|
1935
|
+
/// @brief Mutex, that limits concurrent access to `nodes_`.
|
|
1936
|
+
mutable nodes_mutexes_t nodes_mutexes_{};
|
|
1937
|
+
|
|
1938
|
+
using contexts_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<context_t>;
|
|
1939
|
+
|
|
1940
|
+
/// @brief Array of thread-specific buffers for temporary data.
|
|
1941
|
+
mutable buffer_gt<context_t, contexts_allocator_t> contexts_{};
|
|
1942
|
+
|
|
1943
|
+
public:
|
|
1944
|
+
std::size_t connectivity() const noexcept { return config_.connectivity; }
|
|
1945
|
+
std::size_t capacity() const noexcept { return nodes_capacity_; }
|
|
1946
|
+
std::size_t size() const noexcept { return nodes_count_; }
|
|
1947
|
+
std::size_t max_level() const noexcept { return nodes_count_ ? static_cast<std::size_t>(max_level_) : 0; }
|
|
1948
|
+
index_config_t const& config() const noexcept { return config_; }
|
|
1949
|
+
index_limits_t const& limits() const noexcept { return limits_; }
|
|
1950
|
+
bool is_immutable() const noexcept { return bool(viewed_file_); }
|
|
1951
|
+
|
|
1952
|
+
/**
|
|
1953
|
+
* @section Exceptions
|
|
1954
|
+
* Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
|
|
1955
|
+
*/
|
|
1956
|
+
explicit index_gt( //
|
|
1957
|
+
index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
|
|
1958
|
+
tape_allocator_t tape_allocator = {}) noexcept
|
|
1959
|
+
: config_(config), limits_(0, 0), dynamic_allocator_(std::move(dynamic_allocator)),
|
|
1960
|
+
tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), nodes_count_(0u), max_level_(-1),
|
|
1961
|
+
entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
|
|
1962
|
+
|
|
1963
|
+
/**
|
|
1964
|
+
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
1965
|
+
*/
|
|
1966
|
+
index_gt fork() noexcept { return index_gt{config_, dynamic_allocator_, tape_allocator_}; }
|
|
1967
|
+
|
|
1968
|
+
~index_gt() noexcept { reset(); }
|
|
1969
|
+
|
|
1970
|
+
index_gt(index_gt&& other) noexcept { swap(other); }
|
|
1971
|
+
|
|
1972
|
+
index_gt& operator=(index_gt&& other) noexcept {
|
|
1973
|
+
swap(other);
|
|
1974
|
+
return *this;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
struct copy_result_t {
|
|
1978
|
+
error_t error;
|
|
1979
|
+
index_gt index;
|
|
1980
|
+
|
|
1981
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1982
|
+
copy_result_t failed(error_t message) noexcept {
|
|
1983
|
+
error = std::move(message);
|
|
1984
|
+
return std::move(*this);
|
|
1985
|
+
}
|
|
1986
|
+
};
|
|
1987
|
+
|
|
1988
|
+
copy_result_t copy(index_copy_config_t config = {}) const noexcept {
|
|
1989
|
+
copy_result_t result;
|
|
1990
|
+
index_gt& other = result.index;
|
|
1991
|
+
other = index_gt(config_, dynamic_allocator_, tape_allocator_);
|
|
1992
|
+
if (!other.reserve(limits_))
|
|
1993
|
+
return result.failed("Failed to reserve the contexts");
|
|
1994
|
+
|
|
1995
|
+
// Now all is left - is to allocate new `node_t` instances and populate
|
|
1996
|
+
// the `other.nodes_` array into it.
|
|
1997
|
+
for (std::size_t i = 0; i != nodes_count_; ++i)
|
|
1998
|
+
other.nodes_[i] = other.node_make_copy_(node_bytes_(nodes_[i]));
|
|
1999
|
+
|
|
2000
|
+
other.nodes_count_ = nodes_count_.load();
|
|
2001
|
+
other.max_level_ = max_level_;
|
|
2002
|
+
other.entry_slot_ = entry_slot_;
|
|
2003
|
+
|
|
2004
|
+
// This controls nothing for now :)
|
|
2005
|
+
(void)config;
|
|
2006
|
+
return result;
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
member_citerator_t cbegin() const noexcept { return {this, 0}; }
|
|
2010
|
+
member_citerator_t cend() const noexcept { return {this, size()}; }
|
|
2011
|
+
member_citerator_t begin() const noexcept { return {this, 0}; }
|
|
2012
|
+
member_citerator_t end() const noexcept { return {this, size()}; }
|
|
2013
|
+
member_iterator_t begin() noexcept { return {this, 0}; }
|
|
2014
|
+
member_iterator_t end() noexcept { return {this, size()}; }
|
|
2015
|
+
|
|
2016
|
+
member_ref_t at(std::size_t slot) noexcept { return {nodes_[slot].key(), slot}; }
|
|
2017
|
+
member_cref_t at(std::size_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
|
|
2018
|
+
member_iterator_t iterator_at(std::size_t slot) noexcept { return {this, slot}; }
|
|
2019
|
+
member_citerator_t citerator_at(std::size_t slot) const noexcept { return {this, slot}; }
|
|
2020
|
+
|
|
2021
|
+
dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
|
|
2022
|
+
tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
|
|
2023
|
+
|
|
2024
|
+
#pragma region Adjusting Configuration
|
|
2025
|
+
|
|
2026
|
+
/**
|
|
2027
|
+
* @brief Erases all the vectors from the index.
|
|
2028
|
+
*
|
|
2029
|
+
* Will change `size()` to zero, but will keep the same `capacity()`.
|
|
2030
|
+
* Will keep the number of available threads/contexts the same as it was.
|
|
2031
|
+
*/
|
|
2032
|
+
void clear() noexcept {
|
|
2033
|
+
if (!has_reset<tape_allocator_t>()) {
|
|
2034
|
+
std::size_t n = nodes_count_;
|
|
2035
|
+
for (std::size_t i = 0; i != n; ++i)
|
|
2036
|
+
node_free_(i);
|
|
2037
|
+
} else
|
|
2038
|
+
tape_allocator_.deallocate(nullptr, 0);
|
|
2039
|
+
nodes_count_ = 0;
|
|
2040
|
+
max_level_ = -1;
|
|
2041
|
+
entry_slot_ = 0u;
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
/**
|
|
2045
|
+
* @brief Erases all members from index, closing files, and returning RAM to OS.
|
|
2046
|
+
*
|
|
2047
|
+
* Will change both `size()` and `capacity()` to zero.
|
|
2048
|
+
* Will deallocate all threads/contexts.
|
|
2049
|
+
* If the index is memory-mapped - releases the mapping and the descriptor.
|
|
2050
|
+
*/
|
|
2051
|
+
void reset() noexcept {
|
|
2052
|
+
clear();
|
|
2053
|
+
|
|
2054
|
+
nodes_ = {};
|
|
2055
|
+
contexts_ = {};
|
|
2056
|
+
nodes_mutexes_ = {};
|
|
2057
|
+
limits_ = index_limits_t{0, 0};
|
|
2058
|
+
nodes_capacity_ = 0;
|
|
2059
|
+
viewed_file_ = memory_mapped_file_t{};
|
|
2060
|
+
tape_allocator_ = {};
|
|
2061
|
+
}
|
|
2062
|
+
|
|
2063
|
+
/**
|
|
2064
|
+
* @brief Swaps the underlying memory buffers and thread contexts.
|
|
2065
|
+
*/
|
|
2066
|
+
void swap(index_gt& other) noexcept {
|
|
2067
|
+
std::swap(config_, other.config_);
|
|
2068
|
+
std::swap(limits_, other.limits_);
|
|
2069
|
+
std::swap(dynamic_allocator_, other.dynamic_allocator_);
|
|
2070
|
+
std::swap(tape_allocator_, other.tape_allocator_);
|
|
2071
|
+
std::swap(pre_, other.pre_);
|
|
2072
|
+
std::swap(viewed_file_, other.viewed_file_);
|
|
2073
|
+
std::swap(max_level_, other.max_level_);
|
|
2074
|
+
std::swap(entry_slot_, other.entry_slot_);
|
|
2075
|
+
std::swap(nodes_, other.nodes_);
|
|
2076
|
+
std::swap(nodes_mutexes_, other.nodes_mutexes_);
|
|
2077
|
+
std::swap(contexts_, other.contexts_);
|
|
2078
|
+
|
|
2079
|
+
// Non-atomic parts.
|
|
2080
|
+
std::size_t capacity_copy = nodes_capacity_;
|
|
2081
|
+
std::size_t count_copy = nodes_count_;
|
|
2082
|
+
nodes_capacity_ = other.nodes_capacity_.load();
|
|
2083
|
+
nodes_count_ = other.nodes_count_.load();
|
|
2084
|
+
other.nodes_capacity_ = capacity_copy;
|
|
2085
|
+
other.nodes_count_ = count_copy;
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
/**
|
|
2089
|
+
* @brief Increases the `capacity()` of the index to allow adding more vectors.
|
|
2090
|
+
* @return `true` on success, `false` on memory allocation errors.
|
|
2091
|
+
*/
|
|
2092
|
+
bool reserve(index_limits_t limits) usearch_noexcept_m {
|
|
2093
|
+
|
|
2094
|
+
if (limits.threads_add <= limits_.threads_add //
|
|
2095
|
+
&& limits.threads_search <= limits_.threads_search //
|
|
2096
|
+
&& limits.members <= limits_.members)
|
|
2097
|
+
return true;
|
|
2098
|
+
|
|
2099
|
+
nodes_mutexes_t new_mutexes(limits.members);
|
|
2100
|
+
buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
|
|
2101
|
+
buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
|
|
2102
|
+
if (!new_nodes || !new_contexts || !new_mutexes)
|
|
2103
|
+
return false;
|
|
2104
|
+
|
|
2105
|
+
// Move the nodes info, and deallocate previous buffers.
|
|
2106
|
+
if (nodes_)
|
|
2107
|
+
std::memcpy(new_nodes.data(), nodes_.data(), sizeof(node_t) * size());
|
|
2108
|
+
|
|
2109
|
+
limits_ = limits;
|
|
2110
|
+
nodes_capacity_ = limits.members;
|
|
2111
|
+
nodes_ = std::move(new_nodes);
|
|
2112
|
+
contexts_ = std::move(new_contexts);
|
|
2113
|
+
nodes_mutexes_ = std::move(new_mutexes);
|
|
2114
|
+
return true;
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
#pragma endregion
|
|
2118
|
+
|
|
2119
|
+
#pragma region Construction and Search
|
|
2120
|
+
|
|
2121
|
+
struct add_result_t {
|
|
2122
|
+
error_t error{};
|
|
2123
|
+
std::size_t new_size{};
|
|
2124
|
+
std::size_t visited_members{};
|
|
2125
|
+
std::size_t computed_distances{};
|
|
2126
|
+
std::size_t slot{};
|
|
2127
|
+
|
|
2128
|
+
explicit operator bool() const noexcept { return !error; }
|
|
2129
|
+
add_result_t failed(error_t message) noexcept {
|
|
2130
|
+
error = std::move(message);
|
|
2131
|
+
return std::move(*this);
|
|
2132
|
+
}
|
|
2133
|
+
};
|
|
2134
|
+
|
|
2135
|
+
/// @brief Describes a matched search result, augmenting `member_cref_t`
|
|
2136
|
+
/// contents with `distance` to the query object.
|
|
2137
|
+
struct match_t {
|
|
2138
|
+
member_cref_t member;
|
|
2139
|
+
distance_t distance;
|
|
2140
|
+
|
|
2141
|
+
inline match_t() noexcept : member({nullptr, 0}), distance(std::numeric_limits<distance_t>::max()) {}
|
|
2142
|
+
|
|
2143
|
+
inline match_t(member_cref_t member, distance_t distance) noexcept : member(member), distance(distance) {}
|
|
2144
|
+
|
|
2145
|
+
inline match_t(match_t&& other) noexcept
|
|
2146
|
+
: member({other.member.key.ptr(), other.member.slot}), distance(other.distance) {}
|
|
2147
|
+
|
|
2148
|
+
inline match_t(match_t const& other) noexcept
|
|
2149
|
+
: member({other.member.key.ptr(), other.member.slot}), distance(other.distance) {}
|
|
2150
|
+
|
|
2151
|
+
inline match_t& operator=(match_t const& other) noexcept {
|
|
2152
|
+
member.key.reset(other.member.key.ptr());
|
|
2153
|
+
member.slot = other.member.slot;
|
|
2154
|
+
distance = other.distance;
|
|
2155
|
+
return *this;
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
inline match_t& operator=(match_t&& other) noexcept {
|
|
2159
|
+
member.key.reset(other.member.key.ptr());
|
|
2160
|
+
member.slot = other.member.slot;
|
|
2161
|
+
distance = other.distance;
|
|
2162
|
+
return *this;
|
|
2163
|
+
}
|
|
2164
|
+
};
|
|
2165
|
+
|
|
2166
|
+
class search_result_t {
|
|
2167
|
+
node_t const* nodes_{};
|
|
2168
|
+
top_candidates_t const* top_{};
|
|
2169
|
+
|
|
2170
|
+
friend class index_gt;
|
|
2171
|
+
inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
|
|
2172
|
+
: nodes_(index.nodes_), top_(&top) {}
|
|
2173
|
+
|
|
2174
|
+
public:
|
|
2175
|
+
/** @brief Number of search results found. */
|
|
2176
|
+
std::size_t count{};
|
|
2177
|
+
/** @brief Number of graph nodes traversed. */
|
|
2178
|
+
std::size_t visited_members{};
|
|
2179
|
+
/** @brief Number of times the distances were computed. */
|
|
2180
|
+
std::size_t computed_distances{};
|
|
2181
|
+
error_t error{};
|
|
2182
|
+
|
|
2183
|
+
inline search_result_t() noexcept {}
|
|
2184
|
+
inline search_result_t(search_result_t&&) = default;
|
|
2185
|
+
inline search_result_t& operator=(search_result_t&&) = default;
|
|
2186
|
+
|
|
2187
|
+
explicit operator bool() const noexcept { return !error; }
|
|
2188
|
+
search_result_t failed(error_t message) noexcept {
|
|
2189
|
+
error = std::move(message);
|
|
2190
|
+
return std::move(*this);
|
|
2191
|
+
}
|
|
2192
|
+
|
|
2193
|
+
inline operator std::size_t() const noexcept { return count; }
|
|
2194
|
+
inline std::size_t size() const noexcept { return count; }
|
|
2195
|
+
inline bool empty() const noexcept { return !count; }
|
|
2196
|
+
inline match_t operator[](std::size_t i) const noexcept { return at(i); }
|
|
2197
|
+
inline match_t front() const noexcept { return at(0); }
|
|
2198
|
+
inline match_t back() const noexcept { return at(count - 1); }
|
|
2199
|
+
inline bool contains(vector_key_t key) const noexcept {
|
|
2200
|
+
for (std::size_t i = 0; i != count; ++i)
|
|
2201
|
+
if (at(i).member.key == key)
|
|
2202
|
+
return true;
|
|
2203
|
+
return false;
|
|
2204
|
+
}
|
|
2205
|
+
inline match_t at(std::size_t i) const noexcept {
|
|
2206
|
+
candidate_t const* top_ordered = top_->data();
|
|
2207
|
+
candidate_t candidate = top_ordered[i];
|
|
2208
|
+
node_t node = nodes_[candidate.slot];
|
|
2209
|
+
return {member_cref_t{node.ckey(), candidate.slot}, candidate.distance};
|
|
2210
|
+
}
|
|
2211
|
+
inline std::size_t merge_into( //
|
|
2212
|
+
vector_key_t* keys, distance_t* distances, //
|
|
2213
|
+
std::size_t old_count, std::size_t max_count) const noexcept {
|
|
2214
|
+
|
|
2215
|
+
std::size_t merged_count = old_count;
|
|
2216
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
2217
|
+
match_t result = operator[](i);
|
|
2218
|
+
distance_t* merged_end = distances + merged_count;
|
|
2219
|
+
std::size_t offset = std::lower_bound(distances, merged_end, result.distance) - distances;
|
|
2220
|
+
if (offset == max_count)
|
|
2221
|
+
continue;
|
|
2222
|
+
|
|
2223
|
+
std::size_t count_worse = merged_count - offset - (max_count == merged_count);
|
|
2224
|
+
std::memmove(keys + offset + 1, keys + offset, count_worse * sizeof(vector_key_t));
|
|
2225
|
+
std::memmove(distances + offset + 1, distances + offset, count_worse * sizeof(distance_t));
|
|
2226
|
+
keys[offset] = result.member.key;
|
|
2227
|
+
distances[offset] = result.distance;
|
|
2228
|
+
merged_count += merged_count != max_count;
|
|
2229
|
+
}
|
|
2230
|
+
return merged_count;
|
|
2231
|
+
}
|
|
2232
|
+
inline std::size_t dump_to(vector_key_t* keys, distance_t* distances) const noexcept {
|
|
2233
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
2234
|
+
match_t result = operator[](i);
|
|
2235
|
+
keys[i] = result.member.key;
|
|
2236
|
+
distances[i] = result.distance;
|
|
2237
|
+
}
|
|
2238
|
+
return count;
|
|
2239
|
+
}
|
|
2240
|
+
inline std::size_t dump_to(vector_key_t* keys) const noexcept {
|
|
2241
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
2242
|
+
match_t result = operator[](i);
|
|
2243
|
+
keys[i] = result.member.key;
|
|
2244
|
+
}
|
|
2245
|
+
return count;
|
|
2246
|
+
}
|
|
2247
|
+
};
|
|
2248
|
+
|
|
2249
|
+
struct cluster_result_t {
|
|
2250
|
+
error_t error{};
|
|
2251
|
+
std::size_t visited_members{};
|
|
2252
|
+
std::size_t computed_distances{};
|
|
2253
|
+
match_t cluster{};
|
|
2254
|
+
|
|
2255
|
+
explicit operator bool() const noexcept { return !error; }
|
|
2256
|
+
cluster_result_t failed(error_t message) noexcept {
|
|
2257
|
+
error = std::move(message);
|
|
2258
|
+
return std::move(*this);
|
|
2259
|
+
}
|
|
2260
|
+
};
|
|
2261
|
+
|
|
2262
|
+
/**
|
|
2263
|
+
* @brief Inserts a new entry into the index. Thread-safe. Supports @b heterogeneous lookups.
|
|
2264
|
+
* Expects needed capacity to be reserved ahead of time: `size() < capacity()`.
|
|
2265
|
+
*
|
|
2266
|
+
* @tparam metric_at
|
|
2267
|
+
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2268
|
+
* It should be callable into distinctly different scenarios:
|
|
2269
|
+
* - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
|
|
2270
|
+
* - `distance_t operator() (entry_at, entry_at)` - between existing entries.
|
|
2271
|
+
* Where any possible `entry_at` has both two interfaces: `std::size_t slot()`, `vector_key_t key()`.
|
|
2272
|
+
*
|
|
2273
|
+
* @param[in] key External identifier/name/descriptor for the new entry.
|
|
2274
|
+
* @param[in] value Content that will be compared against other entries to index.
|
|
2275
|
+
* @param[in] metric Callable object measuring distance between ::value and present objects.
|
|
2276
|
+
* @param[in] config Configuration options for this specific operation.
|
|
2277
|
+
* @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
|
|
2278
|
+
*/
|
|
2279
|
+
template < //
|
|
2280
|
+
typename value_at, //
|
|
2281
|
+
typename metric_at, //
|
|
2282
|
+
typename callback_at = dummy_callback_t, //
|
|
2283
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2284
|
+
>
|
|
2285
|
+
add_result_t add( //
|
|
2286
|
+
vector_key_t key, value_at&& value, metric_at&& metric, //
|
|
2287
|
+
index_update_config_t config = {}, //
|
|
2288
|
+
callback_at&& callback = callback_at{}, //
|
|
2289
|
+
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2290
|
+
|
|
2291
|
+
add_result_t result;
|
|
2292
|
+
if (is_immutable())
|
|
2293
|
+
return result.failed("Can't add to an immutable index");
|
|
2294
|
+
|
|
2295
|
+
// Make sure we have enough local memory to perform this request
|
|
2296
|
+
context_t& context = contexts_[config.thread];
|
|
2297
|
+
top_candidates_t& top = context.top_candidates;
|
|
2298
|
+
next_candidates_t& next = context.next_candidates;
|
|
2299
|
+
top.clear();
|
|
2300
|
+
next.clear();
|
|
2301
|
+
|
|
2302
|
+
// The top list needs one more slot than the connectivity of the base level
|
|
2303
|
+
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
2304
|
+
std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
|
|
2305
|
+
std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
|
|
2306
|
+
if (!top.reserve(top_limit))
|
|
2307
|
+
return result.failed("Out of memory!");
|
|
2308
|
+
if (!next.reserve(config.expansion))
|
|
2309
|
+
return result.failed("Out of memory!");
|
|
2310
|
+
|
|
2311
|
+
// Determining how much memory to allocate for the node depends on the target level
|
|
2312
|
+
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
2313
|
+
level_t max_level_copy = max_level_; // Copy under lock
|
|
2314
|
+
std::size_t entry_idx_copy = entry_slot_; // Copy under lock
|
|
2315
|
+
level_t target_level = choose_random_level_(context.level_generator);
|
|
2316
|
+
|
|
2317
|
+
// Make sure we are not overflowing
|
|
2318
|
+
std::size_t capacity = nodes_capacity_.load();
|
|
2319
|
+
std::size_t new_slot = nodes_count_.fetch_add(1);
|
|
2320
|
+
if (new_slot >= capacity) {
|
|
2321
|
+
nodes_count_.fetch_sub(1);
|
|
2322
|
+
return result.failed("Reserve capacity ahead of insertions!");
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
// Allocate the neighbors
|
|
2326
|
+
node_t node = node_make_(key, target_level);
|
|
2327
|
+
if (!node) {
|
|
2328
|
+
nodes_count_.fetch_sub(1);
|
|
2329
|
+
return result.failed("Out of memory!");
|
|
2330
|
+
}
|
|
2331
|
+
if (target_level <= max_level_copy)
|
|
2332
|
+
new_level_lock.unlock();
|
|
2333
|
+
|
|
2334
|
+
nodes_[new_slot] = node;
|
|
2335
|
+
result.new_size = new_slot + 1;
|
|
2336
|
+
result.slot = new_slot;
|
|
2337
|
+
callback(at(new_slot));
|
|
2338
|
+
node_lock_t new_lock = node_lock_(new_slot);
|
|
2339
|
+
|
|
2340
|
+
// Do nothing for the first element
|
|
2341
|
+
if (!new_slot) {
|
|
2342
|
+
entry_slot_ = new_slot;
|
|
2343
|
+
max_level_ = target_level;
|
|
2344
|
+
return result;
|
|
2345
|
+
}
|
|
2346
|
+
|
|
2347
|
+
// Pull stats
|
|
2348
|
+
result.computed_distances = context.computed_distances_count;
|
|
2349
|
+
result.visited_members = context.iteration_cycles;
|
|
2350
|
+
|
|
2351
|
+
connect_node_across_levels_( //
|
|
2352
|
+
value, metric, prefetch, //
|
|
2353
|
+
new_slot, entry_idx_copy, max_level_copy, target_level, //
|
|
2354
|
+
config, context);
|
|
2355
|
+
|
|
2356
|
+
// Normalize stats
|
|
2357
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2358
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2359
|
+
|
|
2360
|
+
// Updating the entry point if needed
|
|
2361
|
+
if (target_level > max_level_copy) {
|
|
2362
|
+
entry_slot_ = new_slot;
|
|
2363
|
+
max_level_ = target_level;
|
|
2364
|
+
}
|
|
2365
|
+
return result;
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
/**
|
|
2369
|
+
* @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
|
|
2370
|
+
*
|
|
2371
|
+
* @tparam metric_at
|
|
2372
|
+
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2373
|
+
* It should be callable into distinctly different scenarios:
|
|
2374
|
+
* - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
|
|
2375
|
+
* - `distance_t operator() (entry_at, entry_at)` - between existing entries.
|
|
2376
|
+
* For any possible `entry_at` following interfaces will work:
|
|
2377
|
+
* - `std::size_t get_slot(entry_at const &)`
|
|
2378
|
+
* - `vector_key_t get_key(entry_at const &)`
|
|
2379
|
+
*
|
|
2380
|
+
* @param[in] iterator Iterator pointing to an existing entry to be replaced.
|
|
2381
|
+
* @param[in] key External identifier/name/descriptor for the entry.
|
|
2382
|
+
* @param[in] value Content that will be compared against other entries in the index.
|
|
2383
|
+
* @param[in] metric Callable object measuring distance between ::value and present objects.
|
|
2384
|
+
* @param[in] config Configuration options for this specific operation.
|
|
2385
|
+
* @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
|
|
2386
|
+
*/
|
|
2387
|
+
template < //
|
|
2388
|
+
typename value_at, //
|
|
2389
|
+
typename metric_at, //
|
|
2390
|
+
typename callback_at = dummy_callback_t, //
|
|
2391
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2392
|
+
>
|
|
2393
|
+
add_result_t update( //
|
|
2394
|
+
member_iterator_t iterator, //
|
|
2395
|
+
vector_key_t key, //
|
|
2396
|
+
value_at&& value, //
|
|
2397
|
+
metric_at&& metric, //
|
|
2398
|
+
index_update_config_t config = {}, //
|
|
2399
|
+
callback_at&& callback = callback_at{}, //
|
|
2400
|
+
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2401
|
+
|
|
2402
|
+
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
2403
|
+
add_result_t result;
|
|
2404
|
+
std::size_t old_slot = iterator.slot_;
|
|
2405
|
+
|
|
2406
|
+
// Make sure we have enough local memory to perform this request
|
|
2407
|
+
context_t& context = contexts_[config.thread];
|
|
2408
|
+
top_candidates_t& top = context.top_candidates;
|
|
2409
|
+
next_candidates_t& next = context.next_candidates;
|
|
2410
|
+
top.clear();
|
|
2411
|
+
next.clear();
|
|
2412
|
+
|
|
2413
|
+
// The top list needs one more slot than the connectivity of the base level
|
|
2414
|
+
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
2415
|
+
std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
|
|
2416
|
+
std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
|
|
2417
|
+
if (!top.reserve(top_limit))
|
|
2418
|
+
return result.failed("Out of memory!");
|
|
2419
|
+
if (!next.reserve(config.expansion))
|
|
2420
|
+
return result.failed("Out of memory!");
|
|
2421
|
+
|
|
2422
|
+
node_lock_t new_lock = node_lock_(old_slot);
|
|
2423
|
+
node_t node = node_at_(old_slot);
|
|
2424
|
+
|
|
2425
|
+
level_t node_level = node.level();
|
|
2426
|
+
span_bytes_t node_bytes = node_bytes_(node);
|
|
2427
|
+
std::memset(node_bytes.data(), 0, node_bytes.size());
|
|
2428
|
+
node.level(node_level);
|
|
2429
|
+
|
|
2430
|
+
// Pull stats
|
|
2431
|
+
result.computed_distances = context.computed_distances_count;
|
|
2432
|
+
result.visited_members = context.iteration_cycles;
|
|
2433
|
+
|
|
2434
|
+
connect_node_across_levels_( //
|
|
2435
|
+
value, metric, prefetch, //
|
|
2436
|
+
old_slot, entry_slot_, max_level_, node_level, //
|
|
2437
|
+
config, context);
|
|
2438
|
+
node.key(key);
|
|
2439
|
+
|
|
2440
|
+
// Normalize stats
|
|
2441
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2442
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2443
|
+
result.slot = old_slot;
|
|
2444
|
+
|
|
2445
|
+
callback(at(old_slot));
|
|
2446
|
+
return result;
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
/**
|
|
2450
|
+
* @brief Searches for the closest elements to the given ::query. Thread-safe.
|
|
2451
|
+
*
|
|
2452
|
+
* @param[in] query Content that will be compared against other entries in the index.
|
|
2453
|
+
* @param[in] wanted The upper bound for the number of results to return.
|
|
2454
|
+
* @param[in] config Configuration options for this specific operation.
|
|
2455
|
+
* @param[in] predicate Optional filtering predicate for `member_cref_t`.
|
|
2456
|
+
* @return Smart object referencing temporary memory. Valid until next `search()`, `add()`, or `cluster()`.
|
|
2457
|
+
*/
|
|
2458
|
+
template < //
|
|
2459
|
+
typename value_at, //
|
|
2460
|
+
typename metric_at, //
|
|
2461
|
+
typename predicate_at = dummy_predicate_t, //
|
|
2462
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2463
|
+
>
|
|
2464
|
+
search_result_t search( //
|
|
2465
|
+
value_at&& query, //
|
|
2466
|
+
std::size_t wanted, //
|
|
2467
|
+
metric_at&& metric, //
|
|
2468
|
+
index_search_config_t config = {}, //
|
|
2469
|
+
predicate_at&& predicate = predicate_at{}, //
|
|
2470
|
+
prefetch_at&& prefetch = prefetch_at{}) const noexcept {
|
|
2471
|
+
|
|
2472
|
+
context_t& context = contexts_[config.thread];
|
|
2473
|
+
top_candidates_t& top = context.top_candidates;
|
|
2474
|
+
search_result_t result{*this, top};
|
|
2475
|
+
if (!nodes_count_)
|
|
2476
|
+
return result;
|
|
2477
|
+
|
|
2478
|
+
// Go down the level, tracking only the closest match
|
|
2479
|
+
result.computed_distances = context.computed_distances_count;
|
|
2480
|
+
result.visited_members = context.iteration_cycles;
|
|
2481
|
+
|
|
2482
|
+
if (config.exact) {
|
|
2483
|
+
if (!top.reserve(wanted))
|
|
2484
|
+
return result.failed("Out of memory!");
|
|
2485
|
+
search_exact_(query, metric, predicate, wanted, context);
|
|
2486
|
+
} else {
|
|
2487
|
+
next_candidates_t& next = context.next_candidates;
|
|
2488
|
+
std::size_t expansion = (std::max)(config.expansion, wanted);
|
|
2489
|
+
if (!next.reserve(expansion))
|
|
2490
|
+
return result.failed("Out of memory!");
|
|
2491
|
+
if (!top.reserve(expansion))
|
|
2492
|
+
return result.failed("Out of memory!");
|
|
2493
|
+
|
|
2494
|
+
std::size_t closest_slot = search_for_one_(query, metric, prefetch, entry_slot_, max_level_, 0, context);
|
|
2495
|
+
|
|
2496
|
+
// For bottom layer we need a more optimized procedure
|
|
2497
|
+
if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
|
|
2498
|
+
return result.failed("Out of memory!");
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
top.sort_ascending();
|
|
2502
|
+
top.shrink(wanted);
|
|
2503
|
+
|
|
2504
|
+
// Normalize stats
|
|
2505
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2506
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2507
|
+
result.count = top.size();
|
|
2508
|
+
return result;
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
/**
|
|
2512
|
+
* @brief Identifies the closest cluster to the given ::query. Thread-safe.
|
|
2513
|
+
*
|
|
2514
|
+
* @param[in] query Content that will be compared against other entries in the index.
|
|
2515
|
+
* @param[in] level The index level to target. Higher means lower resolution.
|
|
2516
|
+
* @param[in] config Configuration options for this specific operation.
|
|
2517
|
+
* @param[in] predicate Optional filtering predicate for `member_cref_t`.
|
|
2518
|
+
* @return Smart object referencing temporary memory. Valid until next `search()`, `add()`, or `cluster()`.
|
|
2519
|
+
*/
|
|
2520
|
+
template < //
|
|
2521
|
+
typename value_at, //
|
|
2522
|
+
typename metric_at, //
|
|
2523
|
+
typename predicate_at = dummy_predicate_t, //
|
|
2524
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2525
|
+
>
|
|
2526
|
+
cluster_result_t cluster( //
|
|
2527
|
+
value_at&& query, //
|
|
2528
|
+
std::size_t level, //
|
|
2529
|
+
metric_at&& metric, //
|
|
2530
|
+
index_cluster_config_t config = {}, //
|
|
2531
|
+
predicate_at&& predicate = predicate_at{}, //
|
|
2532
|
+
prefetch_at&& prefetch = prefetch_at{}) const noexcept {
|
|
2533
|
+
|
|
2534
|
+
context_t& context = contexts_[config.thread];
|
|
2535
|
+
cluster_result_t result;
|
|
2536
|
+
if (!nodes_count_)
|
|
2537
|
+
return result.failed("No clusters to identify");
|
|
2538
|
+
|
|
2539
|
+
// Go down the level, tracking only the closest match
|
|
2540
|
+
result.computed_distances = context.computed_distances_count;
|
|
2541
|
+
result.visited_members = context.iteration_cycles;
|
|
2542
|
+
|
|
2543
|
+
next_candidates_t& next = context.next_candidates;
|
|
2544
|
+
std::size_t expansion = config.expansion;
|
|
2545
|
+
if (!next.reserve(expansion))
|
|
2546
|
+
return result.failed("Out of memory!");
|
|
2547
|
+
|
|
2548
|
+
result.cluster.member = at(search_for_one_(query, metric, prefetch, entry_slot_, max_level_,
|
|
2549
|
+
static_cast<level_t>(level - 1), context));
|
|
2550
|
+
result.cluster.distance = context.measure(query, result.cluster.member, metric);
|
|
2551
|
+
|
|
2552
|
+
// Normalize stats
|
|
2553
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2554
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2555
|
+
|
|
2556
|
+
(void)predicate;
|
|
2557
|
+
return result;
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
#pragma endregion
|
|
2561
|
+
|
|
2562
|
+
#pragma region Metadata
|
|
2563
|
+
|
|
2564
|
+
struct stats_t {
|
|
2565
|
+
std::size_t nodes{};
|
|
2566
|
+
std::size_t edges{};
|
|
2567
|
+
std::size_t max_edges{};
|
|
2568
|
+
std::size_t allocated_bytes{};
|
|
2569
|
+
};
|
|
2570
|
+
|
|
2571
|
+
stats_t stats() const noexcept {
|
|
2572
|
+
stats_t result{};
|
|
2573
|
+
|
|
2574
|
+
for (std::size_t i = 0; i != size(); ++i) {
|
|
2575
|
+
node_t node = node_at_(i);
|
|
2576
|
+
std::size_t max_edges = node.level() * config_.connectivity + config_.connectivity_base;
|
|
2577
|
+
std::size_t edges = 0;
|
|
2578
|
+
for (level_t level = 0; level <= node.level(); ++level)
|
|
2579
|
+
edges += neighbors_(node, level).size();
|
|
2580
|
+
|
|
2581
|
+
++result.nodes;
|
|
2582
|
+
result.allocated_bytes += node_bytes_(node).size();
|
|
2583
|
+
result.edges += edges;
|
|
2584
|
+
result.max_edges += max_edges;
|
|
2585
|
+
}
|
|
2586
|
+
return result;
|
|
2587
|
+
}
|
|
2588
|
+
|
|
2589
|
+
stats_t stats(std::size_t level) const noexcept {
|
|
2590
|
+
stats_t result{};
|
|
2591
|
+
|
|
2592
|
+
std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
|
|
2593
|
+
for (std::size_t i = 0; i != size(); ++i) {
|
|
2594
|
+
node_t node = node_at_(i);
|
|
2595
|
+
if (static_cast<std::size_t>(node.level()) < level)
|
|
2596
|
+
continue;
|
|
2597
|
+
|
|
2598
|
+
++result.nodes;
|
|
2599
|
+
result.edges += neighbors_(node, level).size();
|
|
2600
|
+
result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2603
|
+
std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
|
|
2604
|
+
result.max_edges = result.nodes * max_edges_per_node;
|
|
2605
|
+
return result;
|
|
2606
|
+
}
|
|
2607
|
+
|
|
2608
|
+
stats_t stats(stats_t* stats_per_level, std::size_t max_level) const noexcept {
|
|
2609
|
+
|
|
2610
|
+
std::size_t head_bytes = node_head_bytes_();
|
|
2611
|
+
for (std::size_t i = 0; i != size(); ++i) {
|
|
2612
|
+
node_t node = node_at_(i);
|
|
2613
|
+
|
|
2614
|
+
stats_per_level[0].nodes++;
|
|
2615
|
+
stats_per_level[0].edges += neighbors_(node, 0).size();
|
|
2616
|
+
stats_per_level[0].allocated_bytes += pre_.neighbors_base_bytes + head_bytes;
|
|
2617
|
+
|
|
2618
|
+
level_t node_level = static_cast<level_t>(node.level());
|
|
2619
|
+
for (level_t l = 1; l <= (std::min)(node_level, static_cast<level_t>(max_level)); ++l) {
|
|
2620
|
+
stats_per_level[l].nodes++;
|
|
2621
|
+
stats_per_level[l].edges += neighbors_(node, l).size();
|
|
2622
|
+
stats_per_level[l].allocated_bytes += pre_.neighbors_bytes;
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
|
|
2626
|
+
// The `max_edges` parameter can be inferred from `nodes`
|
|
2627
|
+
stats_per_level[0].max_edges = stats_per_level[0].nodes * config_.connectivity_base;
|
|
2628
|
+
for (std::size_t l = 1; l <= max_level; ++l)
|
|
2629
|
+
stats_per_level[l].max_edges = stats_per_level[l].nodes * config_.connectivity;
|
|
2630
|
+
|
|
2631
|
+
// Aggregate stats across levels
|
|
2632
|
+
stats_t result{};
|
|
2633
|
+
for (std::size_t l = 0; l <= max_level; ++l)
|
|
2634
|
+
result.nodes += stats_per_level[l].nodes, //
|
|
2635
|
+
result.edges += stats_per_level[l].edges, //
|
|
2636
|
+
result.allocated_bytes += stats_per_level[l].allocated_bytes, //
|
|
2637
|
+
result.max_edges += stats_per_level[l].max_edges; //
|
|
2638
|
+
|
|
2639
|
+
return result;
|
|
2640
|
+
}
|
|
2641
|
+
|
|
2642
|
+
/**
|
|
2643
|
+
* @brief A relatively accurate lower bound on the amount of memory consumed by the system.
|
|
2644
|
+
* In practice it's error will be below 10%.
|
|
2645
|
+
*
|
|
2646
|
+
* @see `serialized_length` for the length of the binary serialized representation.
|
|
2647
|
+
*/
|
|
2648
|
+
std::size_t memory_usage(std::size_t allocator_entry_bytes = default_allocator_entry_bytes()) const noexcept {
|
|
2649
|
+
std::size_t total = 0;
|
|
2650
|
+
if (!viewed_file_) {
|
|
2651
|
+
stats_t s = stats();
|
|
2652
|
+
total += s.allocated_bytes;
|
|
2653
|
+
total += s.nodes * allocator_entry_bytes;
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
// Temporary data-structures, proportional to the number of nodes:
|
|
2657
|
+
total += limits_.members * sizeof(node_t) + allocator_entry_bytes;
|
|
2658
|
+
|
|
2659
|
+
// Temporary data-structures, proportional to the number of threads:
|
|
2660
|
+
total += limits_.threads() * sizeof(context_t) + allocator_entry_bytes * 3;
|
|
2661
|
+
return total;
|
|
2662
|
+
}
|
|
2663
|
+
|
|
2664
|
+
std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
|
|
2665
|
+
|
|
2666
|
+
#pragma endregion
|
|
2667
|
+
|
|
2668
|
+
#pragma region Serialization
|
|
2669
|
+
|
|
2670
|
+
/**
|
|
2671
|
+
* @brief Estimate the binary length (in bytes) of the serialized index.
|
|
2672
|
+
*/
|
|
2673
|
+
std::size_t serialized_length() const noexcept {
|
|
2674
|
+
std::size_t neighbors_length = 0;
|
|
2675
|
+
for (std::size_t i = 0; i != size(); ++i)
|
|
2676
|
+
neighbors_length += node_bytes_(node_at_(i).level()) + sizeof(level_t);
|
|
2677
|
+
return sizeof(index_serialized_header_t) + neighbors_length;
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
/**
|
|
2681
|
+
* @brief Saves serialized binary index representation to a stream.
|
|
2682
|
+
*/
|
|
2683
|
+
template <typename output_callback_at, typename progress_at = dummy_progress_t>
|
|
2684
|
+
serialization_result_t save_to_stream(output_callback_at&& output, progress_at&& progress = {}) const noexcept {
|
|
2685
|
+
|
|
2686
|
+
serialization_result_t result;
|
|
2687
|
+
|
|
2688
|
+
// Export some basic metadata
|
|
2689
|
+
index_serialized_header_t header;
|
|
2690
|
+
header.size = nodes_count_;
|
|
2691
|
+
header.connectivity = config_.connectivity;
|
|
2692
|
+
header.connectivity_base = config_.connectivity_base;
|
|
2693
|
+
header.max_level = max_level_;
|
|
2694
|
+
header.entry_slot = entry_slot_;
|
|
2695
|
+
if (!output(&header, sizeof(header)))
|
|
2696
|
+
return result.failed("Failed to serialize the header into stream");
|
|
2697
|
+
|
|
2698
|
+
// Progress status
|
|
2699
|
+
std::size_t processed = 0;
|
|
2700
|
+
std::size_t const total = 2 * header.size;
|
|
2701
|
+
|
|
2702
|
+
// Export the number of levels per node
|
|
2703
|
+
// That is both enough to estimate the overall memory consumption,
|
|
2704
|
+
// and to be able to estimate the offsets of every entry in the file.
|
|
2705
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2706
|
+
node_t node = node_at_(i);
|
|
2707
|
+
level_t level = node.level();
|
|
2708
|
+
if (!output(&level, sizeof(level)))
|
|
2709
|
+
return result.failed("Failed to serialize into stream");
|
|
2710
|
+
if (!progress(++processed, total))
|
|
2711
|
+
return result.failed("Terminated by user");
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
// After that dump the nodes themselves
|
|
2715
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2716
|
+
span_bytes_t node_bytes = node_bytes_(node_at_(i));
|
|
2717
|
+
if (!output(node_bytes.data(), node_bytes.size()))
|
|
2718
|
+
return result.failed("Failed to serialize into stream");
|
|
2719
|
+
if (!progress(++processed, total))
|
|
2720
|
+
return result.failed("Terminated by user");
|
|
2721
|
+
}
|
|
2722
|
+
|
|
2723
|
+
return {};
|
|
2724
|
+
}
|
|
2725
|
+
|
|
2726
|
+
/**
|
|
2727
|
+
* @brief Symmetric to `save_from_stream`, pulls data from a stream.
|
|
2728
|
+
*/
|
|
2729
|
+
template <typename input_callback_at, typename progress_at = dummy_progress_t>
|
|
2730
|
+
serialization_result_t load_from_stream(input_callback_at&& input, progress_at&& progress = {}) noexcept {
|
|
2731
|
+
|
|
2732
|
+
serialization_result_t result;
|
|
2733
|
+
|
|
2734
|
+
// Remove previously stored objects
|
|
2735
|
+
reset();
|
|
2736
|
+
|
|
2737
|
+
// Pull basic metadata
|
|
2738
|
+
index_serialized_header_t header;
|
|
2739
|
+
if (!input(&header, sizeof(header)))
|
|
2740
|
+
return result.failed("Failed to pull the header from the stream");
|
|
2741
|
+
|
|
2742
|
+
// We are loading an empty index, no more work to do
|
|
2743
|
+
if (!header.size) {
|
|
2744
|
+
reset();
|
|
2745
|
+
return result;
|
|
2746
|
+
}
|
|
2747
|
+
|
|
2748
|
+
// Allocate some dynamic memory to read all the levels
|
|
2749
|
+
using levels_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<level_t>;
|
|
2750
|
+
buffer_gt<level_t, levels_allocator_t> levels(header.size);
|
|
2751
|
+
if (!levels)
|
|
2752
|
+
return result.failed("Out of memory");
|
|
2753
|
+
if (!input(levels, header.size * sizeof(level_t)))
|
|
2754
|
+
return result.failed("Failed to pull nodes levels from the stream");
|
|
2755
|
+
|
|
2756
|
+
// Submit metadata
|
|
2757
|
+
config_.connectivity = header.connectivity;
|
|
2758
|
+
config_.connectivity_base = header.connectivity_base;
|
|
2759
|
+
pre_ = precompute_(config_);
|
|
2760
|
+
index_limits_t limits;
|
|
2761
|
+
limits.members = header.size;
|
|
2762
|
+
if (!reserve(limits)) {
|
|
2763
|
+
reset();
|
|
2764
|
+
return result.failed("Out of memory");
|
|
2765
|
+
}
|
|
2766
|
+
nodes_count_ = header.size;
|
|
2767
|
+
max_level_ = static_cast<level_t>(header.max_level);
|
|
2768
|
+
entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
|
|
2769
|
+
|
|
2770
|
+
// Load the nodes
|
|
2771
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2772
|
+
span_bytes_t node_bytes = node_malloc_(levels[i]);
|
|
2773
|
+
if (!input(node_bytes.data(), node_bytes.size())) {
|
|
2774
|
+
reset();
|
|
2775
|
+
return result.failed("Failed to pull nodes from the stream");
|
|
2776
|
+
}
|
|
2777
|
+
nodes_[i] = node_t{node_bytes.data()};
|
|
2778
|
+
if (!progress(i + 1, header.size))
|
|
2779
|
+
return result.failed("Terminated by user");
|
|
2780
|
+
}
|
|
2781
|
+
return {};
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
template <typename progress_at = dummy_progress_t>
|
|
2785
|
+
serialization_result_t save(char const* file_path, progress_at&& progress = {}) const noexcept {
|
|
2786
|
+
return save(output_file_t(file_path), std::forward<progress_at>(progress));
|
|
2787
|
+
}
|
|
2788
|
+
|
|
2789
|
+
template <typename progress_at = dummy_progress_t>
|
|
2790
|
+
serialization_result_t load(char const* file_path, progress_at&& progress = {}) noexcept {
|
|
2791
|
+
return load(input_file_t(file_path), std::forward<progress_at>(progress));
|
|
2792
|
+
}
|
|
2793
|
+
|
|
2794
|
+
/**
|
|
2795
|
+
* @brief Saves serialized binary index representation to a file, generally on disk.
|
|
2796
|
+
*/
|
|
2797
|
+
template <typename progress_at = dummy_progress_t>
|
|
2798
|
+
serialization_result_t save(output_file_t file, progress_at&& progress = {}) const noexcept {
|
|
2799
|
+
|
|
2800
|
+
serialization_result_t io_result = file.open_if_not();
|
|
2801
|
+
if (!io_result)
|
|
2802
|
+
return io_result;
|
|
2803
|
+
|
|
2804
|
+
serialization_result_t stream_result = save_to_stream(
|
|
2805
|
+
[&](void* buffer, std::size_t length) {
|
|
2806
|
+
io_result = file.write(buffer, length);
|
|
2807
|
+
return !!io_result;
|
|
2808
|
+
},
|
|
2809
|
+
std::forward<progress_at>(progress));
|
|
2810
|
+
|
|
2811
|
+
if (!stream_result)
|
|
2812
|
+
return stream_result;
|
|
2813
|
+
return io_result;
|
|
2814
|
+
}
|
|
2815
|
+
|
|
2816
|
+
/**
|
|
2817
|
+
* @brief Memory-maps the serialized binary index representation from disk,
|
|
2818
|
+
* @b without copying data into RAM, and fetching it on-demand.
|
|
2819
|
+
*/
|
|
2820
|
+
template <typename progress_at = dummy_progress_t>
|
|
2821
|
+
serialization_result_t save(memory_mapped_file_t file, std::size_t offset = 0,
|
|
2822
|
+
progress_at&& progress = {}) const noexcept {
|
|
2823
|
+
|
|
2824
|
+
serialization_result_t io_result = file.open_if_not();
|
|
2825
|
+
if (!io_result)
|
|
2826
|
+
return io_result;
|
|
2827
|
+
|
|
2828
|
+
serialization_result_t stream_result = save_to_stream(
|
|
2829
|
+
[&](void* buffer, std::size_t length) {
|
|
2830
|
+
if (offset + length > file.size())
|
|
2831
|
+
return false;
|
|
2832
|
+
std::memcpy(file.data() + offset, buffer, length);
|
|
2833
|
+
offset += length;
|
|
2834
|
+
return true;
|
|
2835
|
+
},
|
|
2836
|
+
std::forward<progress_at>(progress));
|
|
2837
|
+
|
|
2838
|
+
return stream_result;
|
|
2839
|
+
}
|
|
2840
|
+
|
|
2841
|
+
/**
|
|
2842
|
+
* @brief Loads the serialized binary index representation from disk to RAM.
|
|
2843
|
+
* Adjusts the configuration properties of the constructed index to
|
|
2844
|
+
* match the settings in the file.
|
|
2845
|
+
*/
|
|
2846
|
+
template <typename progress_at = dummy_progress_t>
|
|
2847
|
+
serialization_result_t load(input_file_t file, progress_at&& progress = {}) noexcept {
|
|
2848
|
+
|
|
2849
|
+
serialization_result_t io_result = file.open_if_not();
|
|
2850
|
+
if (!io_result)
|
|
2851
|
+
return io_result;
|
|
2852
|
+
|
|
2853
|
+
serialization_result_t stream_result = load_from_stream(
|
|
2854
|
+
[&](void* buffer, std::size_t length) {
|
|
2855
|
+
io_result = file.read(buffer, length);
|
|
2856
|
+
return !!io_result;
|
|
2857
|
+
},
|
|
2858
|
+
std::forward<progress_at>(progress));
|
|
2859
|
+
|
|
2860
|
+
if (!stream_result)
|
|
2861
|
+
return stream_result;
|
|
2862
|
+
return io_result;
|
|
2863
|
+
}
|
|
2864
|
+
|
|
2865
|
+
/**
|
|
2866
|
+
* @brief Loads the serialized binary index representation from disk to RAM.
|
|
2867
|
+
* Adjusts the configuration properties of the constructed index to
|
|
2868
|
+
* match the settings in the file.
|
|
2869
|
+
*/
|
|
2870
|
+
template <typename progress_at = dummy_progress_t>
|
|
2871
|
+
serialization_result_t load(memory_mapped_file_t file, std::size_t offset = 0,
|
|
2872
|
+
progress_at&& progress = {}) noexcept {
|
|
2873
|
+
|
|
2874
|
+
serialization_result_t io_result = file.open_if_not();
|
|
2875
|
+
if (!io_result)
|
|
2876
|
+
return io_result;
|
|
2877
|
+
|
|
2878
|
+
serialization_result_t stream_result = load_from_stream(
|
|
2879
|
+
[&](void* buffer, std::size_t length) {
|
|
2880
|
+
if (offset + length > file.size())
|
|
2881
|
+
return false;
|
|
2882
|
+
std::memcpy(buffer, file.data() + offset, length);
|
|
2883
|
+
offset += length;
|
|
2884
|
+
return true;
|
|
2885
|
+
},
|
|
2886
|
+
std::forward<progress_at>(progress));
|
|
2887
|
+
|
|
2888
|
+
return stream_result;
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
/**
|
|
2892
|
+
* @brief Memory-maps the serialized binary index representation from disk,
|
|
2893
|
+
* @b without copying data into RAM, and fetching it on-demand.
|
|
2894
|
+
*/
|
|
2895
|
+
template <typename progress_at = dummy_progress_t>
|
|
2896
|
+
serialization_result_t view(memory_mapped_file_t file, std::size_t offset = 0,
|
|
2897
|
+
progress_at&& progress = {}) noexcept {
|
|
2898
|
+
|
|
2899
|
+
// Remove previously stored objects
|
|
2900
|
+
reset();
|
|
2901
|
+
|
|
2902
|
+
serialization_result_t result = file.open_if_not();
|
|
2903
|
+
if (!result)
|
|
2904
|
+
return result;
|
|
2905
|
+
|
|
2906
|
+
// Pull basic metadata
|
|
2907
|
+
index_serialized_header_t header;
|
|
2908
|
+
if (file.size() - offset < sizeof(header))
|
|
2909
|
+
return result.failed("File is corrupted and lacks a header");
|
|
2910
|
+
std::memcpy(&header, file.data() + offset, sizeof(header));
|
|
2911
|
+
|
|
2912
|
+
if (!header.size) {
|
|
2913
|
+
reset();
|
|
2914
|
+
return result;
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
// Precompute offsets of every node, but before that we need to update the configs
|
|
2918
|
+
// This could have been done with `std::exclusive_scan`, but it's only available from C++17.
|
|
2919
|
+
using offsets_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
|
|
2920
|
+
buffer_gt<std::size_t, offsets_allocator_t> offsets(header.size);
|
|
2921
|
+
if (!offsets)
|
|
2922
|
+
return result.failed("Out of memory");
|
|
2923
|
+
|
|
2924
|
+
config_.connectivity = header.connectivity;
|
|
2925
|
+
config_.connectivity_base = header.connectivity_base;
|
|
2926
|
+
pre_ = precompute_(config_);
|
|
2927
|
+
misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
|
|
2928
|
+
offsets[0u] = offset + sizeof(header) + sizeof(level_t) * header.size;
|
|
2929
|
+
for (std::size_t i = 1; i < header.size; ++i)
|
|
2930
|
+
offsets[i] = offsets[i - 1] + node_bytes_(levels[i - 1]);
|
|
2931
|
+
|
|
2932
|
+
std::size_t total_bytes = offsets[header.size - 1] + node_bytes_(levels[header.size - 1]);
|
|
2933
|
+
if (file.size() < total_bytes) {
|
|
2934
|
+
reset();
|
|
2935
|
+
return result.failed("File is corrupted and can't fit all the nodes");
|
|
2936
|
+
}
|
|
2937
|
+
|
|
2938
|
+
// Submit metadata and reserve memory
|
|
2939
|
+
index_limits_t limits;
|
|
2940
|
+
limits.members = header.size;
|
|
2941
|
+
if (!reserve(limits)) {
|
|
2942
|
+
reset();
|
|
2943
|
+
return result.failed("Out of memory");
|
|
2944
|
+
}
|
|
2945
|
+
nodes_count_ = header.size;
|
|
2946
|
+
max_level_ = static_cast<level_t>(header.max_level);
|
|
2947
|
+
entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
|
|
2948
|
+
|
|
2949
|
+
// Rapidly address all the nodes
|
|
2950
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2951
|
+
nodes_[i] = node_t{(byte_t*)file.data() + offsets[i]};
|
|
2952
|
+
if (!progress(i + 1, header.size))
|
|
2953
|
+
return result.failed("Terminated by user");
|
|
2954
|
+
}
|
|
2955
|
+
viewed_file_ = std::move(file);
|
|
2956
|
+
return {};
|
|
2957
|
+
}
|
|
2958
|
+
|
|
2959
|
+
#pragma endregion
|
|
2960
|
+
|
|
2961
|
+
/**
|
|
2962
|
+
* @brief Performs compaction on the whole HNSW index, purging some entries
|
|
2963
|
+
* and links to them, while also generating a more efficient mapping,
|
|
2964
|
+
* putting the more frequently used entries closer together.
|
|
2965
|
+
*
|
|
2966
|
+
*
|
|
2967
|
+
* Scans the whole collection, removing the links leading towards
|
|
2968
|
+
* banned entries. This essentially isolates some nodes from the rest
|
|
2969
|
+
* of the graph, while keeping their outgoing links, in case the node
|
|
2970
|
+
* is structurally relevant and has a crucial role in the index.
|
|
2971
|
+
* It won't reclaim the memory.
|
|
2972
|
+
*
|
|
2973
|
+
* @param[in] allow_member Predicate to mark nodes for isolation.
|
|
2974
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
2975
|
+
* @param[in] progress Callback to report the execution progress.
|
|
2976
|
+
*/
|
|
2977
|
+
template <typename values_at, typename metric_at, //
|
|
2978
|
+
typename slot_transition_at = dummy_key_to_key_mapping_t, //
|
|
2979
|
+
typename executor_at = dummy_executor_t, //
|
|
2980
|
+
typename progress_at = dummy_progress_t, //
|
|
2981
|
+
typename prefetch_at = dummy_prefetch_t>
|
|
2982
|
+
void compact( //
|
|
2983
|
+
values_at&& values, //
|
|
2984
|
+
metric_at&& metric, //
|
|
2985
|
+
slot_transition_at&& slot_transition, //
|
|
2986
|
+
|
|
2987
|
+
executor_at&& executor = executor_at{}, //
|
|
2988
|
+
progress_at&& progress = progress_at{}, //
|
|
2989
|
+
prefetch_at&& prefetch = prefetch_at{}) noexcept {
|
|
2990
|
+
|
|
2991
|
+
// Export all the keys, slots, and levels.
|
|
2992
|
+
// Partition them with the predicate.
|
|
2993
|
+
// Sort the allowed entries in descending order of their level.
|
|
2994
|
+
// Create a new array mapping old slots to the new ones (INT_MAX for deleted items).
|
|
2995
|
+
struct slot_level_t {
|
|
2996
|
+
compressed_slot_t old_slot;
|
|
2997
|
+
compressed_slot_t cluster;
|
|
2998
|
+
level_t level;
|
|
2999
|
+
};
|
|
3000
|
+
using slot_level_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<slot_level_t>;
|
|
3001
|
+
buffer_gt<slot_level_t, slot_level_allocator_t> slots_and_levels(size());
|
|
3002
|
+
|
|
3003
|
+
// Progress status
|
|
3004
|
+
std::atomic<bool> do_tasks{true};
|
|
3005
|
+
std::atomic<std::size_t> processed{0};
|
|
3006
|
+
std::size_t const total = 3 * slots_and_levels.size();
|
|
3007
|
+
|
|
3008
|
+
// For every bottom level node, determine its parent cluster
|
|
3009
|
+
executor.dynamic(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
|
|
3010
|
+
context_t& context = contexts_[thread_idx];
|
|
3011
|
+
std::size_t cluster = search_for_one_( //
|
|
3012
|
+
values[citerator_at(old_slot)], //
|
|
3013
|
+
metric, prefetch, //
|
|
3014
|
+
entry_slot_, max_level_, 0, context);
|
|
3015
|
+
slots_and_levels[old_slot] = { //
|
|
3016
|
+
static_cast<compressed_slot_t>(old_slot), //
|
|
3017
|
+
static_cast<compressed_slot_t>(cluster), //
|
|
3018
|
+
node_at_(old_slot).level()};
|
|
3019
|
+
++processed;
|
|
3020
|
+
if (thread_idx == 0)
|
|
3021
|
+
do_tasks = progress(processed.load(), total);
|
|
3022
|
+
return do_tasks.load();
|
|
3023
|
+
});
|
|
3024
|
+
if (!do_tasks.load())
|
|
3025
|
+
return;
|
|
3026
|
+
|
|
3027
|
+
// Where the actual permutation happens:
|
|
3028
|
+
std::sort(slots_and_levels.begin(), slots_and_levels.end(), [](slot_level_t const& a, slot_level_t const& b) {
|
|
3029
|
+
return a.level == b.level ? a.cluster < b.cluster : a.level > b.level;
|
|
3030
|
+
});
|
|
3031
|
+
|
|
3032
|
+
using size_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
|
|
3033
|
+
buffer_gt<std::size_t, size_allocator_t> old_slot_to_new(slots_and_levels.size());
|
|
3034
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot)
|
|
3035
|
+
old_slot_to_new[slots_and_levels[new_slot].old_slot] = new_slot;
|
|
3036
|
+
|
|
3037
|
+
// Erase all the incoming links
|
|
3038
|
+
buffer_gt<node_t, nodes_allocator_t> reordered_nodes(slots_and_levels.size());
|
|
3039
|
+
tape_allocator_t reordered_tape;
|
|
3040
|
+
|
|
3041
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
|
|
3042
|
+
std::size_t old_slot = slots_and_levels[new_slot].old_slot;
|
|
3043
|
+
node_t old_node = node_at_(old_slot);
|
|
3044
|
+
|
|
3045
|
+
std::size_t node_bytes = node_bytes_(old_node.level());
|
|
3046
|
+
byte_t* new_data = (byte_t*)reordered_tape.allocate(node_bytes);
|
|
3047
|
+
node_t new_node{new_data};
|
|
3048
|
+
std::memcpy(new_data, old_node.tape(), node_bytes);
|
|
3049
|
+
|
|
3050
|
+
for (level_t level = 0; level <= old_node.level(); ++level)
|
|
3051
|
+
for (misaligned_ref_gt<compressed_slot_t> neighbor : neighbors_(new_node, level))
|
|
3052
|
+
neighbor = static_cast<compressed_slot_t>(old_slot_to_new[compressed_slot_t(neighbor)]);
|
|
3053
|
+
|
|
3054
|
+
reordered_nodes[new_slot] = new_node;
|
|
3055
|
+
if (!progress(++processed, total))
|
|
3056
|
+
return;
|
|
3057
|
+
}
|
|
3058
|
+
|
|
3059
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
|
|
3060
|
+
std::size_t old_slot = slots_and_levels[new_slot].old_slot;
|
|
3061
|
+
slot_transition(node_at_(old_slot).ckey(), //
|
|
3062
|
+
static_cast<compressed_slot_t>(old_slot), //
|
|
3063
|
+
static_cast<compressed_slot_t>(new_slot));
|
|
3064
|
+
if (!progress(++processed, total))
|
|
3065
|
+
return;
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
nodes_ = std::move(reordered_nodes);
|
|
3069
|
+
tape_allocator_ = std::move(reordered_tape);
|
|
3070
|
+
entry_slot_ = old_slot_to_new[entry_slot_];
|
|
3071
|
+
}
|
|
3072
|
+
|
|
3073
|
+
/**
|
|
3074
|
+
* @brief Scans the whole collection, removing the links leading towards
|
|
3075
|
+
* banned entries. This essentially isolates some nodes from the rest
|
|
3076
|
+
* of the graph, while keeping their outgoing links, in case the node
|
|
3077
|
+
* is structurally relevant and has a crucial role in the index.
|
|
3078
|
+
* It won't reclaim the memory.
|
|
3079
|
+
*
|
|
3080
|
+
* @param[in] allow_member Predicate to mark nodes for isolation.
|
|
3081
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
3082
|
+
* @param[in] progress Callback to report the execution progress.
|
|
3083
|
+
*/
|
|
3084
|
+
template < //
|
|
3085
|
+
typename allow_member_at = dummy_predicate_t, //
|
|
3086
|
+
typename executor_at = dummy_executor_t, //
|
|
3087
|
+
typename progress_at = dummy_progress_t //
|
|
3088
|
+
>
|
|
3089
|
+
void isolate( //
|
|
3090
|
+
allow_member_at&& allow_member, //
|
|
3091
|
+
executor_at&& executor = executor_at{}, //
|
|
3092
|
+
progress_at&& progress = progress_at{}) noexcept {
|
|
3093
|
+
|
|
3094
|
+
// Progress status
|
|
3095
|
+
std::atomic<bool> do_tasks{true};
|
|
3096
|
+
std::atomic<std::size_t> processed{0};
|
|
3097
|
+
|
|
3098
|
+
// Erase all the incoming links
|
|
3099
|
+
std::size_t nodes_count = size();
|
|
3100
|
+
executor.dynamic(nodes_count, [&](std::size_t thread_idx, std::size_t node_idx) {
|
|
3101
|
+
node_t node = node_at_(node_idx);
|
|
3102
|
+
for (level_t level = 0; level <= node.level(); ++level) {
|
|
3103
|
+
neighbors_ref_t neighbors = neighbors_(node, level);
|
|
3104
|
+
std::size_t old_size = neighbors.size();
|
|
3105
|
+
neighbors.clear();
|
|
3106
|
+
for (std::size_t i = 0; i != old_size; ++i) {
|
|
3107
|
+
compressed_slot_t neighbor_slot = neighbors[i];
|
|
3108
|
+
node_t neighbor = node_at_(neighbor_slot);
|
|
3109
|
+
if (allow_member(member_cref_t{neighbor.ckey(), neighbor_slot}))
|
|
3110
|
+
neighbors.push_back(neighbor_slot);
|
|
3111
|
+
}
|
|
3112
|
+
}
|
|
3113
|
+
++processed;
|
|
3114
|
+
if (thread_idx == 0)
|
|
3115
|
+
do_tasks = progress(processed.load(), nodes_count);
|
|
3116
|
+
return do_tasks.load();
|
|
3117
|
+
});
|
|
3118
|
+
|
|
3119
|
+
// At the end report the latest numbers, because the reporter thread may be finished earlier
|
|
3120
|
+
progress(processed.load(), nodes_count);
|
|
3121
|
+
}
|
|
3122
|
+
|
|
3123
|
+
private:
|
|
3124
|
+
inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
|
|
3125
|
+
precomputed_constants_t pre;
|
|
3126
|
+
pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
|
|
3127
|
+
pre.neighbors_bytes = config.connectivity * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
|
|
3128
|
+
pre.neighbors_base_bytes = config.connectivity_base * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
|
|
3129
|
+
return pre;
|
|
3130
|
+
}
|
|
3131
|
+
|
|
3132
|
+
using span_bytes_t = span_gt<byte_t>;
|
|
3133
|
+
|
|
3134
|
+
inline span_bytes_t node_bytes_(node_t node) const noexcept { return {node.tape(), node_bytes_(node.level())}; }
|
|
3135
|
+
inline std::size_t node_bytes_(level_t level) const noexcept {
|
|
3136
|
+
return node_head_bytes_() + node_neighbors_bytes_(level);
|
|
3137
|
+
}
|
|
3138
|
+
inline std::size_t node_neighbors_bytes_(node_t node) const noexcept { return node_neighbors_bytes_(node.level()); }
|
|
3139
|
+
inline std::size_t node_neighbors_bytes_(level_t level) const noexcept {
|
|
3140
|
+
return pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
|
|
3141
|
+
}
|
|
3142
|
+
|
|
3143
|
+
span_bytes_t node_malloc_(level_t level) noexcept {
|
|
3144
|
+
std::size_t node_bytes = node_bytes_(level);
|
|
3145
|
+
byte_t* data = (byte_t*)tape_allocator_.allocate(node_bytes);
|
|
3146
|
+
return data ? span_bytes_t{data, node_bytes} : span_bytes_t{};
|
|
3147
|
+
}
|
|
3148
|
+
|
|
3149
|
+
node_t node_make_(vector_key_t key, level_t level) noexcept {
|
|
3150
|
+
span_bytes_t node_bytes = node_malloc_(level);
|
|
3151
|
+
if (!node_bytes)
|
|
3152
|
+
return {};
|
|
3153
|
+
|
|
3154
|
+
std::memset(node_bytes.data(), 0, node_bytes.size());
|
|
3155
|
+
node_t node{(byte_t*)node_bytes.data()};
|
|
3156
|
+
node.key(key);
|
|
3157
|
+
node.level(level);
|
|
3158
|
+
return node;
|
|
3159
|
+
}
|
|
3160
|
+
|
|
3161
|
+
node_t node_make_copy_(span_bytes_t old_bytes) noexcept {
|
|
3162
|
+
byte_t* data = (byte_t*)tape_allocator_.allocate(old_bytes.size());
|
|
3163
|
+
if (!data)
|
|
3164
|
+
return {};
|
|
3165
|
+
std::memcpy(data, old_bytes.data(), old_bytes.size());
|
|
3166
|
+
return node_t{data};
|
|
3167
|
+
}
|
|
3168
|
+
|
|
3169
|
+
void node_free_(std::size_t idx) noexcept {
|
|
3170
|
+
if (viewed_file_)
|
|
3171
|
+
return;
|
|
3172
|
+
|
|
3173
|
+
node_t& node = nodes_[idx];
|
|
3174
|
+
tape_allocator_.deallocate(node.tape(), node_bytes_(node).size());
|
|
3175
|
+
node = node_t{};
|
|
3176
|
+
}
|
|
3177
|
+
|
|
3178
|
+
inline node_t node_at_(std::size_t idx) const noexcept { return nodes_[idx]; }
|
|
3179
|
+
inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
|
|
3180
|
+
|
|
3181
|
+
inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
|
|
3182
|
+
return {node.neighbors_tape() + pre_.neighbors_base_bytes + (level - 1) * pre_.neighbors_bytes};
|
|
3183
|
+
}
|
|
3184
|
+
|
|
3185
|
+
inline neighbors_ref_t neighbors_(node_t node, level_t level) const noexcept {
|
|
3186
|
+
return level ? neighbors_non_base_(node, level) : neighbors_base_(node);
|
|
3187
|
+
}
|
|
3188
|
+
|
|
3189
|
+
struct node_lock_t {
|
|
3190
|
+
nodes_mutexes_t& mutexes;
|
|
3191
|
+
std::size_t slot;
|
|
3192
|
+
inline ~node_lock_t() noexcept { mutexes.atomic_reset(slot); }
|
|
3193
|
+
};
|
|
3194
|
+
|
|
3195
|
+
inline node_lock_t node_lock_(std::size_t slot) const noexcept {
|
|
3196
|
+
while (nodes_mutexes_.atomic_set(slot))
|
|
3197
|
+
;
|
|
3198
|
+
return {nodes_mutexes_, slot};
|
|
3199
|
+
}
|
|
3200
|
+
|
|
3201
|
+
template <typename value_at, typename metric_at, typename prefetch_at>
|
|
3202
|
+
void connect_node_across_levels_( //
|
|
3203
|
+
value_at&& value, metric_at&& metric, prefetch_at&& prefetch, //
|
|
3204
|
+
std::size_t node_slot, std::size_t entry_slot, level_t max_level, level_t target_level, //
|
|
3205
|
+
index_update_config_t const& config, context_t& context) usearch_noexcept_m {
|
|
3206
|
+
|
|
3207
|
+
// Go down the level, tracking only the closest match
|
|
3208
|
+
std::size_t closest_slot = search_for_one_( //
|
|
3209
|
+
value, metric, prefetch, //
|
|
3210
|
+
entry_slot, max_level, target_level, context);
|
|
3211
|
+
|
|
3212
|
+
// From `target_level` down perform proper extensive search
|
|
3213
|
+
for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
|
|
3214
|
+
// TODO: Handle out of memory conditions
|
|
3215
|
+
search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
|
|
3216
|
+
closest_slot = connect_new_node_(metric, node_slot, level, context);
|
|
3217
|
+
reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
|
|
3218
|
+
}
|
|
3219
|
+
}
|
|
3220
|
+
|
|
3221
|
+
template <typename metric_at>
|
|
3222
|
+
std::size_t connect_new_node_( //
|
|
3223
|
+
metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
|
|
3224
|
+
|
|
3225
|
+
node_t new_node = node_at_(new_slot);
|
|
3226
|
+
top_candidates_t& top = context.top_candidates;
|
|
3227
|
+
|
|
3228
|
+
// Outgoing links from `new_slot`:
|
|
3229
|
+
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3230
|
+
{
|
|
3231
|
+
usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
|
|
3232
|
+
candidates_view_t top_view = refine_(metric, config_.connectivity, top, context);
|
|
3233
|
+
|
|
3234
|
+
for (std::size_t idx = 0; idx != top_view.size(); idx++) {
|
|
3235
|
+
usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
3236
|
+
usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
|
|
3237
|
+
new_neighbors.push_back(top_view[idx].slot);
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
|
|
3241
|
+
return new_neighbors[0];
|
|
3242
|
+
}
|
|
3243
|
+
|
|
3244
|
+
template <typename value_at, typename metric_at>
|
|
3245
|
+
void reconnect_neighbor_nodes_( //
|
|
3246
|
+
metric_at&& metric, std::size_t new_slot, value_at&& value, level_t level,
|
|
3247
|
+
context_t& context) usearch_noexcept_m {
|
|
3248
|
+
|
|
3249
|
+
node_t new_node = node_at_(new_slot);
|
|
3250
|
+
top_candidates_t& top = context.top_candidates;
|
|
3251
|
+
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3252
|
+
|
|
3253
|
+
// Reverse links from the neighbors:
|
|
3254
|
+
std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
|
|
3255
|
+
for (compressed_slot_t close_slot : new_neighbors) {
|
|
3256
|
+
if (close_slot == new_slot)
|
|
3257
|
+
continue;
|
|
3258
|
+
node_lock_t close_lock = node_lock_(close_slot);
|
|
3259
|
+
node_t close_node = node_at_(close_slot);
|
|
3260
|
+
|
|
3261
|
+
neighbors_ref_t close_header = neighbors_(close_node, level);
|
|
3262
|
+
usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
|
|
3263
|
+
usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
|
|
3264
|
+
usearch_assert_m(level <= close_node.level(), "Linking to missing level");
|
|
3265
|
+
|
|
3266
|
+
// If `new_slot` is already present in the neighboring connections of `close_slot`
|
|
3267
|
+
// then no need to modify any connections or run the heuristics.
|
|
3268
|
+
if (close_header.size() < connectivity_max) {
|
|
3269
|
+
close_header.push_back(static_cast<compressed_slot_t>(new_slot));
|
|
3270
|
+
continue;
|
|
3271
|
+
}
|
|
3272
|
+
|
|
3273
|
+
// To fit a new connection we need to drop an existing one.
|
|
3274
|
+
top.clear();
|
|
3275
|
+
usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
|
|
3276
|
+
top.insert_reserved(
|
|
3277
|
+
{context.measure(value, citerator_at(close_slot), metric), static_cast<compressed_slot_t>(new_slot)});
|
|
3278
|
+
for (compressed_slot_t successor_slot : close_header)
|
|
3279
|
+
top.insert_reserved(
|
|
3280
|
+
{context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
|
|
3281
|
+
|
|
3282
|
+
// Export the results:
|
|
3283
|
+
close_header.clear();
|
|
3284
|
+
candidates_view_t top_view = refine_(metric, connectivity_max, top, context);
|
|
3285
|
+
for (std::size_t idx = 0; idx != top_view.size(); idx++)
|
|
3286
|
+
close_header.push_back(top_view[idx].slot);
|
|
3287
|
+
}
|
|
3288
|
+
}
|
|
3289
|
+
|
|
3290
|
+
level_t choose_random_level_(std::default_random_engine& level_generator) const noexcept {
|
|
3291
|
+
std::uniform_real_distribution<double> distribution(0.0, 1.0);
|
|
3292
|
+
double r = -std::log(distribution(level_generator)) * pre_.inverse_log_connectivity;
|
|
3293
|
+
return (level_t)r;
|
|
3294
|
+
}
|
|
3295
|
+
|
|
3296
|
+
struct candidates_range_t;
|
|
3297
|
+
class candidates_iterator_t {
|
|
3298
|
+
friend struct candidates_range_t;
|
|
3299
|
+
|
|
3300
|
+
index_gt const& index_;
|
|
3301
|
+
neighbors_ref_t neighbors_;
|
|
3302
|
+
visits_hash_set_t& visits_;
|
|
3303
|
+
std::size_t current_;
|
|
3304
|
+
|
|
3305
|
+
candidates_iterator_t& skip_missing() noexcept {
|
|
3306
|
+
if (!visits_.size())
|
|
3307
|
+
return *this;
|
|
3308
|
+
while (current_ != neighbors_.size()) {
|
|
3309
|
+
compressed_slot_t neighbor_slot = neighbors_[current_];
|
|
3310
|
+
if (visits_.test(neighbor_slot))
|
|
3311
|
+
current_++;
|
|
3312
|
+
else
|
|
3313
|
+
break;
|
|
3314
|
+
}
|
|
3315
|
+
return *this;
|
|
3316
|
+
}
|
|
3317
|
+
|
|
3318
|
+
public:
|
|
3319
|
+
using element_t = compressed_slot_t;
|
|
3320
|
+
using iterator_category = std::forward_iterator_tag;
|
|
3321
|
+
using value_type = element_t;
|
|
3322
|
+
using difference_type = std::ptrdiff_t;
|
|
3323
|
+
using pointer = misaligned_ptr_gt<element_t>;
|
|
3324
|
+
using reference = misaligned_ref_gt<element_t>;
|
|
3325
|
+
|
|
3326
|
+
reference operator*() const noexcept { return slot(); }
|
|
3327
|
+
candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_hash_set_t& visits,
|
|
3328
|
+
std::size_t progress) noexcept
|
|
3329
|
+
: index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
|
|
3330
|
+
candidates_iterator_t operator++(int) noexcept {
|
|
3331
|
+
return candidates_iterator_t(index_, visits_, neighbors_, current_ + 1).skip_missing();
|
|
3332
|
+
}
|
|
3333
|
+
candidates_iterator_t& operator++() noexcept {
|
|
3334
|
+
++current_;
|
|
3335
|
+
skip_missing();
|
|
3336
|
+
return *this;
|
|
3337
|
+
}
|
|
3338
|
+
bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
|
|
3339
|
+
bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
|
|
3340
|
+
|
|
3341
|
+
vector_key_t key() const noexcept { return index_->node_at_(slot()).key(); }
|
|
3342
|
+
compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
|
|
3343
|
+
friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
|
|
3344
|
+
friend inline vector_key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
|
|
3345
|
+
};
|
|
3346
|
+
|
|
3347
|
+
struct candidates_range_t {
|
|
3348
|
+
index_gt const& index;
|
|
3349
|
+
neighbors_ref_t neighbors;
|
|
3350
|
+
visits_hash_set_t& visits;
|
|
3351
|
+
|
|
3352
|
+
candidates_iterator_t begin() const noexcept {
|
|
3353
|
+
return candidates_iterator_t{index, neighbors, visits, 0}.skip_missing();
|
|
3354
|
+
}
|
|
3355
|
+
candidates_iterator_t end() const noexcept { return {index, neighbors, visits, neighbors.size()}; }
|
|
3356
|
+
};
|
|
3357
|
+
|
|
3358
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
3359
|
+
std::size_t search_for_one_( //
|
|
3360
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
3361
|
+
std::size_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
|
|
3362
|
+
|
|
3363
|
+
visits_hash_set_t& visits = context.visits;
|
|
3364
|
+
visits.clear();
|
|
3365
|
+
|
|
3366
|
+
// Optional prefetching
|
|
3367
|
+
if (!is_dummy<prefetch_at>())
|
|
3368
|
+
prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
|
|
3369
|
+
|
|
3370
|
+
distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
|
|
3371
|
+
for (level_t level = begin_level; level > end_level; --level) {
|
|
3372
|
+
bool changed;
|
|
3373
|
+
do {
|
|
3374
|
+
changed = false;
|
|
3375
|
+
node_lock_t closest_lock = node_lock_(closest_slot);
|
|
3376
|
+
neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
|
|
3377
|
+
|
|
3378
|
+
// Optional prefetching
|
|
3379
|
+
if (!is_dummy<prefetch_at>()) {
|
|
3380
|
+
candidates_range_t missing_candidates{*this, closest_neighbors, visits};
|
|
3381
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3382
|
+
}
|
|
3383
|
+
|
|
3384
|
+
// Actual traversal
|
|
3385
|
+
for (compressed_slot_t candidate_slot : closest_neighbors) {
|
|
3386
|
+
distance_t candidate_dist = context.measure(query, citerator_at(candidate_slot), metric);
|
|
3387
|
+
if (candidate_dist < closest_dist) {
|
|
3388
|
+
closest_dist = candidate_dist;
|
|
3389
|
+
closest_slot = candidate_slot;
|
|
3390
|
+
changed = true;
|
|
3391
|
+
}
|
|
3392
|
+
}
|
|
3393
|
+
context.iteration_cycles++;
|
|
3394
|
+
} while (changed);
|
|
3395
|
+
}
|
|
3396
|
+
return closest_slot;
|
|
3397
|
+
}
|
|
3398
|
+
|
|
3399
|
+
/**
|
|
3400
|
+
* @brief Traverses a layer of a graph, to find the best place to insert a new node.
|
|
3401
|
+
* Locks the nodes in the process, assuming other threads are updating neighbors lists.
|
|
3402
|
+
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3403
|
+
*/
|
|
3404
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
3405
|
+
bool search_to_insert_( //
|
|
3406
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
3407
|
+
std::size_t start_slot, std::size_t new_slot, level_t level, std::size_t top_limit,
|
|
3408
|
+
context_t& context) noexcept {
|
|
3409
|
+
|
|
3410
|
+
visits_hash_set_t& visits = context.visits;
|
|
3411
|
+
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
3412
|
+
top_candidates_t& top = context.top_candidates; // pop max, push
|
|
3413
|
+
|
|
3414
|
+
visits.clear();
|
|
3415
|
+
next.clear();
|
|
3416
|
+
top.clear();
|
|
3417
|
+
if (!visits.reserve(config_.connectivity_base + 1u))
|
|
3418
|
+
return false;
|
|
3419
|
+
|
|
3420
|
+
// Optional prefetching
|
|
3421
|
+
if (!is_dummy<prefetch_at>())
|
|
3422
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
3423
|
+
|
|
3424
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
3425
|
+
next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3426
|
+
top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3427
|
+
visits.set(static_cast<compressed_slot_t>(start_slot));
|
|
3428
|
+
|
|
3429
|
+
while (!next.empty()) {
|
|
3430
|
+
|
|
3431
|
+
candidate_t candidacy = next.top();
|
|
3432
|
+
if ((-candidacy.distance) > radius && top.size() == top_limit)
|
|
3433
|
+
break;
|
|
3434
|
+
|
|
3435
|
+
next.pop();
|
|
3436
|
+
context.iteration_cycles++;
|
|
3437
|
+
|
|
3438
|
+
compressed_slot_t candidate_slot = candidacy.slot;
|
|
3439
|
+
if (new_slot == candidate_slot)
|
|
3440
|
+
continue;
|
|
3441
|
+
node_t candidate_ref = node_at_(candidate_slot);
|
|
3442
|
+
node_lock_t candidate_lock = node_lock_(candidate_slot);
|
|
3443
|
+
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
3444
|
+
|
|
3445
|
+
// Optional prefetching
|
|
3446
|
+
if (!is_dummy<prefetch_at>()) {
|
|
3447
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3448
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3449
|
+
}
|
|
3450
|
+
|
|
3451
|
+
// Assume the worst-case when reserving memory
|
|
3452
|
+
if (!visits.reserve(visits.size() + candidate_neighbors.size()))
|
|
3453
|
+
return false;
|
|
3454
|
+
|
|
3455
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
3456
|
+
if (visits.set(successor_slot))
|
|
3457
|
+
continue;
|
|
3458
|
+
|
|
3459
|
+
// node_lock_t successor_lock = node_lock_(successor_slot);
|
|
3460
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3461
|
+
if (top.size() < top_limit || successor_dist < radius) {
|
|
3462
|
+
// This can substantially grow our priority queue:
|
|
3463
|
+
next.insert({-successor_dist, successor_slot});
|
|
3464
|
+
// This will automatically evict poor matches:
|
|
3465
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
3466
|
+
radius = top.top().distance;
|
|
3467
|
+
}
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
return true;
|
|
3471
|
+
}
|
|
3472
|
+
|
|
3473
|
+
/**
|
|
3474
|
+
* @brief Traverses the @b base layer of a graph, to find a close match.
|
|
3475
|
+
* Doesn't lock any nodes, assuming read-only simultaneous access.
|
|
3476
|
+
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3477
|
+
*/
|
|
3478
|
+
template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
|
|
3479
|
+
bool search_to_find_in_base_( //
|
|
3480
|
+
value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
|
|
3481
|
+
std::size_t start_slot, std::size_t expansion, context_t& context) const noexcept {
|
|
3482
|
+
|
|
3483
|
+
visits_hash_set_t& visits = context.visits;
|
|
3484
|
+
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
3485
|
+
top_candidates_t& top = context.top_candidates; // pop max, push
|
|
3486
|
+
std::size_t const top_limit = expansion;
|
|
3487
|
+
|
|
3488
|
+
visits.clear();
|
|
3489
|
+
next.clear();
|
|
3490
|
+
top.clear();
|
|
3491
|
+
if (!visits.reserve(config_.connectivity_base + 1u))
|
|
3492
|
+
return false;
|
|
3493
|
+
|
|
3494
|
+
// Optional prefetching
|
|
3495
|
+
if (!is_dummy<prefetch_at>())
|
|
3496
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
3497
|
+
|
|
3498
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
3499
|
+
next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3500
|
+
top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3501
|
+
visits.set(static_cast<compressed_slot_t>(start_slot));
|
|
3502
|
+
|
|
3503
|
+
while (!next.empty()) {
|
|
3504
|
+
|
|
3505
|
+
candidate_t candidate = next.top();
|
|
3506
|
+
if ((-candidate.distance) > radius)
|
|
3507
|
+
break;
|
|
3508
|
+
|
|
3509
|
+
next.pop();
|
|
3510
|
+
context.iteration_cycles++;
|
|
3511
|
+
|
|
3512
|
+
neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
|
|
3513
|
+
|
|
3514
|
+
// Optional prefetching
|
|
3515
|
+
if (!is_dummy<prefetch_at>()) {
|
|
3516
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3517
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3518
|
+
}
|
|
3519
|
+
|
|
3520
|
+
// Assume the worst-case when reserving memory
|
|
3521
|
+
if (!visits.reserve(visits.size() + candidate_neighbors.size()))
|
|
3522
|
+
return false;
|
|
3523
|
+
|
|
3524
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
3525
|
+
if (visits.set(successor_slot))
|
|
3526
|
+
continue;
|
|
3527
|
+
|
|
3528
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3529
|
+
if (top.size() < top_limit || successor_dist < radius) {
|
|
3530
|
+
// This can substantially grow our priority queue:
|
|
3531
|
+
next.insert({-successor_dist, successor_slot});
|
|
3532
|
+
if (!is_dummy<predicate_at>())
|
|
3533
|
+
if (!predicate(member_cref_t{node_at_(successor_slot).ckey(), successor_slot}))
|
|
3534
|
+
continue;
|
|
3535
|
+
|
|
3536
|
+
// This will automatically evict poor matches:
|
|
3537
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
3538
|
+
radius = top.top().distance;
|
|
3539
|
+
}
|
|
3540
|
+
}
|
|
3541
|
+
}
|
|
3542
|
+
|
|
3543
|
+
return true;
|
|
3544
|
+
}
|
|
3545
|
+
|
|
3546
|
+
/**
|
|
3547
|
+
* @brief Iterates through all members, without actually touching the index.
|
|
3548
|
+
*/
|
|
3549
|
+
template <typename value_at, typename metric_at, typename predicate_at>
|
|
3550
|
+
void search_exact_( //
|
|
3551
|
+
value_at&& query, metric_at&& metric, predicate_at&& predicate, //
|
|
3552
|
+
std::size_t count, context_t& context) const noexcept {
|
|
3553
|
+
|
|
3554
|
+
top_candidates_t& top = context.top_candidates;
|
|
3555
|
+
top.clear();
|
|
3556
|
+
top.reserve(count);
|
|
3557
|
+
for (std::size_t i = 0; i != size(); ++i) {
|
|
3558
|
+
if (!is_dummy<predicate_at>())
|
|
3559
|
+
if (!predicate(at(i)))
|
|
3560
|
+
continue;
|
|
3561
|
+
|
|
3562
|
+
distance_t distance = context.measure(query, citerator_at(i), metric);
|
|
3563
|
+
top.insert(candidate_t{distance, static_cast<compressed_slot_t>(i)}, count);
|
|
3564
|
+
}
|
|
3565
|
+
}
|
|
3566
|
+
|
|
3567
|
+
/**
|
|
3568
|
+
* @brief This algorithm from the original paper implements a heuristic,
|
|
3569
|
+
* that massively reduces the number of connections a point has,
|
|
3570
|
+
* to keep only the neighbors, that are from each other.
|
|
3571
|
+
*/
|
|
3572
|
+
template <typename metric_at>
|
|
3573
|
+
candidates_view_t refine_( //
|
|
3574
|
+
metric_at&& metric, //
|
|
3575
|
+
std::size_t needed, top_candidates_t& top, context_t& context) const noexcept {
|
|
3576
|
+
|
|
3577
|
+
top.sort_ascending();
|
|
3578
|
+
candidate_t* top_data = top.data();
|
|
3579
|
+
std::size_t const top_count = top.size();
|
|
3580
|
+
if (top_count < needed)
|
|
3581
|
+
return {top_data, top_count};
|
|
3582
|
+
|
|
3583
|
+
std::size_t submitted_count = 1;
|
|
3584
|
+
std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
|
|
3585
|
+
while (submitted_count < needed && consumed_count < top_count) {
|
|
3586
|
+
candidate_t candidate = top_data[consumed_count];
|
|
3587
|
+
bool good = true;
|
|
3588
|
+
for (std::size_t idx = 0; idx < submitted_count; idx++) {
|
|
3589
|
+
candidate_t submitted = top_data[idx];
|
|
3590
|
+
distance_t inter_result_dist = context.measure( //
|
|
3591
|
+
citerator_at(candidate.slot), //
|
|
3592
|
+
citerator_at(submitted.slot), //
|
|
3593
|
+
metric);
|
|
3594
|
+
if (inter_result_dist < candidate.distance) {
|
|
3595
|
+
good = false;
|
|
3596
|
+
break;
|
|
3597
|
+
}
|
|
3598
|
+
}
|
|
3599
|
+
|
|
3600
|
+
if (good) {
|
|
3601
|
+
top_data[submitted_count] = top_data[consumed_count];
|
|
3602
|
+
submitted_count++;
|
|
3603
|
+
}
|
|
3604
|
+
consumed_count++;
|
|
3605
|
+
}
|
|
3606
|
+
|
|
3607
|
+
top.shrink(submitted_count);
|
|
3608
|
+
return {top_data, submitted_count};
|
|
3609
|
+
}
|
|
3610
|
+
};
|
|
3611
|
+
|
|
3612
|
+
struct join_result_t {
|
|
3613
|
+
error_t error{};
|
|
3614
|
+
std::size_t intersection_size{};
|
|
3615
|
+
std::size_t engagements{};
|
|
3616
|
+
std::size_t visited_members{};
|
|
3617
|
+
std::size_t computed_distances{};
|
|
3618
|
+
|
|
3619
|
+
explicit operator bool() const noexcept { return !error; }
|
|
3620
|
+
join_result_t failed(error_t message) noexcept {
|
|
3621
|
+
error = std::move(message);
|
|
3622
|
+
return std::move(*this);
|
|
3623
|
+
}
|
|
3624
|
+
};
|
|
3625
|
+
|
|
3626
|
+
/**
|
|
3627
|
+
* @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
|
|
3628
|
+
* to perform fast one-to-one matching between two large collections
|
|
3629
|
+
* of vectors, using approximate nearest neighbors search.
|
|
3630
|
+
*
|
|
3631
|
+
* @param[inout] man_to_woman Container to map ::men keys to ::women.
|
|
3632
|
+
* @param[inout] woman_to_man Container to map ::women keys to ::men.
|
|
3633
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
3634
|
+
* @param[in] progress Callback to report the execution progress.
|
|
3635
|
+
*/
|
|
3636
|
+
template < //
|
|
3637
|
+
|
|
3638
|
+
typename men_at, //
|
|
3639
|
+
typename women_at, //
|
|
3640
|
+
typename men_values_at, //
|
|
3641
|
+
typename women_values_at, //
|
|
3642
|
+
typename men_metric_at, //
|
|
3643
|
+
typename women_metric_at, //
|
|
3644
|
+
|
|
3645
|
+
typename man_to_woman_at = dummy_key_to_key_mapping_t, //
|
|
3646
|
+
typename woman_to_man_at = dummy_key_to_key_mapping_t, //
|
|
3647
|
+
typename executor_at = dummy_executor_t, //
|
|
3648
|
+
typename progress_at = dummy_progress_t //
|
|
3649
|
+
>
|
|
3650
|
+
static join_result_t join( //
|
|
3651
|
+
men_at const& men, //
|
|
3652
|
+
women_at const& women, //
|
|
3653
|
+
men_values_at const& men_values, //
|
|
3654
|
+
women_values_at const& women_values, //
|
|
3655
|
+
men_metric_at&& men_metric, //
|
|
3656
|
+
women_metric_at&& women_metric, //
|
|
3657
|
+
|
|
3658
|
+
index_join_config_t config = {}, //
|
|
3659
|
+
man_to_woman_at&& man_to_woman = man_to_woman_at{}, //
|
|
3660
|
+
woman_to_man_at&& woman_to_man = woman_to_man_at{}, //
|
|
3661
|
+
executor_at&& executor = executor_at{}, //
|
|
3662
|
+
progress_at&& progress = progress_at{}) noexcept {
|
|
3663
|
+
|
|
3664
|
+
if (women.size() < men.size())
|
|
3665
|
+
return unum::usearch::join( //
|
|
3666
|
+
women, men, //
|
|
3667
|
+
women_values, men_values, //
|
|
3668
|
+
std::forward<women_metric_at>(women_metric), std::forward<men_metric_at>(men_metric), //
|
|
3669
|
+
|
|
3670
|
+
config, //
|
|
3671
|
+
std::forward<woman_to_man_at>(woman_to_man), //
|
|
3672
|
+
std::forward<man_to_woman_at>(man_to_woman), //
|
|
3673
|
+
std::forward<executor_at>(executor), //
|
|
3674
|
+
std::forward<progress_at>(progress));
|
|
3675
|
+
|
|
3676
|
+
join_result_t result;
|
|
3677
|
+
|
|
3678
|
+
// Sanity checks and argument validation:
|
|
3679
|
+
if (&men == &women)
|
|
3680
|
+
return result.failed("Can't join with itself, consider copying");
|
|
3681
|
+
|
|
3682
|
+
if (config.max_proposals == 0)
|
|
3683
|
+
config.max_proposals = std::log(men.size()) + executor.size();
|
|
3684
|
+
|
|
3685
|
+
using proposals_count_t = std::uint16_t;
|
|
3686
|
+
config.max_proposals = (std::min)(men.size(), config.max_proposals);
|
|
3687
|
+
|
|
3688
|
+
using distance_t = typename men_at::distance_t;
|
|
3689
|
+
using dynamic_allocator_traits_t = typename men_at::dynamic_allocator_traits_t;
|
|
3690
|
+
using man_key_t = typename men_at::vector_key_t;
|
|
3691
|
+
using woman_key_t = typename women_at::vector_key_t;
|
|
3692
|
+
|
|
3693
|
+
// Use the `compressed_slot_t` type of the larger collection
|
|
3694
|
+
using compressed_slot_t = typename women_at::compressed_slot_t;
|
|
3695
|
+
using compressed_slot_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<compressed_slot_t>;
|
|
3696
|
+
using proposals_count_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<proposals_count_t>;
|
|
3697
|
+
|
|
3698
|
+
// Create an atomic queue, as a ring structure, from/to which
|
|
3699
|
+
// free men will be added/pulled.
|
|
3700
|
+
std::mutex free_men_mutex{};
|
|
3701
|
+
ring_gt<compressed_slot_t, compressed_slot_allocator_t> free_men;
|
|
3702
|
+
free_men.reserve(men.size());
|
|
3703
|
+
for (std::size_t i = 0; i != men.size(); ++i)
|
|
3704
|
+
free_men.push(static_cast<compressed_slot_t>(i));
|
|
3705
|
+
|
|
3706
|
+
// We are gonna need some temporary memory.
|
|
3707
|
+
buffer_gt<proposals_count_t, proposals_count_allocator_t> proposal_counts(men.size());
|
|
3708
|
+
buffer_gt<compressed_slot_t, compressed_slot_allocator_t> man_to_woman_slots(men.size());
|
|
3709
|
+
buffer_gt<compressed_slot_t, compressed_slot_allocator_t> woman_to_man_slots(women.size());
|
|
3710
|
+
if (!proposal_counts || !man_to_woman_slots || !woman_to_man_slots)
|
|
3711
|
+
return result.failed("Can't temporary mappings");
|
|
3712
|
+
|
|
3713
|
+
compressed_slot_t missing_slot;
|
|
3714
|
+
std::memset((void*)&missing_slot, 0xFF, sizeof(compressed_slot_t));
|
|
3715
|
+
std::memset((void*)man_to_woman_slots.data(), 0xFF, sizeof(compressed_slot_t) * men.size());
|
|
3716
|
+
std::memset((void*)woman_to_man_slots.data(), 0xFF, sizeof(compressed_slot_t) * women.size());
|
|
3717
|
+
std::memset(proposal_counts.data(), 0, sizeof(proposals_count_t) * men.size());
|
|
3718
|
+
|
|
3719
|
+
// Define locks, to limit concurrent accesses to `man_to_woman_slots` and `woman_to_man_slots`.
|
|
3720
|
+
bitset_t men_locks(men.size()), women_locks(women.size());
|
|
3721
|
+
if (!men_locks || !women_locks)
|
|
3722
|
+
return result.failed("Can't allocate locks");
|
|
3723
|
+
|
|
3724
|
+
std::atomic<std::size_t> rounds{0};
|
|
3725
|
+
std::atomic<std::size_t> engagements{0};
|
|
3726
|
+
std::atomic<std::size_t> computed_distances{0};
|
|
3727
|
+
std::atomic<std::size_t> visited_members{0};
|
|
3728
|
+
std::atomic<char const*> atomic_error{nullptr};
|
|
3729
|
+
|
|
3730
|
+
// Concurrently process all the men
|
|
3731
|
+
executor.parallel([&](std::size_t thread_idx) {
|
|
3732
|
+
index_search_config_t search_config;
|
|
3733
|
+
search_config.expansion = config.expansion;
|
|
3734
|
+
search_config.exact = config.exact;
|
|
3735
|
+
search_config.thread = thread_idx;
|
|
3736
|
+
compressed_slot_t free_man_slot;
|
|
3737
|
+
|
|
3738
|
+
// While there exist a free man who still has a woman to propose to.
|
|
3739
|
+
while (!atomic_error.load(std::memory_order_relaxed)) {
|
|
3740
|
+
std::size_t passed_rounds = 0;
|
|
3741
|
+
std::size_t total_rounds = 0;
|
|
3742
|
+
{
|
|
3743
|
+
std::unique_lock<std::mutex> pop_lock(free_men_mutex);
|
|
3744
|
+
if (!free_men.try_pop(free_man_slot))
|
|
3745
|
+
// Primary exit path, we have exhausted the list of candidates
|
|
3746
|
+
break;
|
|
3747
|
+
passed_rounds = ++rounds;
|
|
3748
|
+
total_rounds = passed_rounds + free_men.size();
|
|
3749
|
+
}
|
|
3750
|
+
if (thread_idx == 0 && !progress(passed_rounds, total_rounds)) {
|
|
3751
|
+
atomic_error.store("Terminated by user");
|
|
3752
|
+
break;
|
|
3753
|
+
}
|
|
3754
|
+
while (men_locks.atomic_set(free_man_slot))
|
|
3755
|
+
;
|
|
3756
|
+
|
|
3757
|
+
proposals_count_t& free_man_proposals = proposal_counts[free_man_slot];
|
|
3758
|
+
if (free_man_proposals >= config.max_proposals)
|
|
3759
|
+
continue;
|
|
3760
|
+
|
|
3761
|
+
// Find the closest woman, to whom this man hasn't proposed yet.
|
|
3762
|
+
++free_man_proposals;
|
|
3763
|
+
auto candidates = women.search(men_values[free_man_slot], free_man_proposals, women_metric, search_config);
|
|
3764
|
+
visited_members += candidates.visited_members;
|
|
3765
|
+
computed_distances += candidates.computed_distances;
|
|
3766
|
+
if (!candidates) {
|
|
3767
|
+
atomic_error = candidates.error.release();
|
|
3768
|
+
break;
|
|
3769
|
+
}
|
|
3770
|
+
|
|
3771
|
+
auto match = candidates.back();
|
|
3772
|
+
auto woman = match.member;
|
|
3773
|
+
while (women_locks.atomic_set(woman.slot))
|
|
3774
|
+
;
|
|
3775
|
+
|
|
3776
|
+
compressed_slot_t husband_slot = woman_to_man_slots[woman.slot];
|
|
3777
|
+
bool woman_is_free = husband_slot == missing_slot;
|
|
3778
|
+
if (woman_is_free) {
|
|
3779
|
+
// Engagement
|
|
3780
|
+
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
3781
|
+
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3782
|
+
engagements++;
|
|
3783
|
+
} else {
|
|
3784
|
+
distance_t distance_from_husband = women_metric(women_values[woman.slot], men_values[husband_slot]);
|
|
3785
|
+
distance_t distance_from_candidate = match.distance;
|
|
3786
|
+
if (distance_from_husband > distance_from_candidate) {
|
|
3787
|
+
// Break-up
|
|
3788
|
+
while (men_locks.atomic_set(husband_slot))
|
|
3789
|
+
;
|
|
3790
|
+
man_to_woman_slots[husband_slot] = missing_slot;
|
|
3791
|
+
men_locks.atomic_reset(husband_slot);
|
|
3792
|
+
|
|
3793
|
+
// New Engagement
|
|
3794
|
+
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
3795
|
+
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3796
|
+
engagements++;
|
|
3797
|
+
|
|
3798
|
+
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
3799
|
+
free_men.push(husband_slot);
|
|
3800
|
+
} else {
|
|
3801
|
+
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
3802
|
+
free_men.push(free_man_slot);
|
|
3803
|
+
}
|
|
3804
|
+
}
|
|
3805
|
+
|
|
3806
|
+
men_locks.atomic_reset(free_man_slot);
|
|
3807
|
+
women_locks.atomic_reset(woman.slot);
|
|
3808
|
+
}
|
|
3809
|
+
});
|
|
3810
|
+
|
|
3811
|
+
if (atomic_error)
|
|
3812
|
+
return result.failed(atomic_error.load());
|
|
3813
|
+
|
|
3814
|
+
// Export the "slots" into keys:
|
|
3815
|
+
std::size_t intersection_size = 0;
|
|
3816
|
+
for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
|
|
3817
|
+
compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
|
|
3818
|
+
if (woman_slot != missing_slot) {
|
|
3819
|
+
man_key_t man = men.at(man_slot).key;
|
|
3820
|
+
woman_key_t woman = women.at(woman_slot).key;
|
|
3821
|
+
man_to_woman[man] = woman;
|
|
3822
|
+
woman_to_man[woman] = man;
|
|
3823
|
+
intersection_size++;
|
|
3824
|
+
}
|
|
3825
|
+
}
|
|
3826
|
+
|
|
3827
|
+
// Export stats
|
|
3828
|
+
result.engagements = engagements;
|
|
3829
|
+
result.intersection_size = intersection_size;
|
|
3830
|
+
result.computed_distances = computed_distances;
|
|
3831
|
+
result.visited_members = visited_members;
|
|
3832
|
+
return result;
|
|
3833
|
+
}
|
|
3834
|
+
|
|
3835
|
+
} // namespace usearch
|
|
3836
|
+
} // namespace unum
|
|
3837
|
+
|
|
3838
|
+
#endif
|