usearch 0.22.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.releaserc +9 -0
- package/.vscode/launch.json +15 -11
- package/.vscode/settings.json +12 -2
- package/.vscode/tasks.json +4 -4
- package/CMakeLists.txt +6 -1
- package/Cargo.toml +2 -2
- package/README.md +39 -46
- package/build.rs +2 -2
- package/c/CMakeLists.txt +48 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +83 -72
- package/c/test.c +20 -20
- package/c/usearch.h +132 -32
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +3 -1
- package/cpp/README.md +16 -16
- package/cpp/bench.cpp +50 -126
- package/cpp/test.cpp +164 -77
- package/docs/benchmarks.md +26 -5
- package/docs/c/reference.rst +3 -3
- package/docs/compilation.md +49 -45
- package/docs/cpp/index.rst +2 -2
- package/docs/cpp/reference.rst +13 -3
- package/docs/index.rst +8 -8
- package/docs/java/reference.rst +1 -1
- package/golang/lib.go +275 -141
- package/golang/lib_test.go +105 -0
- package/include/usearch/index.hpp +1733 -1659
- package/include/usearch/index_dense.hpp +1400 -0
- package/include/usearch/index_plugins.hpp +1263 -0
- package/java/README.md +1 -1
- package/java/cloud/unum/usearch/Index.java +10 -10
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
- package/java/test/IndexTest.java +1 -1
- package/javascript/README.md +7 -7
- package/javascript/lib.cpp +94 -55
- package/javascript/test.js +25 -11
- package/javascript/usearch.d.ts +87 -0
- package/javascript/usearch.js +2 -118
- package/objc/USearchObjective.mm +57 -31
- package/objc/include/USearchObjective.h +32 -22
- package/package.json +5 -3
- package/pyproject.toml +3 -3
- package/python/README.md +23 -14
- package/python/lib.cpp +472 -444
- package/python/scripts/bench.ipynb +3 -2
- package/python/scripts/bench.py +4 -4
- package/python/scripts/index_faiss.py +5 -5
- package/python/scripts/join.py +4 -4
- package/python/scripts/test.py +152 -61
- package/python/scripts/test_jit.py +10 -12
- package/python/scripts/test_sparse.py +32 -0
- package/python/usearch/client.py +26 -28
- package/python/usearch/eval.py +73 -23
- package/python/usearch/index.py +520 -228
- package/python/usearch/io.py +57 -25
- package/python/usearch/numba.py +4 -4
- package/python/usearch/server.py +10 -10
- package/rust/lib.cpp +22 -27
- package/rust/lib.hpp +11 -12
- package/rust/lib.rs +3 -3
- package/setup.py +5 -2
- package/simsimd/.github/workflows/prerelease.yml +73 -0
- package/simsimd/.github/workflows/release.yml +110 -0
- package/simsimd/.github/workflows/update_version.sh +3 -0
- package/simsimd/.releaserc +77 -0
- package/simsimd/.vscode/launch.json +27 -1
- package/simsimd/.vscode/settings.json +22 -1
- package/simsimd/.vscode/tasks.json +6 -0
- package/simsimd/CMakeLists.txt +3 -2
- package/simsimd/README.md +82 -46
- package/simsimd/bench.cxx +63 -24
- package/simsimd/include/simsimd/simsimd.h +88 -66
- package/simsimd/include/simsimd/simsimd_chem.h +126 -0
- package/simsimd/package.json +8 -0
- package/simsimd/pyproject.toml +42 -0
- package/simsimd/python/python.c +81 -0
- package/simsimd/python/run.sh +1 -0
- package/simsimd/python/test.c +109 -0
- package/simsimd/python/test.py +23 -0
- package/simsimd/setup.py +44 -0
- package/swift/Index+Sugar.swift +75 -23
- package/swift/README.md +26 -2
- package/swift/Test.swift +19 -7
- package/swift/USearch.swift +2 -2
- package/tsconfig.json +8 -0
- package/wasm/CMakeLists.txt +1 -0
- package/wasmer.toml +5 -1
- package/wolfram/CMakeLists.txt +1 -0
- package/wolfram/lib.cpp +11 -13
- package/golang/usearch.h +0 -9
- package/golang/usearch_par_test.go +0 -40
- package/golang/usearch_test.go +0 -82
- package/include/usearch/index_punned_dense.hpp +0 -871
- package/include/usearch/index_punned_helpers.hpp +0 -566
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file index.hpp
|
|
3
|
-
* @author
|
|
3
|
+
* @author Ash Vardanian
|
|
4
4
|
* @brief Single-header Vector Search.
|
|
5
5
|
* @date 2023-04-26
|
|
6
6
|
*
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
#define USEARCH_VERSION_PATCH 0
|
|
15
15
|
|
|
16
16
|
// Inferring C++ version
|
|
17
|
+
// https://stackoverflow.com/a/61552074
|
|
17
18
|
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
18
19
|
#define USEARCH_DEFINED_CPP17
|
|
19
20
|
#endif
|
|
@@ -42,10 +43,13 @@
|
|
|
42
43
|
#endif
|
|
43
44
|
|
|
44
45
|
// Inferring hardware bitness: 32 vs 64
|
|
45
|
-
|
|
46
|
+
// https://stackoverflow.com/a/5273354
|
|
47
|
+
#if INTPTR_MAX == INT64_MAX
|
|
46
48
|
#define USEARCH_64BIT_ENV
|
|
47
|
-
#
|
|
49
|
+
#elif INTPTR_MAX == INT32_MAX
|
|
48
50
|
#define USEARCH_32BIT_ENV
|
|
51
|
+
#else
|
|
52
|
+
#error Unknown pointer size or missing size macros!
|
|
49
53
|
#endif
|
|
50
54
|
|
|
51
55
|
#if !defined(USEARCH_USE_OPENMP)
|
|
@@ -119,51 +123,7 @@
|
|
|
119
123
|
namespace unum {
|
|
120
124
|
namespace usearch {
|
|
121
125
|
|
|
122
|
-
using f64_t = double;
|
|
123
|
-
using f32_t = float;
|
|
124
126
|
using byte_t = char;
|
|
125
|
-
enum b1x8_t : unsigned char {};
|
|
126
|
-
|
|
127
|
-
enum class metric_kind_t : std::uint8_t {
|
|
128
|
-
unknown_k = 0,
|
|
129
|
-
// Classics:
|
|
130
|
-
ip_k = 'i',
|
|
131
|
-
cos_k = 'c',
|
|
132
|
-
l2sq_k = 'e',
|
|
133
|
-
|
|
134
|
-
// Custom:
|
|
135
|
-
pearson_k = 'p',
|
|
136
|
-
haversine_k = 'h',
|
|
137
|
-
|
|
138
|
-
// Sets:
|
|
139
|
-
jaccard_k = 'j',
|
|
140
|
-
hamming_k = 'b',
|
|
141
|
-
tanimoto_k = 't',
|
|
142
|
-
sorensen_k = 's',
|
|
143
|
-
};
|
|
144
|
-
|
|
145
|
-
enum class scalar_kind_t : std::uint8_t {
|
|
146
|
-
unknown_k = 0,
|
|
147
|
-
f64_k,
|
|
148
|
-
f32_k,
|
|
149
|
-
f16_k,
|
|
150
|
-
f8_k,
|
|
151
|
-
b1x8_k,
|
|
152
|
-
};
|
|
153
|
-
|
|
154
|
-
template <typename scalar_at> scalar_kind_t common_scalar_kind() noexcept {
|
|
155
|
-
if (std::is_same<scalar_at, f32_t>())
|
|
156
|
-
return scalar_kind_t::f32_k;
|
|
157
|
-
if (std::is_same<scalar_at, f64_t>())
|
|
158
|
-
return scalar_kind_t::f64_k;
|
|
159
|
-
if (std::is_same<scalar_at, b1x8_t>())
|
|
160
|
-
return scalar_kind_t::b1x8_k;
|
|
161
|
-
return scalar_kind_t::unknown_k;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
|
|
165
|
-
|
|
166
|
-
template <typename at> at square(at value) noexcept { return value * value; }
|
|
167
127
|
|
|
168
128
|
template <std::size_t multiple_ak> std::size_t divide_round_up(std::size_t num) noexcept {
|
|
169
129
|
return (num + multiple_ak - 1) / multiple_ak;
|
|
@@ -187,6 +147,7 @@ inline std::size_t ceil2(std::size_t v) noexcept {
|
|
|
187
147
|
return v;
|
|
188
148
|
}
|
|
189
149
|
|
|
150
|
+
/// @brief Simply dereferencing misaligned pointers can be dangerous.
|
|
190
151
|
template <typename at> void misaligned_store(void* ptr, at v) noexcept {
|
|
191
152
|
static_assert(!std::is_reference<at>::value, "Can't store a reference");
|
|
192
153
|
std::memcpy(ptr, &v, sizeof(at));
|
|
@@ -207,54 +168,80 @@ template <typename at, typename other_at = at> at exchange(at& obj, other_at&& n
|
|
|
207
168
|
return old_value;
|
|
208
169
|
}
|
|
209
170
|
|
|
171
|
+
/// @brief The `std::destroy_at` alternative for C++11.
|
|
172
|
+
template <typename at, typename sfinae_at = at>
|
|
173
|
+
typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
|
|
174
|
+
template <typename at, typename sfinae_at = at>
|
|
175
|
+
typename std::enable_if<!std::is_pod<sfinae_at>::value>::type destroy_at(at* obj) {
|
|
176
|
+
obj->~sfinae_at();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/// @brief The `std::construct_at` alternative for C++11.
|
|
180
|
+
template <typename at, typename sfinae_at = at>
|
|
181
|
+
typename std::enable_if<std::is_pod<sfinae_at>::value>::type construct_at(at*) {}
|
|
182
|
+
template <typename at, typename sfinae_at = at>
|
|
183
|
+
typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* obj) {
|
|
184
|
+
new (obj) at();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* @brief A reference to a misaligned memory location with a specific type.
|
|
189
|
+
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
190
|
+
* indivisible by `sizeof(at)`.
|
|
191
|
+
*/
|
|
210
192
|
template <typename at> class misaligned_ref_gt {
|
|
193
|
+
using element_t = at;
|
|
194
|
+
using mutable_t = typename std::remove_const<element_t>::type;
|
|
211
195
|
byte_t* ptr_;
|
|
212
196
|
|
|
213
197
|
public:
|
|
214
198
|
misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
215
|
-
operator
|
|
216
|
-
misaligned_ref_gt& operator=(
|
|
217
|
-
misaligned_store<
|
|
199
|
+
operator mutable_t() const noexcept { return misaligned_load<mutable_t>(ptr_); }
|
|
200
|
+
misaligned_ref_gt& operator=(mutable_t const& v) noexcept {
|
|
201
|
+
misaligned_store<mutable_t>(ptr_, v);
|
|
218
202
|
return *this;
|
|
219
203
|
}
|
|
220
204
|
|
|
221
205
|
void reset(byte_t* ptr) noexcept { ptr_ = ptr; }
|
|
206
|
+
byte_t* ptr() const noexcept { return ptr_; }
|
|
222
207
|
};
|
|
223
208
|
|
|
209
|
+
/**
|
|
210
|
+
* @brief A pointer to a misaligned memory location with a specific type.
|
|
211
|
+
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
212
|
+
* indivisible by `sizeof(at)`.
|
|
213
|
+
*/
|
|
224
214
|
template <typename at> class misaligned_ptr_gt {
|
|
215
|
+
using element_t = at;
|
|
216
|
+
using mutable_t = typename std::remove_const<element_t>::type;
|
|
225
217
|
byte_t* ptr_;
|
|
226
218
|
|
|
227
219
|
public:
|
|
228
220
|
using iterator_category = std::random_access_iterator_tag;
|
|
229
|
-
using value_type =
|
|
221
|
+
using value_type = element_t;
|
|
230
222
|
using difference_type = std::ptrdiff_t;
|
|
231
|
-
using pointer = misaligned_ptr_gt<
|
|
232
|
-
using reference = misaligned_ref_gt<
|
|
223
|
+
using pointer = misaligned_ptr_gt<element_t>;
|
|
224
|
+
using reference = misaligned_ref_gt<element_t>;
|
|
233
225
|
|
|
234
226
|
reference operator*() const noexcept { return {ptr_}; }
|
|
227
|
+
reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
|
|
228
|
+
value_type operator[](std::size_t i) const noexcept {
|
|
229
|
+
return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
|
|
230
|
+
}
|
|
235
231
|
|
|
236
232
|
misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
237
|
-
misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(
|
|
238
|
-
misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(
|
|
239
|
-
misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(
|
|
240
|
-
misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(
|
|
233
|
+
misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(element_t)); }
|
|
234
|
+
misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(element_t)); }
|
|
235
|
+
misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(element_t)); }
|
|
236
|
+
misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(element_t)); }
|
|
237
|
+
|
|
238
|
+
// clang-format off
|
|
239
|
+
misaligned_ptr_gt& operator++() noexcept { ptr_ += sizeof(element_t); return *this; }
|
|
240
|
+
misaligned_ptr_gt& operator--() noexcept { ptr_ -= sizeof(element_t); return *this; }
|
|
241
|
+
misaligned_ptr_gt& operator+=(difference_type d) noexcept { ptr_ += d * sizeof(element_t); return *this; }
|
|
242
|
+
misaligned_ptr_gt& operator-=(difference_type d) noexcept { ptr_ -= d * sizeof(element_t); return *this; }
|
|
243
|
+
// clang-format on
|
|
241
244
|
|
|
242
|
-
misaligned_ptr_gt& operator++() noexcept {
|
|
243
|
-
ptr_ += sizeof(at);
|
|
244
|
-
return *this;
|
|
245
|
-
}
|
|
246
|
-
misaligned_ptr_gt& operator--() noexcept {
|
|
247
|
-
ptr_ -= sizeof(at);
|
|
248
|
-
return *this;
|
|
249
|
-
}
|
|
250
|
-
misaligned_ptr_gt& operator+=(difference_type d) noexcept {
|
|
251
|
-
ptr_ += d * sizeof(at);
|
|
252
|
-
return *this;
|
|
253
|
-
}
|
|
254
|
-
misaligned_ptr_gt& operator-=(difference_type d) noexcept {
|
|
255
|
-
ptr_ -= d * sizeof(at);
|
|
256
|
-
return *this;
|
|
257
|
-
}
|
|
258
245
|
bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
|
|
259
246
|
bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
|
|
260
247
|
};
|
|
@@ -278,307 +265,56 @@ template <typename scalar_at> class span_gt {
|
|
|
278
265
|
};
|
|
279
266
|
|
|
280
267
|
/**
|
|
281
|
-
* @brief
|
|
282
|
-
|
|
283
|
-
template <typename scalar_at = float, typename result_at = scalar_at> struct ip_gt {
|
|
284
|
-
using scalar_t = scalar_at;
|
|
285
|
-
using view_t = span_gt<scalar_t const>;
|
|
286
|
-
using result_t = result_at;
|
|
287
|
-
using result_type = result_t;
|
|
288
|
-
|
|
289
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::ip_k; }
|
|
290
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
291
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
292
|
-
|
|
293
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
294
|
-
result_type ab{};
|
|
295
|
-
#if USEARCH_USE_OPENMP
|
|
296
|
-
#pragma omp simd reduction(+ : ab)
|
|
297
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
298
|
-
#pragma clang loop vectorize(enable)
|
|
299
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
300
|
-
#pragma GCC ivdep
|
|
301
|
-
#endif
|
|
302
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
303
|
-
ab += result_t(a[i]) * result_t(b[i]);
|
|
304
|
-
return 1 - ab;
|
|
305
|
-
}
|
|
306
|
-
};
|
|
307
|
-
|
|
308
|
-
/**
|
|
309
|
-
* @brief Cosine (Angular) distance.
|
|
310
|
-
* Identical to the Inner Product of normalized vectors.
|
|
311
|
-
* Unless you are running on an tiny embedded platform, this metric
|
|
312
|
-
* is recommended over `::ip_gt` for low-precision scalars.
|
|
313
|
-
*/
|
|
314
|
-
template <typename scalar_at = float, typename result_at = scalar_at> struct cos_gt {
|
|
315
|
-
using scalar_t = scalar_at;
|
|
316
|
-
using view_t = span_gt<scalar_t const>;
|
|
317
|
-
using result_t = result_at;
|
|
318
|
-
using result_type = result_t;
|
|
319
|
-
|
|
320
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::cos_k; }
|
|
321
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
322
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
323
|
-
|
|
324
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
325
|
-
result_t ab{}, a2{}, b2{};
|
|
326
|
-
#if USEARCH_USE_OPENMP
|
|
327
|
-
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
328
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
329
|
-
#pragma clang loop vectorize(enable)
|
|
330
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
331
|
-
#pragma GCC ivdep
|
|
332
|
-
#endif
|
|
333
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
334
|
-
ab += result_t(a[i]) * result_t(b[i]), //
|
|
335
|
-
a2 += square<result_t>(a[i]), //
|
|
336
|
-
b2 += square<result_t>(b[i]);
|
|
337
|
-
return (ab != 0) ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 1;
|
|
338
|
-
}
|
|
339
|
-
};
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* @brief Squared Euclidean (L2) distance.
|
|
343
|
-
* Square root is avoided at the end, as it won't affect the ordering.
|
|
268
|
+
* @brief Similar to `std::vector`, but doesn't support dynamic resizing.
|
|
269
|
+
* On the bright side, this can't throw exceptions.
|
|
344
270
|
*/
|
|
345
|
-
template <typename scalar_at
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
using result_t = result_at;
|
|
349
|
-
using result_type = result_t;
|
|
350
|
-
|
|
351
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::l2sq_k; }
|
|
352
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
353
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
354
|
-
|
|
355
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
356
|
-
result_t ab_deltas_sq{};
|
|
357
|
-
#if USEARCH_USE_OPENMP
|
|
358
|
-
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
359
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
360
|
-
#pragma clang loop vectorize(enable)
|
|
361
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
362
|
-
#pragma GCC ivdep
|
|
363
|
-
#endif
|
|
364
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
365
|
-
ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
|
|
366
|
-
return ab_deltas_sq;
|
|
367
|
-
}
|
|
368
|
-
};
|
|
369
|
-
|
|
370
|
-
/**
|
|
371
|
-
* @brief Hamming distance computes the number of differing bits in
|
|
372
|
-
* two arrays of integers. An example would be a textual document,
|
|
373
|
-
* tokenized and hashed into a fixed-capacity bitset.
|
|
374
|
-
*/
|
|
375
|
-
template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct hamming_gt {
|
|
376
|
-
using scalar_t = scalar_at;
|
|
377
|
-
using view_t = span_gt<scalar_t const>;
|
|
378
|
-
using result_t = result_at;
|
|
379
|
-
using result_type = result_t;
|
|
380
|
-
static_assert( //
|
|
381
|
-
std::is_unsigned<scalar_t>::value ||
|
|
382
|
-
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
383
|
-
"Hamming distance requires unsigned integral words");
|
|
384
|
-
|
|
385
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::hamming_k; }
|
|
386
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
387
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
388
|
-
|
|
389
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
390
|
-
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
391
|
-
result_t matches{};
|
|
392
|
-
#if USEARCH_USE_OPENMP
|
|
393
|
-
#pragma omp simd reduction(+ : matches)
|
|
394
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
395
|
-
#pragma clang loop vectorize(enable)
|
|
396
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
397
|
-
#pragma GCC ivdep
|
|
398
|
-
#endif
|
|
399
|
-
for (std::size_t i = 0; i != words; ++i)
|
|
400
|
-
matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
|
|
401
|
-
return matches;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
271
|
+
template <typename scalar_at, typename allocator_at> class buffer_gt {
|
|
272
|
+
scalar_at* data_;
|
|
273
|
+
std::size_t size_;
|
|
404
274
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::tanimoto_k; }
|
|
421
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
422
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
423
|
-
|
|
424
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
425
|
-
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
426
|
-
result_t and_count{};
|
|
427
|
-
result_t or_count{};
|
|
428
|
-
#if USEARCH_USE_OPENMP
|
|
429
|
-
#pragma omp simd reduction(+ : and_count, or_count)
|
|
430
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
431
|
-
#pragma clang loop vectorize(enable)
|
|
432
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
433
|
-
#pragma GCC ivdep
|
|
434
|
-
#endif
|
|
435
|
-
for (std::size_t i = 0; i != words; ++i)
|
|
436
|
-
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
437
|
-
or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
|
|
438
|
-
return 1 - result_t(and_count) / or_count;
|
|
275
|
+
public:
|
|
276
|
+
buffer_gt() noexcept : data_(nullptr), size_(0u) {}
|
|
277
|
+
buffer_gt(std::size_t size) noexcept : data_(allocator_at{}.allocate(size)), size_(data_ ? size : 0u) {
|
|
278
|
+
if (!std::is_trivially_default_constructible<scalar_at>::value)
|
|
279
|
+
for (std::size_t i = 0; i != size_; ++i)
|
|
280
|
+
construct_at(data_ + i);
|
|
281
|
+
}
|
|
282
|
+
~buffer_gt() noexcept {
|
|
283
|
+
if (!std::is_trivially_destructible<scalar_at>::value)
|
|
284
|
+
for (std::size_t i = 0; i != size_; ++i)
|
|
285
|
+
destroy_at(data_ + i);
|
|
286
|
+
allocator_at{}.deallocate(data_, size_);
|
|
287
|
+
data_ = nullptr;
|
|
288
|
+
size_ = 0;
|
|
439
289
|
}
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
*
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
static_assert( //
|
|
452
|
-
std::is_unsigned<scalar_t>::value ||
|
|
453
|
-
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
454
|
-
"Sorensen-Dice distance requires unsigned integral words");
|
|
455
|
-
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
456
|
-
|
|
457
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::sorensen_k; }
|
|
458
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
459
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
460
|
-
|
|
461
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
462
|
-
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
463
|
-
result_t and_count{};
|
|
464
|
-
result_t any_count{};
|
|
465
|
-
#if USEARCH_USE_OPENMP
|
|
466
|
-
#pragma omp simd reduction(+ : and_count, any_count)
|
|
467
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
468
|
-
#pragma clang loop vectorize(enable)
|
|
469
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
470
|
-
#pragma GCC ivdep
|
|
471
|
-
#endif
|
|
472
|
-
for (std::size_t i = 0; i != words; ++i)
|
|
473
|
-
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
474
|
-
any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
|
|
475
|
-
return 1 - 2 * result_t(and_count) / any_count;
|
|
290
|
+
scalar_at* data() const noexcept { return data_; }
|
|
291
|
+
std::size_t size() const noexcept { return size_; }
|
|
292
|
+
scalar_at* begin() const noexcept { return data_; }
|
|
293
|
+
scalar_at* end() const noexcept { return data_ + size_; }
|
|
294
|
+
operator scalar_at*() const noexcept { return data(); }
|
|
295
|
+
scalar_at& operator[](std::size_t i) noexcept { return data_[i]; }
|
|
296
|
+
scalar_at const& operator[](std::size_t i) const noexcept { return data_[i]; }
|
|
297
|
+
explicit operator bool() const noexcept { return data_; }
|
|
298
|
+
scalar_at* release() noexcept {
|
|
299
|
+
size_ = 0;
|
|
300
|
+
return exchange(data_, nullptr);
|
|
476
301
|
}
|
|
477
|
-
};
|
|
478
302
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
* Can be used to compute the similarity between two textual documents
|
|
482
|
-
* using the IDs of tokens present in them.
|
|
483
|
-
* Similar to `tanimoto_gt` for dense representations.
|
|
484
|
-
*/
|
|
485
|
-
template <typename scalar_at = std::int32_t, typename result_at = float> struct jaccard_gt {
|
|
486
|
-
using scalar_t = scalar_at;
|
|
487
|
-
using view_t = span_gt<scalar_t const>;
|
|
488
|
-
using result_t = result_at;
|
|
489
|
-
using result_type = result_t;
|
|
490
|
-
static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
|
|
491
|
-
|
|
492
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::jaccard_k; }
|
|
493
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
494
|
-
inline result_t operator()(view_t a, view_t b) const noexcept {
|
|
495
|
-
return operator()(a.data(), b.data(), a.size(), b.size());
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
inline result_t operator()( //
|
|
499
|
-
scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
|
|
500
|
-
result_t intersection{};
|
|
501
|
-
std::size_t i{};
|
|
502
|
-
std::size_t j{};
|
|
503
|
-
while (i != a_length && j != b_length) {
|
|
504
|
-
intersection += a[i] == b[j];
|
|
505
|
-
i += a[i] < b[j];
|
|
506
|
-
j += a[i] >= b[j];
|
|
507
|
-
}
|
|
508
|
-
return 1 - intersection / (a_length + b_length - intersection);
|
|
509
|
-
}
|
|
510
|
-
};
|
|
303
|
+
buffer_gt(buffer_gt const&) = delete;
|
|
304
|
+
buffer_gt& operator=(buffer_gt const&) = delete;
|
|
511
305
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
template <typename scalar_at = float, typename result_at = float> struct pearson_correlation_gt {
|
|
518
|
-
using scalar_t = scalar_at;
|
|
519
|
-
using view_t = span_gt<scalar_t const>;
|
|
520
|
-
using result_t = result_at;
|
|
521
|
-
using result_type = result_t;
|
|
522
|
-
|
|
523
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::pearson_k; }
|
|
524
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
525
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
526
|
-
|
|
527
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
528
|
-
result_t a_sum{}, b_sum{}, ab_sum{};
|
|
529
|
-
result_t a_sq_sum{}, b_sq_sum{};
|
|
530
|
-
#if USEARCH_USE_OPENMP
|
|
531
|
-
#pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
|
|
532
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
533
|
-
#pragma clang loop vectorize(enable)
|
|
534
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
535
|
-
#pragma GCC ivdep
|
|
536
|
-
#endif
|
|
537
|
-
for (std::size_t i = 0; i != dim; ++i) {
|
|
538
|
-
a_sum += result_t(a[i]);
|
|
539
|
-
b_sum += result_t(b[i]);
|
|
540
|
-
ab_sum += result_t(a[i]) * result_t(b[i]);
|
|
541
|
-
a_sq_sum += result_t(a[i]) * result_t(a[i]);
|
|
542
|
-
b_sq_sum += result_t(b[i]) * result_t(b[i]);
|
|
543
|
-
}
|
|
544
|
-
result_t denom = std::sqrt((dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum));
|
|
545
|
-
result_t corr = (dim * ab_sum - a_sum * b_sum) / denom;
|
|
546
|
-
return -corr;
|
|
306
|
+
buffer_gt(buffer_gt&& other) noexcept : data_(exchange(other.data_, nullptr)), size_(exchange(other.size_, 0)) {}
|
|
307
|
+
buffer_gt& operator=(buffer_gt&& other) noexcept {
|
|
308
|
+
std::swap(data_, other.data_);
|
|
309
|
+
std::swap(size_, other.size_);
|
|
310
|
+
return *this;
|
|
547
311
|
}
|
|
548
312
|
};
|
|
549
313
|
|
|
550
314
|
/**
|
|
551
|
-
* @brief
|
|
552
|
-
*
|
|
315
|
+
* @brief A lightweight error class for handling error messages,
|
|
316
|
+
* which are expected to be allocated in static memory.
|
|
553
317
|
*/
|
|
554
|
-
template <typename scalar_at = float, typename result_at = scalar_at> struct haversine_gt {
|
|
555
|
-
using scalar_t = scalar_at;
|
|
556
|
-
using view_t = span_gt<scalar_t const>;
|
|
557
|
-
using result_t = result_at;
|
|
558
|
-
using result_type = result_t;
|
|
559
|
-
static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
|
|
560
|
-
|
|
561
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::haversine_k; }
|
|
562
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
563
|
-
|
|
564
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b) const noexcept {
|
|
565
|
-
result_t lat_a = a[0], lon_a = a[1];
|
|
566
|
-
result_t lat_b = b[0], lon_b = b[1];
|
|
567
|
-
|
|
568
|
-
result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a);
|
|
569
|
-
result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a);
|
|
570
|
-
|
|
571
|
-
result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
|
|
572
|
-
result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
|
|
573
|
-
|
|
574
|
-
result_t x = //
|
|
575
|
-
square(std::sin(lat_delta / 2.f)) +
|
|
576
|
-
std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta / 2.f));
|
|
577
|
-
|
|
578
|
-
return std::atan2(std::sqrt(x), std::sqrt(1.f - x));
|
|
579
|
-
}
|
|
580
|
-
};
|
|
581
|
-
|
|
582
318
|
class error_t {
|
|
583
319
|
char const* message_{};
|
|
584
320
|
|
|
@@ -596,11 +332,14 @@ class error_t {
|
|
|
596
332
|
explicit operator bool() const noexcept { return message_ != nullptr; }
|
|
597
333
|
char const* what() const noexcept { return message_; }
|
|
598
334
|
|
|
599
|
-
#if defined(__cpp_exceptions)
|
|
335
|
+
#if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
|
|
600
336
|
~error_t() noexcept(false) {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
337
|
+
#if defined(USEARCH_DEFINED_CPP17)
|
|
338
|
+
if (message_ && std::uncaught_exceptions() == 0)
|
|
339
|
+
#else
|
|
340
|
+
if (message_ && std::uncaught_exception() == 0)
|
|
341
|
+
#endif
|
|
342
|
+
raise();
|
|
604
343
|
}
|
|
605
344
|
void raise() noexcept(false) {
|
|
606
345
|
if (message_)
|
|
@@ -615,6 +354,13 @@ class error_t {
|
|
|
615
354
|
#endif
|
|
616
355
|
};
|
|
617
356
|
|
|
357
|
+
/**
|
|
358
|
+
* @brief Similar to `std::expected` in C++23, wraps a statement evaluation result,
|
|
359
|
+
* or an error. It's used to avoid raising exception, and gracefully propagate
|
|
360
|
+
* the error.
|
|
361
|
+
*
|
|
362
|
+
* @tparam result_at The type of the expected result.
|
|
363
|
+
*/
|
|
618
364
|
template <typename result_at> struct expected_gt {
|
|
619
365
|
result_at result;
|
|
620
366
|
error_t error;
|
|
@@ -637,49 +383,41 @@ template <typename result_at> struct expected_gt {
|
|
|
637
383
|
|
|
638
384
|
/**
|
|
639
385
|
* @brief Light-weight bitset implementation to track visited nodes during graph traversal.
|
|
640
|
-
* Extends basic functionality with atomic operations.
|
|
386
|
+
* Extends basic functionality with @b atomic operations.
|
|
641
387
|
*/
|
|
642
|
-
template <typename allocator_at = std::allocator<
|
|
388
|
+
template <typename allocator_at = std::allocator<byte_t>> class visits_bitset_gt {
|
|
643
389
|
using allocator_t = allocator_at;
|
|
644
390
|
using byte_t = typename allocator_t::value_type;
|
|
645
391
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
646
392
|
|
|
647
|
-
using
|
|
393
|
+
using compressed_slot_t = unsigned long;
|
|
648
394
|
|
|
649
|
-
static constexpr std::size_t bits_per_slot() { return sizeof(
|
|
650
|
-
static constexpr
|
|
395
|
+
static constexpr std::size_t bits_per_slot() { return sizeof(compressed_slot_t) * CHAR_BIT; }
|
|
396
|
+
static constexpr compressed_slot_t bits_mask() { return sizeof(compressed_slot_t) * CHAR_BIT - 1; }
|
|
397
|
+
static constexpr std::size_t slots(std::size_t bits) { return divide_round_up<bits_per_slot()>(bits); }
|
|
651
398
|
|
|
652
|
-
|
|
399
|
+
compressed_slot_t* slots_{};
|
|
653
400
|
/// @brief Number of slots.
|
|
654
401
|
std::size_t count_{};
|
|
655
402
|
|
|
656
403
|
public:
|
|
657
404
|
visits_bitset_gt() noexcept {}
|
|
658
405
|
~visits_bitset_gt() noexcept { reset(); }
|
|
659
|
-
|
|
406
|
+
|
|
407
|
+
explicit operator bool() const noexcept { return slots_; }
|
|
408
|
+
void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(compressed_slot_t)); }
|
|
660
409
|
|
|
661
410
|
void reset() noexcept {
|
|
662
411
|
if (slots_)
|
|
663
|
-
allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(
|
|
412
|
+
allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(compressed_slot_t));
|
|
664
413
|
slots_ = nullptr;
|
|
665
414
|
count_ = 0;
|
|
666
415
|
}
|
|
667
416
|
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
if (count <= count_)
|
|
672
|
-
return true;
|
|
673
|
-
|
|
674
|
-
slot_t* slots = (slot_t*)allocator_t{}.allocate(count * sizeof(slot_t));
|
|
675
|
-
if (!slots)
|
|
676
|
-
return false;
|
|
677
|
-
|
|
678
|
-
reset();
|
|
679
|
-
count_ = count;
|
|
680
|
-
slots_ = slots;
|
|
417
|
+
visits_bitset_gt(std::size_t capacity) noexcept
|
|
418
|
+
: slots_((compressed_slot_t*)allocator_t{}.allocate(slots(capacity) * sizeof(compressed_slot_t))),
|
|
419
|
+
count_(slots_ ? slots(capacity) : 0u) {
|
|
681
420
|
clear();
|
|
682
|
-
return true;
|
|
683
421
|
}
|
|
684
422
|
|
|
685
423
|
visits_bitset_gt(visits_bitset_gt&& other) noexcept {
|
|
@@ -697,33 +435,54 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
|
|
|
697
435
|
visits_bitset_gt& operator=(visits_bitset_gt const&) = delete;
|
|
698
436
|
|
|
699
437
|
inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
|
|
700
|
-
inline
|
|
438
|
+
inline bool set(std::size_t i) noexcept {
|
|
439
|
+
compressed_slot_t& slot = slots_[i / bits_per_slot()];
|
|
440
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
441
|
+
bool value = slot & mask;
|
|
442
|
+
slot |= mask;
|
|
443
|
+
return value;
|
|
444
|
+
}
|
|
701
445
|
|
|
702
446
|
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
703
447
|
|
|
704
448
|
inline bool atomic_set(std::size_t i) noexcept {
|
|
705
|
-
|
|
449
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
706
450
|
return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
|
|
707
451
|
}
|
|
708
452
|
|
|
709
453
|
inline void atomic_reset(std::size_t i) noexcept {
|
|
710
|
-
|
|
454
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
711
455
|
InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
|
|
712
456
|
}
|
|
713
457
|
|
|
714
458
|
#else
|
|
715
459
|
|
|
716
460
|
inline bool atomic_set(std::size_t i) noexcept {
|
|
717
|
-
|
|
461
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
718
462
|
return __atomic_fetch_or(&slots_[i / bits_per_slot()], mask, __ATOMIC_ACQUIRE) & mask;
|
|
719
463
|
}
|
|
720
464
|
|
|
721
465
|
inline void atomic_reset(std::size_t i) noexcept {
|
|
722
|
-
|
|
466
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
723
467
|
__atomic_fetch_and(&slots_[i / bits_per_slot()], ~mask, __ATOMIC_RELEASE);
|
|
724
468
|
}
|
|
725
469
|
|
|
726
470
|
#endif
|
|
471
|
+
|
|
472
|
+
class lock_t {
|
|
473
|
+
visits_bitset_gt& bitset_;
|
|
474
|
+
std::size_t bit_offset_;
|
|
475
|
+
|
|
476
|
+
public:
|
|
477
|
+
inline ~lock_t() noexcept { bitset_.atomic_reset(bit_offset_); }
|
|
478
|
+
inline lock_t(visits_bitset_gt& bitset, std::size_t bit_offset) noexcept
|
|
479
|
+
: bitset_(bitset), bit_offset_(bit_offset) {
|
|
480
|
+
while (bitset_.atomic_set(bit_offset_))
|
|
481
|
+
;
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
inline lock_t lock(std::size_t i) noexcept { return {*this, i}; }
|
|
727
486
|
};
|
|
728
487
|
|
|
729
488
|
using visits_bitset_t = visits_bitset_gt<>;
|
|
@@ -1006,10 +765,10 @@ class usearch_pack_m uint40_t {
|
|
|
1006
765
|
|
|
1007
766
|
public:
|
|
1008
767
|
inline uint40_t() noexcept { broadcast(0); }
|
|
1009
|
-
inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets,
|
|
1010
|
-
|
|
1011
|
-
#
|
|
1012
|
-
inline uint40_t(std::
|
|
768
|
+
inline uint40_t(std::uint32_t n) noexcept { std::memcpy(&octets[1], &n, 4); }
|
|
769
|
+
|
|
770
|
+
#ifdef USEARCH_64BIT_ENV
|
|
771
|
+
inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
|
|
1013
772
|
#endif
|
|
1014
773
|
|
|
1015
774
|
uint40_t(uint40_t&&) = default;
|
|
@@ -1017,47 +776,26 @@ class usearch_pack_m uint40_t {
|
|
|
1017
776
|
uint40_t& operator=(uint40_t&&) = default;
|
|
1018
777
|
uint40_t& operator=(uint40_t const&) = default;
|
|
1019
778
|
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
inline uint40_t& operator+=(std::size_t n) noexcept {
|
|
1028
|
-
unsigned char* n_octets = reinterpret_cast<unsigned char*>(&n);
|
|
1029
|
-
std::uint32_t& n_tail = *reinterpret_cast<std::uint32_t*>(n_octets);
|
|
1030
|
-
std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
|
|
1031
|
-
octets[4] += static_cast<unsigned char>((tail + n_tail) < tail);
|
|
1032
|
-
tail += n_tail;
|
|
1033
|
-
octets[4] += n_octets[4];
|
|
1034
|
-
return *this;
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
inline uint40_t operator+(std::size_t n) noexcept {
|
|
1038
|
-
uint40_t other(*this);
|
|
1039
|
-
other += n;
|
|
1040
|
-
return other;
|
|
779
|
+
#if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
|
|
780
|
+
inline uint40_t(std::size_t n) noexcept {
|
|
781
|
+
#ifdef USEARCH_64BIT_ENV
|
|
782
|
+
std::memcpy(octets, &n, 5);
|
|
783
|
+
#else
|
|
784
|
+
std::memcpy(octets, &n, 4);
|
|
785
|
+
#endif
|
|
1041
786
|
}
|
|
787
|
+
#endif
|
|
1042
788
|
|
|
1043
789
|
inline operator std::size_t() const noexcept {
|
|
1044
790
|
std::size_t result = 0;
|
|
1045
791
|
#ifdef USEARCH_64BIT_ENV
|
|
1046
|
-
std::memcpy(
|
|
792
|
+
std::memcpy(&result, octets, 5);
|
|
1047
793
|
#else
|
|
1048
|
-
std::memcpy(
|
|
794
|
+
std::memcpy(&result, octets + 1, 4);
|
|
1049
795
|
#endif
|
|
1050
796
|
return result;
|
|
1051
797
|
}
|
|
1052
798
|
|
|
1053
|
-
inline uint40_t& operator++() noexcept { return *this += 1u; }
|
|
1054
|
-
|
|
1055
|
-
inline uint40_t operator++(int) noexcept {
|
|
1056
|
-
uint40_t old = *this;
|
|
1057
|
-
*this += 1u;
|
|
1058
|
-
return old;
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
799
|
inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
|
|
1062
800
|
inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
|
|
1063
801
|
};
|
|
@@ -1068,6 +806,9 @@ class usearch_pack_m uint40_t {
|
|
|
1068
806
|
|
|
1069
807
|
static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
|
|
1070
808
|
|
|
809
|
+
/**
|
|
810
|
+
* @brief Basic single-threaded @b ring class, used for all kinds of task queues.
|
|
811
|
+
*/
|
|
1071
812
|
template <typename element_at, typename allocator_at = std::allocator<element_at>> //
|
|
1072
813
|
class ring_gt {
|
|
1073
814
|
public:
|
|
@@ -1202,12 +943,6 @@ constexpr std::size_t default_expansion_search() { return 64; }
|
|
|
1202
943
|
|
|
1203
944
|
constexpr std::size_t default_allocator_entry_bytes() { return 64; }
|
|
1204
945
|
|
|
1205
|
-
/**
|
|
1206
|
-
* @brief The "magic" sequence helps infer the type of the file.
|
|
1207
|
-
* USearch indexes start with the "usearch" string.
|
|
1208
|
-
*/
|
|
1209
|
-
constexpr char const* default_magic() { return "usearch"; }
|
|
1210
|
-
|
|
1211
946
|
/**
|
|
1212
947
|
* @brief Configuration settings for the index construction.
|
|
1213
948
|
* Includes the main `::connectivity` parameter (`M` in the paper)
|
|
@@ -1219,14 +954,16 @@ struct index_config_t {
|
|
|
1219
954
|
/// > It is called `M` in the paper.
|
|
1220
955
|
std::size_t connectivity = default_connectivity();
|
|
1221
956
|
|
|
1222
|
-
/// @brief
|
|
1223
|
-
///
|
|
1224
|
-
///
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
957
|
+
/// @brief Number of neighbors per graph node in base level graph.
|
|
958
|
+
/// Defaults to double of the other levels, so 64 in FAISS and 32 in hnswlib.
|
|
959
|
+
/// > It is called `M0` in the paper.
|
|
960
|
+
std::size_t connectivity_base = default_connectivity() * 2;
|
|
961
|
+
|
|
962
|
+
inline index_config_t() = default;
|
|
963
|
+
inline index_config_t(std::size_t c) noexcept
|
|
964
|
+
: connectivity(c ? c : default_connectivity()), connectivity_base(c ? c * 2 : default_connectivity() * 2) {}
|
|
965
|
+
inline index_config_t(std::size_t c, std::size_t cb) noexcept
|
|
966
|
+
: connectivity(c), connectivity_base((std::max)(c, cb)) {}
|
|
1230
967
|
};
|
|
1231
968
|
|
|
1232
969
|
struct index_limits_t {
|
|
@@ -1234,13 +971,13 @@ struct index_limits_t {
|
|
|
1234
971
|
std::size_t threads_add = std::thread::hardware_concurrency();
|
|
1235
972
|
std::size_t threads_search = std::thread::hardware_concurrency();
|
|
1236
973
|
|
|
1237
|
-
index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
|
|
1238
|
-
index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
1239
|
-
std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
1240
|
-
std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
974
|
+
inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
|
|
975
|
+
inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
976
|
+
inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
977
|
+
inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
1241
978
|
};
|
|
1242
979
|
|
|
1243
|
-
struct
|
|
980
|
+
struct index_update_config_t {
|
|
1244
981
|
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
1245
982
|
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
1246
983
|
/// > It is called `efConstruction` in the paper.
|
|
@@ -1248,12 +985,9 @@ struct add_config_t {
|
|
|
1248
985
|
|
|
1249
986
|
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1250
987
|
std::size_t thread = 0;
|
|
1251
|
-
|
|
1252
|
-
/// @brief Don't copy the ::vector, if it's persisted elsewhere.
|
|
1253
|
-
bool store_vector = true;
|
|
1254
988
|
};
|
|
1255
989
|
|
|
1256
|
-
struct
|
|
990
|
+
struct index_search_config_t {
|
|
1257
991
|
/// @brief Hyper-parameter controlling the quality of search.
|
|
1258
992
|
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1259
993
|
/// > It is called `ef` in the paper.
|
|
@@ -1266,133 +1000,21 @@ struct search_config_t {
|
|
|
1266
1000
|
bool exact = false;
|
|
1267
1001
|
};
|
|
1268
1002
|
|
|
1269
|
-
struct
|
|
1270
|
-
bool copy_vectors = true;
|
|
1271
|
-
};
|
|
1003
|
+
struct index_copy_config_t {};
|
|
1272
1004
|
|
|
1273
|
-
struct
|
|
1005
|
+
struct index_join_config_t {
|
|
1274
1006
|
/// @brief Controls maximum number of proposals per man during stable marriage.
|
|
1275
1007
|
std::size_t max_proposals = 0;
|
|
1008
|
+
|
|
1276
1009
|
/// @brief Hyper-parameter controlling the quality of search.
|
|
1277
1010
|
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1278
1011
|
/// > It is called `ef` in the paper.
|
|
1279
1012
|
std::size_t expansion = default_expansion_search();
|
|
1013
|
+
|
|
1280
1014
|
/// @brief Brute-forces exhaustive search over all entries in the index.
|
|
1281
1015
|
bool exact = false;
|
|
1282
1016
|
};
|
|
1283
1017
|
|
|
1284
|
-
using file_header_t = byte_t[64];
|
|
1285
|
-
|
|
1286
|
-
/**
|
|
1287
|
-
* @brief Serialized binary representations of the USearch index start with metadata.
|
|
1288
|
-
* Metadata is parsed into a `file_head_t`, containing the USearch package version,
|
|
1289
|
-
* and the properties of the index.
|
|
1290
|
-
*
|
|
1291
|
-
* It uses: 13 bytes for file versioning, 22 bytes for structural information = 35 bytes.
|
|
1292
|
-
* The following 24 bytes contain binary size of the graph, of the vectors, and the checksum,
|
|
1293
|
-
* leaving 5 bytes at the end vacant.
|
|
1294
|
-
*/
|
|
1295
|
-
struct file_head_t {
|
|
1296
|
-
|
|
1297
|
-
// Versioning: 7 + 2 * 3 = 13 bytes
|
|
1298
|
-
using magic_t = char[7];
|
|
1299
|
-
using version_major_t = std::uint16_t;
|
|
1300
|
-
using version_minor_t = std::uint16_t;
|
|
1301
|
-
using version_patch_t = std::uint16_t;
|
|
1302
|
-
|
|
1303
|
-
// Structural: 1 * 6 + 8 * 2 = 22 bytes
|
|
1304
|
-
using connectivity_t = std::uint8_t;
|
|
1305
|
-
using max_level_t = std::uint8_t;
|
|
1306
|
-
using vector_alignment_t = std::uint8_t;
|
|
1307
|
-
using bytes_per_label_t = std::uint8_t;
|
|
1308
|
-
using bytes_per_id_t = std::uint8_t;
|
|
1309
|
-
using size_t = std::uint64_t;
|
|
1310
|
-
using entry_idx_t = std::uint64_t;
|
|
1311
|
-
|
|
1312
|
-
// Versioning:
|
|
1313
|
-
char const* magic;
|
|
1314
|
-
misaligned_ref_gt<version_major_t> version_major;
|
|
1315
|
-
misaligned_ref_gt<version_minor_t> version_minor;
|
|
1316
|
-
misaligned_ref_gt<version_patch_t> version_patch;
|
|
1317
|
-
|
|
1318
|
-
// Structural:
|
|
1319
|
-
misaligned_ref_gt<metric_kind_t> metric;
|
|
1320
|
-
misaligned_ref_gt<connectivity_t> connectivity;
|
|
1321
|
-
misaligned_ref_gt<max_level_t> max_level;
|
|
1322
|
-
misaligned_ref_gt<vector_alignment_t> vector_alignment;
|
|
1323
|
-
misaligned_ref_gt<bytes_per_label_t> bytes_per_label;
|
|
1324
|
-
misaligned_ref_gt<bytes_per_id_t> bytes_per_id;
|
|
1325
|
-
misaligned_ref_gt<scalar_kind_t> scalar_kind;
|
|
1326
|
-
misaligned_ref_gt<size_t> size;
|
|
1327
|
-
misaligned_ref_gt<entry_idx_t> entry_idx;
|
|
1328
|
-
|
|
1329
|
-
// Additional:
|
|
1330
|
-
misaligned_ref_gt<size_t> bytes_for_graphs;
|
|
1331
|
-
misaligned_ref_gt<size_t> bytes_for_vectors;
|
|
1332
|
-
misaligned_ref_gt<size_t> bytes_checksum;
|
|
1333
|
-
|
|
1334
|
-
file_head_t(byte_t* ptr) noexcept
|
|
1335
|
-
: magic((char const*)exchange(ptr, ptr + sizeof(magic_t))),
|
|
1336
|
-
version_major(exchange(ptr, ptr + sizeof(version_major_t))),
|
|
1337
|
-
version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
|
|
1338
|
-
version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
|
|
1339
|
-
metric(exchange(ptr, ptr + sizeof(metric_kind_t))), connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
|
|
1340
|
-
max_level(exchange(ptr, ptr + sizeof(max_level_t))),
|
|
1341
|
-
vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
|
|
1342
|
-
bytes_per_label(exchange(ptr, ptr + sizeof(bytes_per_label_t))),
|
|
1343
|
-
bytes_per_id(exchange(ptr, ptr + sizeof(bytes_per_id_t))),
|
|
1344
|
-
scalar_kind(exchange(ptr, ptr + sizeof(scalar_kind_t))), size(exchange(ptr, ptr + sizeof(size_t))),
|
|
1345
|
-
entry_idx(exchange(ptr, ptr + sizeof(entry_idx_t))), bytes_for_graphs(exchange(ptr, ptr + sizeof(size_t))),
|
|
1346
|
-
bytes_for_vectors(exchange(ptr, ptr + sizeof(size_t))), bytes_checksum(exchange(ptr, ptr + sizeof(size_t))) {}
|
|
1347
|
-
};
|
|
1348
|
-
|
|
1349
|
-
struct file_head_result_t {
|
|
1350
|
-
|
|
1351
|
-
using magic_t = file_head_t::magic_t;
|
|
1352
|
-
using version_major_t = file_head_t::version_major_t;
|
|
1353
|
-
using version_minor_t = file_head_t::version_minor_t;
|
|
1354
|
-
using version_patch_t = file_head_t::version_patch_t;
|
|
1355
|
-
|
|
1356
|
-
using connectivity_t = file_head_t::connectivity_t;
|
|
1357
|
-
using max_level_t = file_head_t::max_level_t;
|
|
1358
|
-
using vector_alignment_t = file_head_t::vector_alignment_t;
|
|
1359
|
-
using bytes_per_label_t = file_head_t::bytes_per_label_t;
|
|
1360
|
-
using bytes_per_id_t = file_head_t::bytes_per_id_t;
|
|
1361
|
-
using size_t = file_head_t::size_t;
|
|
1362
|
-
using entry_idx_t = file_head_t::entry_idx_t;
|
|
1363
|
-
|
|
1364
|
-
// Versioning:
|
|
1365
|
-
version_major_t version_major;
|
|
1366
|
-
version_minor_t version_minor;
|
|
1367
|
-
version_patch_t version_patch;
|
|
1368
|
-
|
|
1369
|
-
// Structural:
|
|
1370
|
-
metric_kind_t metric;
|
|
1371
|
-
connectivity_t connectivity;
|
|
1372
|
-
max_level_t max_level;
|
|
1373
|
-
vector_alignment_t vector_alignment;
|
|
1374
|
-
bytes_per_label_t bytes_per_label;
|
|
1375
|
-
bytes_per_id_t bytes_per_id;
|
|
1376
|
-
scalar_kind_t scalar_kind;
|
|
1377
|
-
size_t size;
|
|
1378
|
-
entry_idx_t entry_idx;
|
|
1379
|
-
|
|
1380
|
-
// Additional:
|
|
1381
|
-
size_t bytes_for_graphs;
|
|
1382
|
-
size_t bytes_for_vectors;
|
|
1383
|
-
size_t bytes_checksum;
|
|
1384
|
-
|
|
1385
|
-
error_t error;
|
|
1386
|
-
|
|
1387
|
-
explicit operator bool() const noexcept { return !error; }
|
|
1388
|
-
file_head_result_t failed(error_t message) noexcept {
|
|
1389
|
-
error = std::move(message);
|
|
1390
|
-
return std::move(*this);
|
|
1391
|
-
}
|
|
1392
|
-
};
|
|
1393
|
-
|
|
1394
|
-
static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 bytes");
|
|
1395
|
-
|
|
1396
1018
|
/// @brief C++17 and newer version deprecate the `std::result_of`
|
|
1397
1019
|
template <typename metric_at, typename... args_at>
|
|
1398
1020
|
using return_type_gt =
|
|
@@ -1402,32 +1024,65 @@ using return_type_gt =
|
|
|
1402
1024
|
typename std::result_of<metric_at(args_at...)>::type;
|
|
1403
1025
|
#endif
|
|
1404
1026
|
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
};
|
|
1414
|
-
#else
|
|
1415
|
-
struct viewed_file_t {
|
|
1416
|
-
int file_descriptor{};
|
|
1417
|
-
void* ptr{};
|
|
1418
|
-
size_t length{};
|
|
1419
|
-
explicit operator bool() const noexcept { return file_descriptor != 0; }
|
|
1027
|
+
/**
|
|
1028
|
+
* @brief An example of what a USearch-compatible ad-hoc filter would look like.
|
|
1029
|
+
*
|
|
1030
|
+
* A similar function object can be passed to search queries to further filter entries
|
|
1031
|
+
* on their auxiliary properties, such as some categorical labels stored in an external DBMS.
|
|
1032
|
+
*/
|
|
1033
|
+
struct dummy_predicate_t {
|
|
1034
|
+
template <typename member_at> constexpr bool operator()(member_at&&) const noexcept { return true; }
|
|
1420
1035
|
};
|
|
1421
|
-
#endif
|
|
1422
1036
|
|
|
1423
|
-
|
|
1424
|
-
|
|
1037
|
+
/**
|
|
1038
|
+
* @brief An example of what a USearch-compatible ad-hoc operation on in-flight entries.
|
|
1039
|
+
*
|
|
1040
|
+
* This kind of callbacks is used when the engine is being updated and you want to patch
|
|
1041
|
+
* the entries, while their are still under locks - limiting concurrent access and providing
|
|
1042
|
+
* consistency.
|
|
1043
|
+
*/
|
|
1044
|
+
struct dummy_callback_t {
|
|
1045
|
+
template <typename member_at> void operator()(member_at&&) const noexcept {}
|
|
1425
1046
|
};
|
|
1426
1047
|
|
|
1048
|
+
/**
|
|
1049
|
+
* @brief An example of what a USearch-compatible progress-bar should look like.
|
|
1050
|
+
*
|
|
1051
|
+
* This is particularly helpful when handling long-running tasks, like serialization,
|
|
1052
|
+
* saving, and loading from disk, or index-level joins.
|
|
1053
|
+
*/
|
|
1427
1054
|
struct dummy_progress_t {
|
|
1428
1055
|
inline void operator()(std::size_t /*progress*/, std::size_t /*total*/) const noexcept {}
|
|
1429
1056
|
};
|
|
1430
1057
|
|
|
1058
|
+
/**
|
|
1059
|
+
* @brief An example of what a USearch-compatible values prefetching mechanism should look like.
|
|
1060
|
+
*
|
|
1061
|
+
* USearch is designed to handle very large datasets, that may not fir into RAM. Fetching from
|
|
1062
|
+
* external memory is very expensive, so we've added a pre-fetching mechanism, that accepts
|
|
1063
|
+
* multiple objects at once, to cache in RAM ahead of the computation.
|
|
1064
|
+
* The received iterators support both `get_slot` and `get_key` operations.
|
|
1065
|
+
* An example usage may look like this:
|
|
1066
|
+
*
|
|
1067
|
+
* template <typename member_citerator_like_at>
|
|
1068
|
+
* inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {
|
|
1069
|
+
* for (; begin != end; ++begin)
|
|
1070
|
+
* io_uring_prefetch(offset_in_file(get_key(begin)));
|
|
1071
|
+
* }
|
|
1072
|
+
*/
|
|
1073
|
+
struct dummy_prefetch_t {
|
|
1074
|
+
template <typename member_citerator_like_at>
|
|
1075
|
+
inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {}
|
|
1076
|
+
};
|
|
1077
|
+
|
|
1078
|
+
/**
|
|
1079
|
+
* @brief An example of what a USearch-compatible executor (thread-pool) should look like.
|
|
1080
|
+
*
|
|
1081
|
+
* It's expected to have `execute_bulk(callback)` API to schedule one task per thread;
|
|
1082
|
+
* an identical `execute_bulk(count, callback)` overload that also accepts the number
|
|
1083
|
+
* of tasks, and somehow schedules them between threads; as well as `size()` to
|
|
1084
|
+
* determine the number of available threads.
|
|
1085
|
+
*/
|
|
1431
1086
|
struct dummy_executor_t {
|
|
1432
1087
|
dummy_executor_t() noexcept {}
|
|
1433
1088
|
std::size_t size() const noexcept { return 1; }
|
|
@@ -1444,164 +1099,474 @@ struct dummy_executor_t {
|
|
|
1444
1099
|
}
|
|
1445
1100
|
};
|
|
1446
1101
|
|
|
1102
|
+
/**
|
|
1103
|
+
* @brief An example of what a USearch-compatible label-to-label mapping should look like.
|
|
1104
|
+
*
|
|
1105
|
+
* This is particularly helpful for "Semantic Joins", where we map entries of one collection
|
|
1106
|
+
* to entries of another. In assymetric setups, where A -> B is needed, but B -> A is not,
|
|
1107
|
+
* this can be passed to minimize memory usage.
|
|
1108
|
+
*/
|
|
1447
1109
|
struct dummy_label_to_label_mapping_t {
|
|
1448
1110
|
struct member_ref_t {
|
|
1449
|
-
template <typename
|
|
1111
|
+
template <typename key_at> member_ref_t& operator=(key_at&&) noexcept { return *this; }
|
|
1450
1112
|
};
|
|
1451
|
-
template <typename
|
|
1113
|
+
template <typename key_at> member_ref_t operator[](key_at&&) const noexcept { return {}; }
|
|
1452
1114
|
};
|
|
1453
1115
|
|
|
1454
1116
|
/**
|
|
1455
|
-
* @brief
|
|
1456
|
-
*
|
|
1457
|
-
*
|
|
1458
|
-
* @section Features
|
|
1459
|
-
* - Thread-safe for concurrent construction.
|
|
1460
|
-
* - Doesn't allocate new threads, and reuses the ones its called from.
|
|
1461
|
-
* - Search for vectors of different dimensionality, if ::`metric_at` supports that.
|
|
1462
|
-
*
|
|
1463
|
-
* @tparam metric_at
|
|
1464
|
-
* A function responsible for computing the distance (dis-similarity) between two vectors.
|
|
1465
|
-
* Can have following signatures:
|
|
1466
|
-
* - `distance_t (*) (span_gt<scalar_t const>, span_gt<scalar_t const>)`
|
|
1467
|
-
* - `distance_t (*) (scalar_t const *, scalar_t const *)`
|
|
1468
|
-
*
|
|
1469
|
-
* @tparam label_at
|
|
1470
|
-
* The type of unique labels to assign to vectors.
|
|
1471
|
-
*
|
|
1472
|
-
* @tparam id_at
|
|
1473
|
-
* The smallest unsigned integer type to address indexed elements.
|
|
1474
|
-
* Can be a built-in `uint32_t`, `uint64_t`, or our custom `uint40_t`.
|
|
1475
|
-
*
|
|
1476
|
-
* @tparam allocator_at
|
|
1477
|
-
* Dynamic memory allocator.
|
|
1478
|
-
*
|
|
1479
|
-
* @section Usage
|
|
1480
|
-
*
|
|
1481
|
-
* @subsection Exceptions
|
|
1482
|
-
*
|
|
1483
|
-
* None of the methods throw exceptions in the "Release" compilation mode.
|
|
1484
|
-
* It may only `throw` if your memory ::allocator_at or ::metric_at isn't
|
|
1485
|
-
* safe to copy.
|
|
1486
|
-
*
|
|
1487
|
-
* @section Implementation Details
|
|
1488
|
-
*
|
|
1489
|
-
* Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
|
|
1490
|
-
* Every added vector forms a node in one or more levels of the graph.
|
|
1491
|
-
* Every node is present in the base level. Every following level contains a smaller
|
|
1492
|
-
* fraction of nodes. During search, the operation starts with the smaller levels
|
|
1493
|
-
* and zooms-in on every following iteration of larger graph traversals.
|
|
1494
|
-
*
|
|
1495
|
-
* Just one memory allocation is performed regardless of the number of levels.
|
|
1496
|
-
* The adjacency lists across all levels are concatenated into that single buffer.
|
|
1497
|
-
* That buffer starts with a "head", that stores the metadata, such as the
|
|
1498
|
-
* tallest "level" of the graph that it belongs to, the external "label", and the
|
|
1499
|
-
* number of "dimensions" in the vector.
|
|
1500
|
-
*
|
|
1501
|
-
* @subsection Colocated Nodes
|
|
1502
|
-
*
|
|
1503
|
-
* Every node contains some graph-related data, as well as the original vector.
|
|
1504
|
-
* Assuming the vectors can get large, we allow separating the index from the vectors.
|
|
1505
|
-
* They can either be stored in a single continuous block of memory, or in two separate
|
|
1506
|
-
* locations. Vector storage can also be detached from `index_gt` and managed by the user.
|
|
1507
|
-
* To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
|
|
1508
|
-
* overloads.
|
|
1509
|
-
*
|
|
1510
|
-
* @subsection Missing Functionality
|
|
1511
|
-
*
|
|
1512
|
-
* To simplify the implementation, the `index_gt` lacks endpoints to remove existing
|
|
1513
|
-
* vectors. That, however, is solved by `index_punned_dense_gt`, which also adds automatic casting.
|
|
1514
|
-
*
|
|
1117
|
+
* @brief Checks if the provided object has a dummy type, emulating an interface,
|
|
1118
|
+
* but performaing no real computation.
|
|
1515
1119
|
*/
|
|
1516
|
-
template <typename
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
using id_t = id_at;
|
|
1526
|
-
using allocator_t = allocator_at;
|
|
1527
|
-
using point_allocator_t = point_allocator_at;
|
|
1528
|
-
static_assert(sizeof(label_t) >= sizeof(id_t), "Having tiny labels doesn't make sense.");
|
|
1529
|
-
|
|
1530
|
-
using scalar_t = typename metric_t::scalar_t;
|
|
1531
|
-
using vector_view_t = span_gt<scalar_t const>;
|
|
1532
|
-
using distance_t = return_type_gt<metric_t, vector_view_t, vector_view_t>;
|
|
1120
|
+
template <typename object_at> static constexpr bool is_dummy() {
|
|
1121
|
+
using object_t = typename std::remove_all_extents<object_at>::type;
|
|
1122
|
+
return std::is_same<object_t, dummy_predicate_t>::value || //
|
|
1123
|
+
std::is_same<object_t, dummy_callback_t>::value || //
|
|
1124
|
+
std::is_same<object_t, dummy_progress_t>::value || //
|
|
1125
|
+
std::is_same<object_t, dummy_prefetch_t>::value || //
|
|
1126
|
+
std::is_same<object_t, dummy_executor_t>::value || //
|
|
1127
|
+
std::is_same<object_t, dummy_label_to_label_mapping_t>::value;
|
|
1128
|
+
}
|
|
1533
1129
|
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
id_t id;
|
|
1538
|
-
};
|
|
1130
|
+
template <typename, typename at> struct has_reset_gt {
|
|
1131
|
+
static_assert(std::integral_constant<at, false>::value, "Second template parameter needs to be of function type.");
|
|
1132
|
+
};
|
|
1539
1133
|
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1134
|
+
template <typename check_at, typename return_at, typename... args_at>
|
|
1135
|
+
struct has_reset_gt<check_at, return_at(args_at...)> {
|
|
1136
|
+
private:
|
|
1137
|
+
template <typename at>
|
|
1138
|
+
static constexpr auto check(at*) ->
|
|
1139
|
+
typename std::is_same<decltype(std::declval<at>().reset(std::declval<args_at>()...)), return_at>::type;
|
|
1140
|
+
template <typename> static constexpr std::false_type check(...);
|
|
1544
1141
|
|
|
1545
|
-
|
|
1546
|
-
};
|
|
1142
|
+
typedef decltype(check<check_at>(0)) type;
|
|
1547
1143
|
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
index_t* index_{};
|
|
1552
|
-
std::size_t offset_{};
|
|
1144
|
+
public:
|
|
1145
|
+
static constexpr bool value = type::value;
|
|
1146
|
+
};
|
|
1553
1147
|
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1148
|
+
/**
|
|
1149
|
+
* @brief Checks if a certain class has a member function called `reset`.
|
|
1150
|
+
*/
|
|
1151
|
+
template <typename at> constexpr bool has_reset() { return has_reset_gt<at, void()>::value; }
|
|
1557
1152
|
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
using value_type = ref_t;
|
|
1561
|
-
using difference_type = std::ptrdiff_t;
|
|
1562
|
-
using pointer = void;
|
|
1563
|
-
using reference = ref_t;
|
|
1153
|
+
struct serialization_result_t {
|
|
1154
|
+
error_t error;
|
|
1564
1155
|
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1156
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1157
|
+
serialization_result_t failed(error_t message) noexcept {
|
|
1158
|
+
error = std::move(message);
|
|
1159
|
+
return std::move(*this);
|
|
1160
|
+
}
|
|
1161
|
+
};
|
|
1569
1162
|
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1163
|
+
/**
|
|
1164
|
+
* @brief Smart-pointer wrapping the LibC @b `FILE` for binary file @b outputs.
|
|
1165
|
+
*
|
|
1166
|
+
* This class raises no exceptions and corresponds errors through `serialization_result_t`.
|
|
1167
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1168
|
+
*/
|
|
1169
|
+
class output_file_t {
|
|
1170
|
+
char const* path_ = nullptr;
|
|
1171
|
+
std::FILE* file_ = nullptr;
|
|
1574
1172
|
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1173
|
+
public:
|
|
1174
|
+
output_file_t(char const* path) noexcept : path_(path) {}
|
|
1175
|
+
~output_file_t() noexcept { close(); }
|
|
1176
|
+
output_file_t(output_file_t&& other) noexcept
|
|
1177
|
+
: path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
|
|
1178
|
+
output_file_t& operator=(output_file_t&& other) noexcept {
|
|
1179
|
+
std::swap(path_, other.path_);
|
|
1180
|
+
std::swap(file_, other.file_);
|
|
1181
|
+
return *this;
|
|
1182
|
+
}
|
|
1183
|
+
serialization_result_t open_if_not() noexcept {
|
|
1184
|
+
serialization_result_t result;
|
|
1185
|
+
if (!file_)
|
|
1186
|
+
file_ = std::fopen(path_, "wb");
|
|
1187
|
+
if (!file_)
|
|
1188
|
+
return result.failed(std::strerror(errno));
|
|
1189
|
+
return result;
|
|
1190
|
+
}
|
|
1191
|
+
serialization_result_t write(void* begin, std::size_t length) noexcept {
|
|
1192
|
+
serialization_result_t result;
|
|
1193
|
+
std::size_t written = std::fwrite(begin, length, 1, file_);
|
|
1194
|
+
if (!written)
|
|
1195
|
+
result.failed(std::strerror(errno));
|
|
1196
|
+
return result;
|
|
1197
|
+
}
|
|
1198
|
+
void close() noexcept {
|
|
1199
|
+
if (file_)
|
|
1200
|
+
std::fclose(exchange(file_, nullptr));
|
|
1201
|
+
}
|
|
1202
|
+
};
|
|
1203
|
+
|
|
1204
|
+
/**
|
|
1205
|
+
* @brief Smart-pointer wrapping the LibC @b `FILE` for binary files @b inputs.
|
|
1206
|
+
*
|
|
1207
|
+
* This class raises no exceptions and corresponds errors through `serialization_result_t`.
|
|
1208
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1209
|
+
*/
|
|
1210
|
+
class input_file_t {
|
|
1211
|
+
char const* path_ = nullptr;
|
|
1212
|
+
std::FILE* file_ = nullptr;
|
|
1213
|
+
|
|
1214
|
+
public:
|
|
1215
|
+
input_file_t(char const* path) noexcept : path_(path) {}
|
|
1216
|
+
~input_file_t() noexcept { close(); }
|
|
1217
|
+
input_file_t(input_file_t&& other) noexcept
|
|
1218
|
+
: path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
|
|
1219
|
+
input_file_t& operator=(input_file_t&& other) noexcept {
|
|
1220
|
+
std::swap(path_, other.path_);
|
|
1221
|
+
std::swap(file_, other.file_);
|
|
1222
|
+
return *this;
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
serialization_result_t open_if_not() noexcept {
|
|
1226
|
+
serialization_result_t result;
|
|
1227
|
+
if (!file_)
|
|
1228
|
+
file_ = std::fopen(path_, "rb");
|
|
1229
|
+
if (!file_)
|
|
1230
|
+
return result.failed(std::strerror(errno));
|
|
1231
|
+
return result;
|
|
1232
|
+
}
|
|
1233
|
+
serialization_result_t read(void* begin, std::size_t length) noexcept {
|
|
1234
|
+
serialization_result_t result;
|
|
1235
|
+
std::size_t read = std::fread(begin, length, 1, file_);
|
|
1236
|
+
if (!read)
|
|
1237
|
+
result.failed(std::feof(file_) ? "End of file reached!" : std::strerror(errno));
|
|
1238
|
+
return result;
|
|
1239
|
+
}
|
|
1240
|
+
void close() noexcept {
|
|
1241
|
+
if (file_)
|
|
1242
|
+
std::fclose(exchange(file_, nullptr));
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
explicit operator bool() const noexcept { return file_; }
|
|
1246
|
+
bool seek_to(std::size_t progress) noexcept { return std::fseek(file_, progress, SEEK_SET) == 0; }
|
|
1247
|
+
bool seek_to_end() noexcept { return std::fseek(file_, 0L, SEEK_END) == 0; }
|
|
1248
|
+
bool infer_progress(std::size_t& progress) noexcept {
|
|
1249
|
+
long int result = std::ftell(file_);
|
|
1250
|
+
if (result == -1L)
|
|
1251
|
+
return false;
|
|
1252
|
+
progress = static_cast<std::size_t>(result);
|
|
1253
|
+
return true;
|
|
1254
|
+
}
|
|
1255
|
+
};
|
|
1256
|
+
|
|
1257
|
+
/**
|
|
1258
|
+
* @brief Represents a memory-mapped file or a pre-allocated anonymous memory region.
|
|
1259
|
+
*
|
|
1260
|
+
* This class provides a convenient way to memory-map a file and access its contents as a block of
|
|
1261
|
+
* memory. The class handles platform-specific memory-mapping operations on Windows, Linux, and MacOS.
|
|
1262
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1263
|
+
*/
|
|
1264
|
+
class memory_mapped_file_t {
|
|
1265
|
+
char const* path_{}; /**< The path to the file to be memory-mapped. */
|
|
1266
|
+
void* ptr_{}; /**< A pointer to the memory-mapping. */
|
|
1267
|
+
size_t length_{}; /**< The length of the memory-mapped file in bytes. */
|
|
1268
|
+
|
|
1269
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1270
|
+
HANDLE file_handle_{}; /**< The file handle on Windows. */
|
|
1271
|
+
HANDLE mapping_handle_{}; /**< The mapping handle on Windows. */
|
|
1272
|
+
#else
|
|
1273
|
+
int file_descriptor_{}; /**< The file descriptor on Linux and MacOS. */
|
|
1274
|
+
#endif
|
|
1275
|
+
|
|
1276
|
+
public:
|
|
1277
|
+
explicit operator bool() const noexcept { return ptr_ != nullptr; }
|
|
1278
|
+
byte_t const* data() const noexcept { return reinterpret_cast<byte_t const*>(ptr_); }
|
|
1279
|
+
std::size_t size() const noexcept { return static_cast<std::size_t>(length_); }
|
|
1280
|
+
|
|
1281
|
+
memory_mapped_file_t() noexcept {}
|
|
1282
|
+
memory_mapped_file_t(char const* path) noexcept : path_(path) {}
|
|
1283
|
+
~memory_mapped_file_t() noexcept { close(); }
|
|
1284
|
+
memory_mapped_file_t(memory_mapped_file_t&& other) noexcept
|
|
1285
|
+
: path_(exchange(other.path_, nullptr)), ptr_(exchange(other.ptr_, nullptr)),
|
|
1286
|
+
length_(exchange(other.length_, 0)),
|
|
1287
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1288
|
+
file_handle_(exchange(other.file_handle_, nullptr)), mapping_handle_(exchange(other.mapping_handle_, nullptr))
|
|
1289
|
+
#else
|
|
1290
|
+
file_descriptor_(exchange(other.file_descriptor_, 0))
|
|
1291
|
+
#endif
|
|
1292
|
+
{
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
memory_mapped_file_t(byte_t* data, std::size_t length) noexcept : ptr_(data), length_(length) {}
|
|
1296
|
+
|
|
1297
|
+
memory_mapped_file_t& operator=(memory_mapped_file_t&& other) noexcept {
|
|
1298
|
+
std::swap(path_, other.path_);
|
|
1299
|
+
std::swap(ptr_, other.ptr_);
|
|
1300
|
+
std::swap(length_, other.length_);
|
|
1301
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1302
|
+
std::swap(file_handle_, other.file_handle_);
|
|
1303
|
+
std::swap(mapping_handle_, other.mapping_handle_);
|
|
1304
|
+
#else
|
|
1305
|
+
std::swap(file_descriptor_, other.file_descriptor_);
|
|
1306
|
+
#endif
|
|
1307
|
+
return *this;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
serialization_result_t open_if_not() noexcept {
|
|
1311
|
+
serialization_result_t result;
|
|
1312
|
+
if (!path_ || ptr_)
|
|
1313
|
+
return result;
|
|
1314
|
+
|
|
1315
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1316
|
+
|
|
1317
|
+
HANDLE file_handle =
|
|
1318
|
+
CreateFile(path_, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
|
|
1319
|
+
if (file_handle == INVALID_HANDLE_VALUE)
|
|
1320
|
+
return result.failed("Opening file failed!");
|
|
1321
|
+
|
|
1322
|
+
std::size_t file_length = GetFileSize(file_handle, 0);
|
|
1323
|
+
HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
|
|
1324
|
+
if (mapping_handle == 0) {
|
|
1325
|
+
CloseHandle(file_handle);
|
|
1326
|
+
return result.failed("Mapping file failed!");
|
|
1590
1327
|
}
|
|
1591
|
-
|
|
1592
|
-
|
|
1328
|
+
|
|
1329
|
+
byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
|
|
1330
|
+
if (file == 0) {
|
|
1331
|
+
CloseHandle(mapping_handle);
|
|
1332
|
+
CloseHandle(file_handle);
|
|
1333
|
+
return result.failed("View the map failed!");
|
|
1593
1334
|
}
|
|
1594
|
-
|
|
1595
|
-
|
|
1335
|
+
file_handle_ = file_handle;
|
|
1336
|
+
mapping_handle_ = mapping_handle;
|
|
1337
|
+
ptr_ = file;
|
|
1338
|
+
length_ = file_length;
|
|
1339
|
+
#else
|
|
1340
|
+
|
|
1341
|
+
#if defined(USEARCH_DEFINED_LINUX)
|
|
1342
|
+
int descriptor = open(path_, O_RDONLY | O_NOATIME);
|
|
1343
|
+
#else
|
|
1344
|
+
int descriptor = open(path_, O_RDONLY);
|
|
1345
|
+
#endif
|
|
1346
|
+
|
|
1347
|
+
// Estimate the file size
|
|
1348
|
+
struct stat file_stat;
|
|
1349
|
+
int fstat_status = fstat(descriptor, &file_stat);
|
|
1350
|
+
if (fstat_status < 0) {
|
|
1351
|
+
::close(descriptor);
|
|
1352
|
+
return result.failed(std::strerror(errno));
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
// Map the entire file
|
|
1356
|
+
byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_SHARED, descriptor, 0);
|
|
1357
|
+
if (file == MAP_FAILED) {
|
|
1358
|
+
::close(descriptor);
|
|
1359
|
+
return result.failed(std::strerror(errno));
|
|
1360
|
+
}
|
|
1361
|
+
file_descriptor_ = descriptor;
|
|
1362
|
+
ptr_ = file;
|
|
1363
|
+
length_ = file_stat.st_size;
|
|
1364
|
+
#endif // Platform specific code
|
|
1365
|
+
return result;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
void close() noexcept {
|
|
1369
|
+
if (!path_) {
|
|
1370
|
+
ptr_ = nullptr;
|
|
1371
|
+
length_ = 0;
|
|
1372
|
+
return;
|
|
1596
1373
|
}
|
|
1374
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1375
|
+
UnmapViewOfFile(ptr_);
|
|
1376
|
+
CloseHandle(mapping_handle_);
|
|
1377
|
+
CloseHandle(file_handle_);
|
|
1378
|
+
mapping_handle_ = nullptr;
|
|
1379
|
+
file_handle_ = nullptr;
|
|
1380
|
+
#else
|
|
1381
|
+
munmap(ptr_, length_);
|
|
1382
|
+
::close(file_descriptor_);
|
|
1383
|
+
file_descriptor_ = 0;
|
|
1384
|
+
#endif
|
|
1385
|
+
ptr_ = nullptr;
|
|
1386
|
+
length_ = 0;
|
|
1387
|
+
}
|
|
1388
|
+
};
|
|
1389
|
+
|
|
1390
|
+
struct index_serialized_header_t {
|
|
1391
|
+
std::uint64_t size = 0;
|
|
1392
|
+
std::uint64_t connectivity = 0;
|
|
1393
|
+
std::uint64_t connectivity_base = 0;
|
|
1394
|
+
std::uint64_t max_level = 0;
|
|
1395
|
+
std::uint64_t entry_slot = 0;
|
|
1396
|
+
};
|
|
1397
|
+
|
|
1398
|
+
using default_key_t = std::uint64_t;
|
|
1399
|
+
using default_slot_t = std::uint32_t;
|
|
1400
|
+
using default_distance_t = float;
|
|
1401
|
+
|
|
1402
|
+
template <typename key_at = default_key_t> struct member_gt {
|
|
1403
|
+
key_at key;
|
|
1404
|
+
std::size_t slot;
|
|
1405
|
+
};
|
|
1406
|
+
|
|
1407
|
+
template <typename key_at> inline std::size_t get_slot(member_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1408
|
+
template <typename key_at> inline key_at get_key(member_gt<key_at> const& m) noexcept { return m.key; }
|
|
1409
|
+
|
|
1410
|
+
template <typename key_at = default_key_t> struct member_cref_gt {
|
|
1411
|
+
misaligned_ref_gt<key_at const> key;
|
|
1412
|
+
std::size_t slot;
|
|
1413
|
+
};
|
|
1414
|
+
|
|
1415
|
+
template <typename key_at> inline std::size_t get_slot(member_cref_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1416
|
+
template <typename key_at> inline key_at get_key(member_cref_gt<key_at> const& m) noexcept { return m.key; }
|
|
1417
|
+
|
|
1418
|
+
template <typename key_at = default_key_t> struct member_ref_gt {
|
|
1419
|
+
misaligned_ref_gt<key_at> key;
|
|
1420
|
+
std::size_t slot;
|
|
1421
|
+
|
|
1422
|
+
inline operator member_cref_gt<key_at>() const noexcept { return {key.ptr(), slot}; }
|
|
1423
|
+
};
|
|
1424
|
+
|
|
1425
|
+
template <typename key_at> inline std::size_t get_slot(member_ref_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1426
|
+
template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m) noexcept { return m.key; }
|
|
1427
|
+
|
|
1428
|
+
/**
|
|
1429
|
+
* @brief Approximate Nearest Neighbors Search @b index-structure using the
|
|
1430
|
+
* Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
|
|
1431
|
+
* If classical containers store @b Key->Value mappings, this one can
|
|
1432
|
+
* be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
|
|
1433
|
+
*
|
|
1434
|
+
* Unlike most implementations, this one is generic anc can be used for any search,
|
|
1435
|
+
* not just within equi-dimensional vectors. Examples range from texts to similar Chess
|
|
1436
|
+
* positions.
|
|
1437
|
+
*
|
|
1438
|
+
* @tparam key_at
|
|
1439
|
+
* The type of primary objects stored in the index.
|
|
1440
|
+
* The values, to which those map, are not managed by the same index structure.
|
|
1441
|
+
*
|
|
1442
|
+
* @tparam compressed_slot_at
|
|
1443
|
+
* The smallest unsigned integer type to address indexed elements.
|
|
1444
|
+
* It is used internally to maximize space-efficiency and is generally
|
|
1445
|
+
* up-casted to @b `std::size_t` in public interfaces.
|
|
1446
|
+
* Can be a built-in @b `uint32_t`, `uint64_t`, or our custom @b `uint40_t`.
|
|
1447
|
+
* Which makes the most sense for 4B+ entry indexes.
|
|
1448
|
+
*
|
|
1449
|
+
* @tparam dynamic_allocator_at
|
|
1450
|
+
* Dynamic memory allocator for temporary buffers, visits indicators, and
|
|
1451
|
+
* priority queues, needed during construction and traversals of graphs.
|
|
1452
|
+
* The allocated buffers may be uninitialized.
|
|
1453
|
+
*
|
|
1454
|
+
* @tparam tape_allocator_at
|
|
1455
|
+
* Potentially different memory allocator for primary allocations of nodes and vectors.
|
|
1456
|
+
* It would never `deallocate` separate entries, and would only free all the space at once.
|
|
1457
|
+
* The allocated buffers may be uninitialized.
|
|
1458
|
+
*
|
|
1459
|
+
* @section Features
|
|
1460
|
+
*
|
|
1461
|
+
* - Thread-safe for concurrent construction, search, and updates.
|
|
1462
|
+
* - Doesn't allocate new threads, and reuses the ones its called from.
|
|
1463
|
+
* - Allows storing value externally, managing just the similarity index.
|
|
1464
|
+
* - Joins.
|
|
1465
|
+
|
|
1466
|
+
* @section Usage
|
|
1467
|
+
*
|
|
1468
|
+
* @subsection Exceptions
|
|
1469
|
+
*
|
|
1470
|
+
* None of the methods throw exceptions in the "Release" compilation mode.
|
|
1471
|
+
* It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
|
|
1472
|
+
* safe to copy.
|
|
1473
|
+
*
|
|
1474
|
+
* @subsection Serialization
|
|
1475
|
+
*
|
|
1476
|
+
* When serialized, doesn't include any additional metadata.
|
|
1477
|
+
* It is just the multi-level proximity-graph. You may want to store metadata about
|
|
1478
|
+
* the used metric and key types somewhere else.
|
|
1479
|
+
*
|
|
1480
|
+
* @section Implementation Details
|
|
1481
|
+
*
|
|
1482
|
+
* Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
|
|
1483
|
+
* Every added vector forms a node in one or more levels of the graph.
|
|
1484
|
+
* Every node is present in the base level. Every following level contains a smaller
|
|
1485
|
+
* fraction of nodes. During search, the operation starts with the smaller levels
|
|
1486
|
+
* and zooms-in on every following iteration of larger graph traversals.
|
|
1487
|
+
*
|
|
1488
|
+
* Just one memory allocation is performed regardless of the number of levels.
|
|
1489
|
+
* The adjacency lists across all levels are concatenated into that single buffer.
|
|
1490
|
+
* That buffer starts with a "head", that stores the metadata, such as the
|
|
1491
|
+
* tallest "level" of the graph that it belongs to, the external "key", and the
|
|
1492
|
+
* number of "dimensions" in the vector.
|
|
1493
|
+
*
|
|
1494
|
+
* @section Metrics, Predicates and Callbacks
|
|
1495
|
+
*
|
|
1496
|
+
*
|
|
1497
|
+
* @section Smart References and Iterators
|
|
1498
|
+
*
|
|
1499
|
+
* - `member_citerator_t` and `member_iterator_t` have only slots, no indirections.
|
|
1500
|
+
*
|
|
1501
|
+
* - `member_cref_t` and `member_ref_t` contains the `slot` and a reference
|
|
1502
|
+
* to the key. So it passes through 1 level of visited_members in `nodes_`.
|
|
1503
|
+
* Retrieving the key via `get_key` will cause fetching yet another cache line.
|
|
1504
|
+
*
|
|
1505
|
+
* - `member_gt` contains an already prefetched copy of the key.
|
|
1506
|
+
*
|
|
1507
|
+
*/
|
|
1508
|
+
template <typename distance_at = default_distance_t, //
|
|
1509
|
+
typename key_at = default_key_t, //
|
|
1510
|
+
typename compressed_slot_at = default_slot_t, //
|
|
1511
|
+
typename dynamic_allocator_at = std::allocator<byte_t>, //
|
|
1512
|
+
typename tape_allocator_at = dynamic_allocator_at> //
|
|
1513
|
+
class index_gt {
|
|
1514
|
+
public:
|
|
1515
|
+
using distance_t = distance_at;
|
|
1516
|
+
using key_t = key_at;
|
|
1517
|
+
using compressed_slot_t = compressed_slot_at;
|
|
1518
|
+
using dynamic_allocator_t = dynamic_allocator_at;
|
|
1519
|
+
using tape_allocator_t = tape_allocator_at;
|
|
1520
|
+
static_assert(sizeof(key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
|
|
1521
|
+
|
|
1522
|
+
using member_cref_t = member_cref_gt<key_t>;
|
|
1523
|
+
using member_ref_t = member_ref_gt<key_t>;
|
|
1524
|
+
|
|
1525
|
+
template <typename ref_at, typename index_at> class member_iterator_gt {
|
|
1526
|
+
using ref_t = ref_at;
|
|
1527
|
+
using index_t = index_at;
|
|
1528
|
+
|
|
1529
|
+
friend class index_gt;
|
|
1530
|
+
member_iterator_gt() noexcept {}
|
|
1531
|
+
member_iterator_gt(index_t* index, std::size_t slot) noexcept : index_(index), slot_(slot) {}
|
|
1532
|
+
|
|
1533
|
+
index_t* index_{};
|
|
1534
|
+
std::size_t slot_{};
|
|
1535
|
+
|
|
1536
|
+
public:
|
|
1537
|
+
using iterator_category = std::random_access_iterator_tag;
|
|
1538
|
+
using value_type = ref_t;
|
|
1539
|
+
using difference_type = std::ptrdiff_t;
|
|
1540
|
+
using pointer = void;
|
|
1541
|
+
using reference = ref_t;
|
|
1542
|
+
|
|
1543
|
+
reference operator*() const noexcept { return {index_->node_at_(slot_).key(), slot_}; }
|
|
1544
|
+
key_t key() const noexcept { return index_->node_at_(slot_).key(); }
|
|
1545
|
+
|
|
1546
|
+
friend inline std::size_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
|
|
1547
|
+
friend inline key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
|
|
1548
|
+
|
|
1549
|
+
member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
|
|
1550
|
+
member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
|
|
1551
|
+
member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
|
|
1552
|
+
member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
|
|
1553
|
+
|
|
1554
|
+
// clang-format off
|
|
1555
|
+
member_iterator_gt& operator++() noexcept { slot_ += 1; return *this; }
|
|
1556
|
+
member_iterator_gt& operator--() noexcept { slot_ -= 1; return *this; }
|
|
1557
|
+
member_iterator_gt& operator+=(difference_type d) noexcept { slot_ += d; return *this; }
|
|
1558
|
+
member_iterator_gt& operator-=(difference_type d) noexcept { slot_ -= d; return *this; }
|
|
1559
|
+
bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
|
|
1560
|
+
bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
|
|
1561
|
+
// clang-format on
|
|
1597
1562
|
};
|
|
1598
1563
|
|
|
1599
1564
|
using member_iterator_t = member_iterator_gt<member_ref_t, index_gt>;
|
|
1600
1565
|
using member_citerator_t = member_iterator_gt<member_cref_t, index_gt const>;
|
|
1601
1566
|
|
|
1602
1567
|
// STL compatibility:
|
|
1603
|
-
using value_type =
|
|
1604
|
-
using allocator_type =
|
|
1568
|
+
using value_type = key_t;
|
|
1569
|
+
using allocator_type = dynamic_allocator_t;
|
|
1605
1570
|
using size_type = std::size_t;
|
|
1606
1571
|
using difference_type = std::ptrdiff_t;
|
|
1607
1572
|
using reference = member_ref_t;
|
|
@@ -1613,109 +1578,113 @@ class index_gt {
|
|
|
1613
1578
|
using reverse_iterator = std::reverse_iterator<member_iterator_t>;
|
|
1614
1579
|
using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
|
|
1615
1580
|
|
|
1616
|
-
|
|
1617
|
-
using
|
|
1618
|
-
|
|
1619
|
-
|
|
1581
|
+
using dynamic_allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
|
|
1582
|
+
using byte_t = typename dynamic_allocator_t::value_type;
|
|
1583
|
+
static_assert( //
|
|
1584
|
+
sizeof(byte_t) == 1, //
|
|
1585
|
+
"Primary allocator must allocate separate addressable bytes");
|
|
1620
1586
|
|
|
1621
|
-
using
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
using point_allocator_traits_t = std::allocator_traits<point_allocator_t>;
|
|
1626
|
-
static_assert(sizeof(typename point_allocator_traits_t::value_type) == 1,
|
|
1627
|
-
"Allocator must allocate separate addressable bytes");
|
|
1587
|
+
using tape_allocator_traits_t = std::allocator_traits<tape_allocator_t>;
|
|
1588
|
+
static_assert( //
|
|
1589
|
+
sizeof(typename tape_allocator_traits_t::value_type) == 1, //
|
|
1590
|
+
"Tape allocator must allocate separate addressable bytes");
|
|
1628
1591
|
|
|
1592
|
+
private:
|
|
1629
1593
|
/**
|
|
1630
|
-
* @brief
|
|
1631
|
-
*
|
|
1594
|
+
* @brief Integer for the number of node neighbors at a specific level of the
|
|
1595
|
+
* multi-level graph. It's selected to be `std::uint32_t` to improve the
|
|
1596
|
+
* alignment in most common cases.
|
|
1632
1597
|
*/
|
|
1633
|
-
|
|
1598
|
+
using neighbors_count_t = std::uint32_t;
|
|
1599
|
+
using level_t = std::int16_t;
|
|
1634
1600
|
|
|
1635
1601
|
/**
|
|
1636
1602
|
* @brief How many bytes of memory are needed to form the "head" of the node.
|
|
1637
1603
|
*/
|
|
1638
|
-
static constexpr std::size_t node_head_bytes_() { return sizeof(
|
|
1604
|
+
static constexpr std::size_t node_head_bytes_() { return sizeof(key_t) + sizeof(level_t); }
|
|
1639
1605
|
|
|
1640
|
-
using visits_bitset_t = visits_bitset_gt<
|
|
1606
|
+
using visits_bitset_t = visits_bitset_gt<dynamic_allocator_t>;
|
|
1641
1607
|
|
|
1642
1608
|
struct precomputed_constants_t {
|
|
1643
1609
|
double inverse_log_connectivity{};
|
|
1644
|
-
std::size_t connectivity_max_base{};
|
|
1645
1610
|
std::size_t neighbors_bytes{};
|
|
1646
1611
|
std::size_t neighbors_base_bytes{};
|
|
1647
1612
|
};
|
|
1613
|
+
/// @brief A space-efficient internal data-structure used in graph traversal queues.
|
|
1648
1614
|
struct candidate_t {
|
|
1649
1615
|
distance_t distance;
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
struct compare_by_distance_t {
|
|
1653
|
-
inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
|
|
1616
|
+
compressed_slot_t slot;
|
|
1617
|
+
inline bool operator<(candidate_t other) const noexcept { return distance < other.distance; }
|
|
1654
1618
|
};
|
|
1655
1619
|
|
|
1656
1620
|
using candidates_view_t = span_gt<candidate_t const>;
|
|
1657
|
-
using candidates_allocator_t = typename
|
|
1658
|
-
using top_candidates_t = sorted_buffer_gt<candidate_t,
|
|
1659
|
-
using next_candidates_t = max_heap_gt<candidate_t,
|
|
1621
|
+
using candidates_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<candidate_t>;
|
|
1622
|
+
using top_candidates_t = sorted_buffer_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
|
|
1623
|
+
using next_candidates_t = max_heap_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
|
|
1660
1624
|
|
|
1661
1625
|
/**
|
|
1662
|
-
* @brief A loosely-structured handle for every node. One such node is created for every
|
|
1626
|
+
* @brief A loosely-structured handle for every node. One such node is created for every member.
|
|
1663
1627
|
* To minimize memory usage and maximize the number of entries per cache-line, it only
|
|
1664
|
-
* stores to pointers.
|
|
1665
|
-
*
|
|
1628
|
+
* stores to pointers. The internal tape starts with a `key_t` @b key, then
|
|
1629
|
+
* a `level_t` for the number of graph @b levels in which this member appears,
|
|
1630
|
+
* then the { `neighbors_count_t`, `compressed_slot_t`, `compressed_slot_t` ... } sequences
|
|
1631
|
+
* for @b each-level.
|
|
1666
1632
|
*/
|
|
1667
1633
|
class node_t {
|
|
1668
1634
|
byte_t* tape_{};
|
|
1669
|
-
scalar_t* vector_{};
|
|
1670
1635
|
|
|
1671
1636
|
public:
|
|
1672
|
-
explicit node_t(byte_t* tape
|
|
1637
|
+
explicit node_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1673
1638
|
byte_t* tape() const noexcept { return tape_; }
|
|
1674
|
-
scalar_t* vector() const noexcept { return vector_; }
|
|
1675
1639
|
byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
|
|
1640
|
+
explicit operator bool() const noexcept { return tape_; }
|
|
1676
1641
|
|
|
1677
1642
|
node_t() = default;
|
|
1678
1643
|
node_t(node_t const&) = default;
|
|
1679
1644
|
node_t& operator=(node_t const&) = default;
|
|
1680
1645
|
|
|
1681
|
-
misaligned_ref_gt<
|
|
1682
|
-
misaligned_ref_gt<
|
|
1683
|
-
misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(
|
|
1646
|
+
misaligned_ref_gt<key_t const> ckey() const noexcept { return {tape_}; }
|
|
1647
|
+
misaligned_ref_gt<key_t> key() const noexcept { return {tape_}; }
|
|
1648
|
+
misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(key_t)}; }
|
|
1684
1649
|
|
|
1685
|
-
void
|
|
1686
|
-
void
|
|
1687
|
-
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
|
|
1688
|
-
|
|
1689
|
-
operator vector_view_t() const noexcept { return {vector(), dim()}; }
|
|
1690
|
-
vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
|
|
1691
|
-
explicit operator member_ref_t() noexcept { return {{tape_}, vector_view()}; }
|
|
1692
|
-
explicit operator member_cref_t() const noexcept { return {label(), vector_view()}; }
|
|
1693
|
-
explicit operator bool() const noexcept { return tape_; }
|
|
1650
|
+
void key(key_t v) noexcept { return misaligned_store<key_t>(tape_, v); }
|
|
1651
|
+
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(key_t), v); }
|
|
1694
1652
|
};
|
|
1695
1653
|
|
|
1654
|
+
static_assert(std::is_trivially_copy_constructible<node_t>::value, "Nodes must be light!");
|
|
1655
|
+
static_assert(std::is_trivially_destructible<node_t>::value, "Nodes must be light!");
|
|
1656
|
+
|
|
1696
1657
|
/**
|
|
1697
1658
|
* @brief A slice of the node's tape, containing a the list of neighbors
|
|
1698
1659
|
* for a node in a single graph level. It's pre-allocated to fit
|
|
1699
|
-
* as many neighbors
|
|
1660
|
+
* as many neighbors "slots", as may be needed at the target level,
|
|
1700
1661
|
* and starts with a single integer `neighbors_count_t` counter.
|
|
1701
1662
|
*/
|
|
1702
1663
|
class neighbors_ref_t {
|
|
1703
1664
|
byte_t* tape_;
|
|
1704
1665
|
|
|
1705
|
-
static constexpr std::size_t shift(std::size_t i = 0) {
|
|
1666
|
+
static constexpr std::size_t shift(std::size_t i = 0) {
|
|
1667
|
+
return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
|
|
1668
|
+
}
|
|
1706
1669
|
|
|
1707
1670
|
public:
|
|
1708
1671
|
neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1709
|
-
misaligned_ptr_gt<
|
|
1710
|
-
misaligned_ptr_gt<
|
|
1711
|
-
misaligned_ptr_gt<
|
|
1712
|
-
misaligned_ptr_gt<
|
|
1713
|
-
|
|
1672
|
+
misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
|
|
1673
|
+
misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
|
|
1674
|
+
misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
|
|
1675
|
+
misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
|
|
1676
|
+
compressed_slot_t operator[](std::size_t i) const noexcept {
|
|
1677
|
+
return misaligned_load<compressed_slot_t>(tape_ + shift(i));
|
|
1678
|
+
}
|
|
1714
1679
|
std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
|
|
1715
|
-
void clear() noexcept {
|
|
1716
|
-
|
|
1680
|
+
void clear() noexcept {
|
|
1681
|
+
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1682
|
+
std::memset(tape_, 0, shift(n));
|
|
1683
|
+
// misaligned_store<neighbors_count_t>(tape_, 0);
|
|
1684
|
+
}
|
|
1685
|
+
void push_back(compressed_slot_t slot) noexcept {
|
|
1717
1686
|
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1718
|
-
misaligned_store<
|
|
1687
|
+
misaligned_store<compressed_slot_t>(tape_ + shift(n), slot);
|
|
1719
1688
|
misaligned_store<neighbors_count_t>(tape_, n + 1);
|
|
1720
1689
|
}
|
|
1721
1690
|
};
|
|
@@ -1730,44 +1699,72 @@ class index_gt {
|
|
|
1730
1699
|
next_candidates_t next_candidates{};
|
|
1731
1700
|
visits_bitset_t visits{};
|
|
1732
1701
|
std::default_random_engine level_generator{};
|
|
1733
|
-
metric_t metric{};
|
|
1734
1702
|
std::size_t iteration_cycles{};
|
|
1735
|
-
std::size_t
|
|
1703
|
+
std::size_t computed_distances_count{};
|
|
1704
|
+
|
|
1705
|
+
template <typename value_at, typename metric_at, typename entry_at> //
|
|
1706
|
+
inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1707
|
+
static_assert( //
|
|
1708
|
+
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1709
|
+
"Unexpected type");
|
|
1736
1710
|
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1711
|
+
computed_distances_count++;
|
|
1712
|
+
return metric(first, second);
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
template <typename metric_at, typename entry_at> //
|
|
1716
|
+
inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1717
|
+
static_assert( //
|
|
1718
|
+
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1719
|
+
"Unexpected type");
|
|
1720
|
+
|
|
1721
|
+
computed_distances_count++;
|
|
1722
|
+
return metric(first, second);
|
|
1740
1723
|
}
|
|
1741
1724
|
};
|
|
1742
1725
|
|
|
1743
1726
|
index_config_t config_{};
|
|
1744
1727
|
index_limits_t limits_{};
|
|
1745
|
-
|
|
1746
|
-
mutable
|
|
1747
|
-
|
|
1728
|
+
|
|
1729
|
+
mutable dynamic_allocator_t dynamic_allocator_{};
|
|
1730
|
+
tape_allocator_t tape_allocator_{};
|
|
1731
|
+
|
|
1748
1732
|
precomputed_constants_t pre_{};
|
|
1749
|
-
|
|
1733
|
+
memory_mapped_file_t viewed_file_{};
|
|
1734
|
+
|
|
1735
|
+
/// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
|
|
1736
|
+
usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
|
|
1750
1737
|
|
|
1751
|
-
|
|
1752
|
-
usearch_align_m mutable std::atomic<std::size_t>
|
|
1738
|
+
/// @brief Number of "slots" already storing non-null nodes.
|
|
1739
|
+
usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
|
|
1753
1740
|
|
|
1754
|
-
/// @brief Controls access to `max_level_` and `
|
|
1741
|
+
/// @brief Controls access to `max_level_` and `entry_slot_`.
|
|
1755
1742
|
/// If any thread is updating those values, no other threads can `add()` or `search()`.
|
|
1756
1743
|
std::mutex global_mutex_{};
|
|
1744
|
+
|
|
1745
|
+
/// @brief The level of the top-most graph in the index. Grows as the logarithm of size, starts from zero.
|
|
1757
1746
|
level_t max_level_{};
|
|
1758
|
-
id_t entry_id_{};
|
|
1759
1747
|
|
|
1760
|
-
|
|
1761
|
-
|
|
1748
|
+
/// @brief The slot in which the only node of the top-level graph is stored.
|
|
1749
|
+
std::size_t entry_slot_{};
|
|
1750
|
+
|
|
1751
|
+
using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
|
|
1752
|
+
|
|
1753
|
+
/// @brief C-style array of `node_t` smart-pointers.
|
|
1754
|
+
buffer_gt<node_t, nodes_allocator_t> nodes_{};
|
|
1755
|
+
|
|
1756
|
+
/// @brief Mutex, that limits concurrent access to `nodes_`.
|
|
1762
1757
|
mutable visits_bitset_t nodes_mutexes_{};
|
|
1763
1758
|
|
|
1764
|
-
using
|
|
1765
|
-
|
|
1759
|
+
using contexts_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<context_t>;
|
|
1760
|
+
|
|
1761
|
+
/// @brief Array of thread-specific buffers for temporary data.
|
|
1762
|
+
mutable buffer_gt<context_t, contexts_allocator_t> contexts_{};
|
|
1766
1763
|
|
|
1767
1764
|
public:
|
|
1768
1765
|
std::size_t connectivity() const noexcept { return config_.connectivity; }
|
|
1769
|
-
std::size_t capacity() const noexcept { return
|
|
1770
|
-
std::size_t size() const noexcept { return
|
|
1766
|
+
std::size_t capacity() const noexcept { return nodes_capacity_; }
|
|
1767
|
+
std::size_t size() const noexcept { return nodes_count_; }
|
|
1771
1768
|
std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
|
|
1772
1769
|
index_config_t const& config() const noexcept { return config_; }
|
|
1773
1770
|
index_limits_t const& limits() const noexcept { return limits_; }
|
|
@@ -1777,16 +1774,17 @@ class index_gt {
|
|
|
1777
1774
|
* @section Exceptions
|
|
1778
1775
|
* Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
|
|
1779
1776
|
*/
|
|
1780
|
-
explicit index_gt(
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1777
|
+
explicit index_gt( //
|
|
1778
|
+
index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
|
|
1779
|
+
tape_allocator_t tape_allocator = {}) noexcept
|
|
1780
|
+
: config_(config), limits_(0, 0), dynamic_allocator_(std::move(dynamic_allocator)),
|
|
1781
|
+
tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), nodes_count_(0u), max_level_(-1),
|
|
1782
|
+
entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
|
|
1785
1783
|
|
|
1786
1784
|
/**
|
|
1787
1785
|
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
1788
1786
|
*/
|
|
1789
|
-
index_gt fork() noexcept { return index_gt{config_,
|
|
1787
|
+
index_gt fork() noexcept { return index_gt{config_, dynamic_allocator_, tape_allocator_}; }
|
|
1790
1788
|
|
|
1791
1789
|
~index_gt() noexcept { reset(); }
|
|
1792
1790
|
|
|
@@ -1808,21 +1806,21 @@ class index_gt {
|
|
|
1808
1806
|
}
|
|
1809
1807
|
};
|
|
1810
1808
|
|
|
1811
|
-
copy_result_t copy(
|
|
1809
|
+
copy_result_t copy(index_copy_config_t /*config*/ = {}) const noexcept {
|
|
1812
1810
|
copy_result_t result;
|
|
1813
1811
|
index_gt& other = result.index;
|
|
1814
|
-
other = index_gt(config_,
|
|
1812
|
+
other = index_gt(config_, dynamic_allocator_, tape_allocator_);
|
|
1815
1813
|
if (!other.reserve(limits_))
|
|
1816
1814
|
return result.failed("Failed to reserve the contexts");
|
|
1817
1815
|
|
|
1818
1816
|
// Now all is left - is to allocate new `node_t` instances and populate
|
|
1819
1817
|
// the `other.nodes_` array into it.
|
|
1820
|
-
for (std::size_t i = 0; i !=
|
|
1821
|
-
other.nodes_[i] = other.node_make_copy_(
|
|
1818
|
+
for (std::size_t i = 0; i != nodes_count_; ++i)
|
|
1819
|
+
other.nodes_[i] = other.node_make_copy_(node_bytes_(nodes_[i]));
|
|
1822
1820
|
|
|
1823
|
-
other.
|
|
1821
|
+
other.nodes_count_ = nodes_count_.load();
|
|
1824
1822
|
other.max_level_ = max_level_;
|
|
1825
|
-
other.
|
|
1823
|
+
other.entry_slot_ = entry_slot_;
|
|
1826
1824
|
return result;
|
|
1827
1825
|
}
|
|
1828
1826
|
|
|
@@ -1833,43 +1831,51 @@ class index_gt {
|
|
|
1833
1831
|
member_iterator_t begin() noexcept { return {this, 0}; }
|
|
1834
1832
|
member_iterator_t end() noexcept { return {this, size()}; }
|
|
1835
1833
|
|
|
1836
|
-
|
|
1837
|
-
|
|
1834
|
+
member_ref_t at(std::size_t slot) noexcept { return {nodes_[slot].key(), slot}; }
|
|
1835
|
+
member_cref_t at(std::size_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
|
|
1836
|
+
member_iterator_t iterator_at(std::size_t slot) noexcept { return {this, slot}; }
|
|
1837
|
+
member_citerator_t citerator_at(std::size_t slot) const noexcept { return {this, slot}; }
|
|
1838
|
+
|
|
1839
|
+
dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
|
|
1840
|
+
tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
|
|
1838
1841
|
|
|
1839
1842
|
#pragma region Adjusting Configuration
|
|
1840
1843
|
|
|
1841
1844
|
/**
|
|
1842
|
-
* @brief
|
|
1843
|
-
*
|
|
1844
|
-
*
|
|
1845
|
+
* @brief Erases all the vectors from the index.
|
|
1846
|
+
*
|
|
1847
|
+
* Will change `size()` to zero, but will keep the same `capacity()`.
|
|
1848
|
+
* Will keep the number of available threads/contexts the same as it was.
|
|
1845
1849
|
*/
|
|
1846
1850
|
void clear() noexcept {
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
+
if (!has_reset<tape_allocator_t>()) {
|
|
1852
|
+
std::size_t n = nodes_count_;
|
|
1853
|
+
for (std::size_t i = 0; i != n; ++i)
|
|
1854
|
+
node_free_(i);
|
|
1855
|
+
} else
|
|
1856
|
+
tape_allocator_.deallocate(nullptr, 0);
|
|
1857
|
+
nodes_count_ = 0;
|
|
1851
1858
|
max_level_ = -1;
|
|
1852
|
-
|
|
1859
|
+
entry_slot_ = 0u;
|
|
1853
1860
|
}
|
|
1854
1861
|
|
|
1855
1862
|
/**
|
|
1856
|
-
* @brief
|
|
1857
|
-
*
|
|
1858
|
-
*
|
|
1863
|
+
* @brief Erases all members from index, closing files, and returning RAM to OS.
|
|
1864
|
+
*
|
|
1865
|
+
* Will change both `size()` and `capacity()` to zero.
|
|
1866
|
+
* Will deallocate all threads/contexts.
|
|
1867
|
+
* If the index is memory-mapped - releases the mapping and the descriptor.
|
|
1859
1868
|
*/
|
|
1860
1869
|
void reset() noexcept {
|
|
1861
1870
|
clear();
|
|
1862
1871
|
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
for (std::size_t i = 0; i != limits_.threads(); ++i)
|
|
1867
|
-
contexts_[i].~context_t();
|
|
1868
|
-
context_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
|
|
1869
|
-
}
|
|
1872
|
+
nodes_ = {};
|
|
1873
|
+
contexts_ = {};
|
|
1874
|
+
nodes_mutexes_ = {};
|
|
1870
1875
|
limits_ = index_limits_t{0, 0};
|
|
1871
|
-
|
|
1872
|
-
|
|
1876
|
+
nodes_capacity_ = 0;
|
|
1877
|
+
viewed_file_ = memory_mapped_file_t{};
|
|
1878
|
+
tape_allocator_ = {};
|
|
1873
1879
|
}
|
|
1874
1880
|
|
|
1875
1881
|
/**
|
|
@@ -1878,24 +1884,23 @@ class index_gt {
|
|
|
1878
1884
|
void swap(index_gt& other) noexcept {
|
|
1879
1885
|
std::swap(config_, other.config_);
|
|
1880
1886
|
std::swap(limits_, other.limits_);
|
|
1881
|
-
std::swap(
|
|
1882
|
-
std::swap(
|
|
1883
|
-
std::swap(point_allocator_, other.point_allocator_);
|
|
1887
|
+
std::swap(dynamic_allocator_, other.dynamic_allocator_);
|
|
1888
|
+
std::swap(tape_allocator_, other.tape_allocator_);
|
|
1884
1889
|
std::swap(pre_, other.pre_);
|
|
1885
1890
|
std::swap(viewed_file_, other.viewed_file_);
|
|
1886
1891
|
std::swap(max_level_, other.max_level_);
|
|
1887
|
-
std::swap(
|
|
1892
|
+
std::swap(entry_slot_, other.entry_slot_);
|
|
1888
1893
|
std::swap(nodes_, other.nodes_);
|
|
1889
1894
|
std::swap(nodes_mutexes_, other.nodes_mutexes_);
|
|
1890
1895
|
std::swap(contexts_, other.contexts_);
|
|
1891
1896
|
|
|
1892
1897
|
// Non-atomic parts.
|
|
1893
|
-
std::size_t capacity_copy =
|
|
1894
|
-
std::size_t
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
other.
|
|
1898
|
-
other.
|
|
1898
|
+
std::size_t capacity_copy = nodes_capacity_;
|
|
1899
|
+
std::size_t count_copy = nodes_count_;
|
|
1900
|
+
nodes_capacity_ = other.nodes_capacity_.load();
|
|
1901
|
+
nodes_count_ = other.nodes_count_.load();
|
|
1902
|
+
other.nodes_capacity_ = capacity_copy;
|
|
1903
|
+
other.nodes_count_ = count_copy;
|
|
1899
1904
|
}
|
|
1900
1905
|
|
|
1901
1906
|
/**
|
|
@@ -1909,72 +1914,30 @@ class index_gt {
|
|
|
1909
1914
|
&& limits.members <= limits_.members)
|
|
1910
1915
|
return true;
|
|
1911
1916
|
|
|
1912
|
-
|
|
1917
|
+
visits_bitset_t new_mutexes(limits.members);
|
|
1918
|
+
buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
|
|
1919
|
+
buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
|
|
1920
|
+
if (!new_nodes || !new_contexts || !new_mutexes)
|
|
1913
1921
|
return false;
|
|
1914
1922
|
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
return false;
|
|
1919
|
-
|
|
1920
|
-
std::size_t limits_threads = limits.threads();
|
|
1921
|
-
context_allocator_t context_allocator;
|
|
1922
|
-
context_t* new_contexts = context_allocator.allocate(limits_threads);
|
|
1923
|
-
if (!new_contexts) {
|
|
1924
|
-
node_allocator.deallocate(new_nodes, limits.members);
|
|
1925
|
-
return false;
|
|
1926
|
-
}
|
|
1927
|
-
for (std::size_t i = 0; i != limits_threads; ++i) {
|
|
1928
|
-
context_t& context = new_contexts[i];
|
|
1929
|
-
new (&context) context_t();
|
|
1930
|
-
context.metric = metric_;
|
|
1931
|
-
if (!context.visits.resize(limits.members)) {
|
|
1932
|
-
for (std::size_t j = 0; j != i; ++j)
|
|
1933
|
-
context.visits.reset();
|
|
1934
|
-
node_allocator.deallocate(new_nodes, limits.members);
|
|
1935
|
-
context_allocator.deallocate(new_contexts, limits_threads);
|
|
1923
|
+
for (context_t& context : new_contexts) {
|
|
1924
|
+
context.visits = visits_bitset_t(limits.members);
|
|
1925
|
+
if (!context.visits)
|
|
1936
1926
|
return false;
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1939
|
-
|
|
1940
|
-
// We have passed all the require memory allocations.
|
|
1941
|
-
// The remaining code can't fail. Let's just reuse some of our existing buffers.
|
|
1942
|
-
for (std::size_t i = 0; i != limits_.threads(); ++i) {
|
|
1943
|
-
context_t& old_context = contexts_[i];
|
|
1944
|
-
context_t& context = new_contexts[i];
|
|
1945
|
-
std::swap(old_context.top_candidates, context.top_candidates);
|
|
1946
|
-
std::swap(old_context.next_candidates, context.next_candidates);
|
|
1947
|
-
std::swap(old_context.iteration_cycles, context.iteration_cycles);
|
|
1948
|
-
std::swap(old_context.measurements_count, context.measurements_count);
|
|
1949
|
-
old_context.visits.reset();
|
|
1950
1927
|
}
|
|
1951
1928
|
|
|
1952
1929
|
// Move the nodes info, and deallocate previous buffers.
|
|
1953
1930
|
if (nodes_)
|
|
1954
|
-
std::memcpy(new_nodes, nodes_, sizeof(node_t) * size())
|
|
1955
|
-
if (contexts_)
|
|
1956
|
-
context_allocator.deallocate(contexts_, limits_.threads());
|
|
1931
|
+
std::memcpy(new_nodes.data(), nodes_.data(), sizeof(node_t) * size());
|
|
1957
1932
|
|
|
1958
1933
|
limits_ = limits;
|
|
1959
|
-
|
|
1960
|
-
nodes_ = new_nodes;
|
|
1961
|
-
contexts_ = new_contexts;
|
|
1934
|
+
nodes_capacity_ = limits.members;
|
|
1935
|
+
nodes_ = std::move(new_nodes);
|
|
1936
|
+
contexts_ = std::move(new_contexts);
|
|
1937
|
+
nodes_mutexes_ = std::move(new_mutexes);
|
|
1962
1938
|
return true;
|
|
1963
1939
|
}
|
|
1964
1940
|
|
|
1965
|
-
/**
|
|
1966
|
-
* @brief Optimizes configuration options to fit the maximum number
|
|
1967
|
-
* of neighbors in CPU-cache-aligned buffers.
|
|
1968
|
-
*/
|
|
1969
|
-
static index_config_t optimize(index_config_t config) noexcept {
|
|
1970
|
-
precomputed_constants_t pre = precompute_(config);
|
|
1971
|
-
std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
|
|
1972
|
-
std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
|
|
1973
|
-
std::size_t added_connections = (rounded_size - bytes_per_node_base) / sizeof(id_t);
|
|
1974
|
-
config.connectivity = config.connectivity + added_connections / base_level_multiple_();
|
|
1975
|
-
return config;
|
|
1976
|
-
}
|
|
1977
|
-
|
|
1978
1941
|
#pragma endregion
|
|
1979
1942
|
|
|
1980
1943
|
#pragma region Construction and Search
|
|
@@ -1982,9 +1945,9 @@ class index_gt {
|
|
|
1982
1945
|
struct add_result_t {
|
|
1983
1946
|
error_t error{};
|
|
1984
1947
|
std::size_t new_size{};
|
|
1985
|
-
std::size_t
|
|
1986
|
-
std::size_t
|
|
1987
|
-
|
|
1948
|
+
std::size_t visited_members{};
|
|
1949
|
+
std::size_t computed_distances{};
|
|
1950
|
+
std::size_t slot{};
|
|
1988
1951
|
|
|
1989
1952
|
explicit operator bool() const noexcept { return !error; }
|
|
1990
1953
|
add_result_t failed(error_t message) noexcept {
|
|
@@ -1993,29 +1956,31 @@ class index_gt {
|
|
|
1993
1956
|
}
|
|
1994
1957
|
};
|
|
1995
1958
|
|
|
1959
|
+
/// @brief Describes a matched search result, augmenting `member_cref_t`
|
|
1960
|
+
/// contents with `distance` to the query object.
|
|
1996
1961
|
struct match_t {
|
|
1997
1962
|
member_cref_t member;
|
|
1998
1963
|
distance_t distance;
|
|
1999
1964
|
};
|
|
2000
1965
|
|
|
2001
1966
|
class search_result_t {
|
|
2002
|
-
|
|
2003
|
-
top_candidates_t
|
|
1967
|
+
node_t const* nodes_{};
|
|
1968
|
+
top_candidates_t const* top_{};
|
|
2004
1969
|
|
|
2005
1970
|
friend class index_gt;
|
|
2006
|
-
inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
|
|
1971
|
+
inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
|
|
1972
|
+
: nodes_(index.nodes_), top_(&top) {}
|
|
2007
1973
|
|
|
2008
1974
|
public:
|
|
1975
|
+
/** @brief Number of search results found. */
|
|
2009
1976
|
std::size_t count{};
|
|
2010
|
-
|
|
2011
|
-
std::size_t
|
|
1977
|
+
/** @brief Number of graph nodes traversed. */
|
|
1978
|
+
std::size_t visited_members{};
|
|
1979
|
+
/** @brief Number of times the distances were computed. */
|
|
1980
|
+
std::size_t computed_distances{};
|
|
2012
1981
|
error_t error{};
|
|
2013
1982
|
|
|
2014
|
-
|
|
2015
|
-
// given that search_result_t can represent a failed result (which implies no valid top_candidates_t &top_),
|
|
2016
|
-
// the top_ probably needs to be a pointer.
|
|
2017
|
-
inline search_result_t(index_gt const& index) noexcept
|
|
2018
|
-
: index_(index), top_(index.contexts_[0].top_candidates) {}
|
|
1983
|
+
inline search_result_t() noexcept {}
|
|
2019
1984
|
inline search_result_t(search_result_t&&) = default;
|
|
2020
1985
|
inline search_result_t& operator=(search_result_t&&) = default;
|
|
2021
1986
|
|
|
@@ -2031,46 +1996,88 @@ class index_gt {
|
|
|
2031
1996
|
inline match_t operator[](std::size_t i) const noexcept { return at(i); }
|
|
2032
1997
|
inline match_t front() const noexcept { return at(0); }
|
|
2033
1998
|
inline match_t back() const noexcept { return at(count - 1); }
|
|
2034
|
-
inline bool contains(
|
|
1999
|
+
inline bool contains(key_t key) const noexcept {
|
|
2035
2000
|
for (std::size_t i = 0; i != count; ++i)
|
|
2036
|
-
if (at(i).member.
|
|
2001
|
+
if (at(i).member.key == key)
|
|
2037
2002
|
return true;
|
|
2038
2003
|
return false;
|
|
2039
2004
|
}
|
|
2040
2005
|
inline match_t at(std::size_t i) const noexcept {
|
|
2041
|
-
candidate_t const* top_ordered = top_
|
|
2006
|
+
candidate_t const* top_ordered = top_->data();
|
|
2042
2007
|
candidate_t candidate = top_ordered[i];
|
|
2043
|
-
node_t node =
|
|
2044
|
-
return {member_cref_t{node.
|
|
2008
|
+
node_t node = nodes_[candidate.slot];
|
|
2009
|
+
return {member_cref_t{node.ckey(), candidate.slot}, candidate.distance};
|
|
2010
|
+
}
|
|
2011
|
+
inline std::size_t merge_into( //
|
|
2012
|
+
key_t* keys, distance_t* distances, //
|
|
2013
|
+
std::size_t old_count, std::size_t max_count) const noexcept {
|
|
2014
|
+
|
|
2015
|
+
std::size_t merged_count = old_count;
|
|
2016
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
2017
|
+
match_t result = operator[](i);
|
|
2018
|
+
distance_t* merged_end = distances + merged_count;
|
|
2019
|
+
std::size_t offset = std::lower_bound(distances, merged_end, result.distance) - distances;
|
|
2020
|
+
if (offset == max_count)
|
|
2021
|
+
continue;
|
|
2022
|
+
|
|
2023
|
+
std::size_t count_worse = merged_count - offset - (max_count == merged_count);
|
|
2024
|
+
std::memmove(keys + offset + 1, keys + offset, count_worse * sizeof(key_t));
|
|
2025
|
+
std::memmove(distances + offset + 1, distances + offset, count_worse * sizeof(distance_t));
|
|
2026
|
+
keys[offset] = result.member.key;
|
|
2027
|
+
distances[offset] = result.distance;
|
|
2028
|
+
merged_count += merged_count != max_count;
|
|
2029
|
+
}
|
|
2030
|
+
return merged_count;
|
|
2045
2031
|
}
|
|
2046
|
-
inline std::size_t dump_to(
|
|
2032
|
+
inline std::size_t dump_to(key_t* keys, distance_t* distances) const noexcept {
|
|
2047
2033
|
for (std::size_t i = 0; i != count; ++i) {
|
|
2048
2034
|
match_t result = operator[](i);
|
|
2049
|
-
|
|
2035
|
+
keys[i] = result.member.key;
|
|
2050
2036
|
distances[i] = result.distance;
|
|
2051
2037
|
}
|
|
2052
2038
|
return count;
|
|
2053
2039
|
}
|
|
2054
|
-
inline std::size_t dump_to(
|
|
2040
|
+
inline std::size_t dump_to(key_t* keys) const noexcept {
|
|
2055
2041
|
for (std::size_t i = 0; i != count; ++i) {
|
|
2056
2042
|
match_t result = operator[](i);
|
|
2057
|
-
|
|
2043
|
+
keys[i] = result.member.key;
|
|
2058
2044
|
}
|
|
2059
2045
|
return count;
|
|
2060
2046
|
}
|
|
2061
2047
|
};
|
|
2062
2048
|
|
|
2063
2049
|
/**
|
|
2064
|
-
* @brief
|
|
2050
|
+
* @brief Inserts a new entry into the index. Thread-safe. Supports @b heterogeneous lookups.
|
|
2051
|
+
* Expects needed capacity to be reserved ahead of time: `size() < capacity()`.
|
|
2052
|
+
*
|
|
2053
|
+
* @tparam metric_at
|
|
2054
|
+
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2055
|
+
* It should be callable into distinctly different scenarios:
|
|
2056
|
+
* - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
|
|
2057
|
+
* - `distance_t operator() (entry_at, entry_at)` - between existing entries.
|
|
2058
|
+
* Where any possible `entry_at` has both two interfaces: `std::size_t slot()`, `key_t key()`.
|
|
2065
2059
|
*
|
|
2066
|
-
* @param[in]
|
|
2067
|
-
* @param[in]
|
|
2060
|
+
* @param[in] key External identifier/name/descriptor for the new entry.
|
|
2061
|
+
* @param[in] value Content that will be compared against other entries to index.
|
|
2062
|
+
* @param[in] metric Callable object measuring distance between ::value and present objects.
|
|
2068
2063
|
* @param[in] config Configuration options for this specific operation.
|
|
2064
|
+
* @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
|
|
2069
2065
|
*/
|
|
2070
|
-
|
|
2066
|
+
template < //
|
|
2067
|
+
typename value_at, //
|
|
2068
|
+
typename metric_at, //
|
|
2069
|
+
typename callback_at = dummy_callback_t, //
|
|
2070
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2071
|
+
>
|
|
2072
|
+
add_result_t add( //
|
|
2073
|
+
key_t key, value_at&& value, metric_at&& metric, //
|
|
2074
|
+
index_update_config_t config = {}, //
|
|
2075
|
+
callback_at&& callback = callback_at{}, //
|
|
2076
|
+
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2071
2077
|
|
|
2072
|
-
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
2073
2078
|
add_result_t result;
|
|
2079
|
+
if (is_immutable())
|
|
2080
|
+
return result.failed("Can't add to an immutable index");
|
|
2074
2081
|
|
|
2075
2082
|
// Make sure we have enough local memory to perform this request
|
|
2076
2083
|
context_t& context = contexts_[config.thread];
|
|
@@ -2081,7 +2088,8 @@ class index_gt {
|
|
|
2081
2088
|
|
|
2082
2089
|
// The top list needs one more slot than the connectivity of the base level
|
|
2083
2090
|
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
2084
|
-
std::size_t
|
|
2091
|
+
std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
|
|
2092
|
+
std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
|
|
2085
2093
|
if (!top.reserve(top_limit))
|
|
2086
2094
|
return result.failed("Out of memory!");
|
|
2087
2095
|
if (!next.reserve(config.expansion))
|
|
@@ -2089,60 +2097,98 @@ class index_gt {
|
|
|
2089
2097
|
|
|
2090
2098
|
// Determining how much memory to allocate for the node depends on the target level
|
|
2091
2099
|
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
2092
|
-
level_t max_level_copy = max_level_;
|
|
2093
|
-
|
|
2100
|
+
level_t max_level_copy = max_level_; // Copy under lock
|
|
2101
|
+
std::size_t entry_idx_copy = entry_slot_; // Copy under lock
|
|
2094
2102
|
level_t target_level = choose_random_level_(context.level_generator);
|
|
2095
|
-
|
|
2096
|
-
|
|
2103
|
+
|
|
2104
|
+
// Make sure we are not overflowing
|
|
2105
|
+
std::size_t capacity = nodes_capacity_.load();
|
|
2106
|
+
std::size_t new_slot = nodes_count_.fetch_add(1);
|
|
2107
|
+
if (new_slot >= capacity) {
|
|
2108
|
+
nodes_count_.fetch_sub(1);
|
|
2109
|
+
return result.failed("Reserve capacity ahead of insertions!");
|
|
2110
|
+
}
|
|
2097
2111
|
|
|
2098
2112
|
// Allocate the neighbors
|
|
2099
|
-
node_t node = node_make_(
|
|
2100
|
-
if (!node)
|
|
2113
|
+
node_t node = node_make_(key, target_level);
|
|
2114
|
+
if (!node) {
|
|
2115
|
+
nodes_count_.fetch_sub(1);
|
|
2101
2116
|
return result.failed("Out of memory!");
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2117
|
+
}
|
|
2118
|
+
if (target_level <= max_level_copy)
|
|
2119
|
+
new_level_lock.unlock();
|
|
2120
|
+
|
|
2121
|
+
nodes_[new_slot] = node;
|
|
2122
|
+
result.new_size = new_slot + 1;
|
|
2123
|
+
result.slot = new_slot;
|
|
2124
|
+
callback(at(new_slot));
|
|
2125
|
+
node_lock_t new_lock = node_lock_(new_slot);
|
|
2108
2126
|
|
|
2109
2127
|
// Do nothing for the first element
|
|
2110
|
-
if (!
|
|
2111
|
-
|
|
2128
|
+
if (!new_slot) {
|
|
2129
|
+
entry_slot_ = new_slot;
|
|
2112
2130
|
max_level_ = target_level;
|
|
2113
2131
|
return result;
|
|
2114
2132
|
}
|
|
2115
2133
|
|
|
2116
2134
|
// Pull stats
|
|
2117
|
-
result.
|
|
2118
|
-
result.
|
|
2135
|
+
result.computed_distances = context.computed_distances_count;
|
|
2136
|
+
result.visited_members = context.iteration_cycles;
|
|
2119
2137
|
|
|
2120
|
-
connect_node_across_levels_(
|
|
2138
|
+
connect_node_across_levels_( //
|
|
2139
|
+
value, metric, prefetch, //
|
|
2140
|
+
new_slot, entry_idx_copy, max_level_copy, target_level, //
|
|
2141
|
+
config, context);
|
|
2121
2142
|
|
|
2122
2143
|
// Normalize stats
|
|
2123
|
-
result.
|
|
2124
|
-
result.
|
|
2144
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2145
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2125
2146
|
|
|
2126
2147
|
// Updating the entry point if needed
|
|
2127
2148
|
if (target_level > max_level_copy) {
|
|
2128
|
-
|
|
2149
|
+
entry_slot_ = new_slot;
|
|
2129
2150
|
max_level_ = target_level;
|
|
2130
2151
|
}
|
|
2131
2152
|
return result;
|
|
2132
2153
|
}
|
|
2133
2154
|
|
|
2134
2155
|
/**
|
|
2135
|
-
* @brief
|
|
2156
|
+
* @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
|
|
2136
2157
|
*
|
|
2137
|
-
* @
|
|
2138
|
-
*
|
|
2139
|
-
*
|
|
2158
|
+
* @tparam metric_at
|
|
2159
|
+
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2160
|
+
* It should be callable into distinctly different scenarios:
|
|
2161
|
+
* - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
|
|
2162
|
+
* - `distance_t operator() (entry_at, entry_at)` - between existing entries.
|
|
2163
|
+
* For any possible `entry_at` following interfaces will work:
|
|
2164
|
+
* - `std::size_t get_slot(entry_at const &)`
|
|
2165
|
+
* - `key_t get_key(entry_at const &)`
|
|
2166
|
+
*
|
|
2167
|
+
* @param[in] iterator Iterator pointing to an existing entry to be replaced.
|
|
2168
|
+
* @param[in] key External identifier/name/descriptor for the entry.
|
|
2169
|
+
* @param[in] value Content that will be compared against other entries in the index.
|
|
2170
|
+
* @param[in] metric Callable object measuring distance between ::value and present objects.
|
|
2140
2171
|
* @param[in] config Configuration options for this specific operation.
|
|
2172
|
+
* @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
|
|
2141
2173
|
*/
|
|
2142
|
-
|
|
2174
|
+
template < //
|
|
2175
|
+
typename value_at, //
|
|
2176
|
+
typename metric_at, //
|
|
2177
|
+
typename callback_at = dummy_callback_t, //
|
|
2178
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2179
|
+
>
|
|
2180
|
+
add_result_t update( //
|
|
2181
|
+
member_iterator_t iterator, //
|
|
2182
|
+
key_t key, //
|
|
2183
|
+
value_at&& value, //
|
|
2184
|
+
metric_at&& metric, //
|
|
2185
|
+
index_update_config_t config = {}, //
|
|
2186
|
+
callback_at&& callback = callback_at{}, //
|
|
2187
|
+
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2143
2188
|
|
|
2144
2189
|
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
2145
2190
|
add_result_t result;
|
|
2191
|
+
std::size_t old_slot = iterator.slot_;
|
|
2146
2192
|
|
|
2147
2193
|
// Make sure we have enough local memory to perform this request
|
|
2148
2194
|
context_t& context = contexts_[config.thread];
|
|
@@ -2153,57 +2199,77 @@ class index_gt {
|
|
|
2153
2199
|
|
|
2154
2200
|
// The top list needs one more slot than the connectivity of the base level
|
|
2155
2201
|
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
2156
|
-
std::size_t
|
|
2202
|
+
std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
|
|
2203
|
+
std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
|
|
2157
2204
|
if (!top.reserve(top_limit))
|
|
2158
2205
|
return result.failed("Out of memory!");
|
|
2159
2206
|
if (!next.reserve(config.expansion))
|
|
2160
2207
|
return result.failed("Out of memory!");
|
|
2161
2208
|
|
|
2162
|
-
node_lock_t new_lock = node_lock_(
|
|
2163
|
-
node_t node =
|
|
2209
|
+
node_lock_t new_lock = node_lock_(old_slot);
|
|
2210
|
+
node_t node = node_at_(old_slot);
|
|
2211
|
+
|
|
2212
|
+
level_t node_level = node.level();
|
|
2213
|
+
span_bytes_t node_bytes = node_bytes_(node);
|
|
2214
|
+
std::memset(node_bytes.data(), 0, node_bytes.size());
|
|
2215
|
+
node.level(node_level);
|
|
2164
2216
|
|
|
2165
2217
|
// Pull stats
|
|
2166
|
-
result.
|
|
2167
|
-
result.
|
|
2218
|
+
result.computed_distances = context.computed_distances_count;
|
|
2219
|
+
result.visited_members = context.iteration_cycles;
|
|
2168
2220
|
|
|
2169
|
-
connect_node_across_levels_(
|
|
2170
|
-
|
|
2221
|
+
connect_node_across_levels_( //
|
|
2222
|
+
value, metric, prefetch, //
|
|
2223
|
+
old_slot, entry_slot_, max_level_, node_level, //
|
|
2224
|
+
config, context);
|
|
2225
|
+
node.key(key);
|
|
2171
2226
|
|
|
2172
2227
|
// Normalize stats
|
|
2173
|
-
result.
|
|
2174
|
-
result.
|
|
2228
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2229
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2230
|
+
result.slot = old_slot;
|
|
2175
2231
|
|
|
2232
|
+
callback(at(old_slot));
|
|
2176
2233
|
return result;
|
|
2177
2234
|
}
|
|
2178
2235
|
|
|
2179
2236
|
/**
|
|
2180
2237
|
* @brief Searches for the closest elements to the given ::query. Thread-safe.
|
|
2181
2238
|
*
|
|
2182
|
-
* @param[in] query
|
|
2239
|
+
* @param[in] query Content that will be compared against other entries in the index.
|
|
2183
2240
|
* @param[in] wanted The upper bound for the number of results to return.
|
|
2184
2241
|
* @param[in] config Configuration options for this specific operation.
|
|
2185
2242
|
* @param[in] predicate Optional filtering predicate for `member_cref_t`.
|
|
2186
2243
|
* @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
|
|
2187
2244
|
*/
|
|
2188
|
-
template <
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
predicate_at
|
|
2245
|
+
template < //
|
|
2246
|
+
typename value_at, //
|
|
2247
|
+
typename metric_at, //
|
|
2248
|
+
typename predicate_at = dummy_predicate_t, //
|
|
2249
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2250
|
+
>
|
|
2251
|
+
search_result_t search( //
|
|
2252
|
+
value_at&& query, //
|
|
2253
|
+
std::size_t wanted, //
|
|
2254
|
+
metric_at&& metric, //
|
|
2255
|
+
index_search_config_t config = {}, //
|
|
2256
|
+
predicate_at&& predicate = predicate_at{}, //
|
|
2257
|
+
prefetch_at&& prefetch = prefetch_at{}) const noexcept {
|
|
2192
2258
|
|
|
2193
2259
|
context_t& context = contexts_[config.thread];
|
|
2194
2260
|
top_candidates_t& top = context.top_candidates;
|
|
2195
2261
|
search_result_t result{*this, top};
|
|
2196
|
-
if (!
|
|
2262
|
+
if (!nodes_count_)
|
|
2197
2263
|
return result;
|
|
2198
2264
|
|
|
2199
2265
|
// Go down the level, tracking only the closest match
|
|
2200
|
-
result.
|
|
2201
|
-
result.
|
|
2266
|
+
result.computed_distances = context.computed_distances_count;
|
|
2267
|
+
result.visited_members = context.iteration_cycles;
|
|
2202
2268
|
|
|
2203
2269
|
if (config.exact) {
|
|
2204
2270
|
if (!top.reserve(wanted))
|
|
2205
2271
|
return result.failed("Out of memory!");
|
|
2206
|
-
search_exact_(query, wanted, context
|
|
2272
|
+
search_exact_(query, metric, predicate, wanted, context);
|
|
2207
2273
|
} else {
|
|
2208
2274
|
next_candidates_t& next = context.next_candidates;
|
|
2209
2275
|
std::size_t expansion = (std::max)(config.expansion, wanted);
|
|
@@ -2212,9 +2278,10 @@ class index_gt {
|
|
|
2212
2278
|
if (!top.reserve(expansion))
|
|
2213
2279
|
return result.failed("Out of memory!");
|
|
2214
2280
|
|
|
2215
|
-
|
|
2281
|
+
std::size_t closest_slot = search_for_one_(query, metric, prefetch, entry_slot_, max_level_, 0, context);
|
|
2282
|
+
|
|
2216
2283
|
// For bottom layer we need a more optimized procedure
|
|
2217
|
-
if (!search_to_find_in_base_(
|
|
2284
|
+
if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
|
|
2218
2285
|
return result.failed("Out of memory!");
|
|
2219
2286
|
}
|
|
2220
2287
|
|
|
@@ -2222,8 +2289,8 @@ class index_gt {
|
|
|
2222
2289
|
top.shrink(wanted);
|
|
2223
2290
|
|
|
2224
2291
|
// Normalize stats
|
|
2225
|
-
result.
|
|
2226
|
-
result.
|
|
2292
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2293
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2227
2294
|
result.count = top.size();
|
|
2228
2295
|
return result;
|
|
2229
2296
|
}
|
|
@@ -2243,13 +2310,13 @@ class index_gt {
|
|
|
2243
2310
|
stats_t result{};
|
|
2244
2311
|
result.nodes = size();
|
|
2245
2312
|
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
2246
|
-
node_t node =
|
|
2247
|
-
std::size_t max_edges = node.level() * config_.connectivity +
|
|
2313
|
+
node_t node = node_at_(i);
|
|
2314
|
+
std::size_t max_edges = node.level() * config_.connectivity + config_.connectivity_base;
|
|
2248
2315
|
std::size_t edges = 0;
|
|
2249
2316
|
for (level_t level = 0; level <= node.level(); ++level)
|
|
2250
2317
|
edges += neighbors_(node, level).size();
|
|
2251
2318
|
|
|
2252
|
-
result.allocated_bytes += node_bytes_(node);
|
|
2319
|
+
result.allocated_bytes += node_bytes_(node).size();
|
|
2253
2320
|
result.edges += edges;
|
|
2254
2321
|
result.max_edges += max_edges;
|
|
2255
2322
|
}
|
|
@@ -2262,15 +2329,15 @@ class index_gt {
|
|
|
2262
2329
|
|
|
2263
2330
|
std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
|
|
2264
2331
|
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
2265
|
-
node_t node =
|
|
2332
|
+
node_t node = node_at_(i);
|
|
2266
2333
|
if (static_cast<std::size_t>(node.level()) < level)
|
|
2267
2334
|
continue;
|
|
2268
2335
|
|
|
2269
2336
|
result.edges += neighbors_(node, level).size();
|
|
2270
|
-
result.allocated_bytes += node_head_bytes_() +
|
|
2337
|
+
result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
|
|
2271
2338
|
}
|
|
2272
2339
|
|
|
2273
|
-
std::size_t max_edges_per_node = level ? config_.
|
|
2340
|
+
std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
|
|
2274
2341
|
result.max_edges = result.nodes * max_edges_per_node;
|
|
2275
2342
|
return result;
|
|
2276
2343
|
}
|
|
@@ -2278,6 +2345,8 @@ class index_gt {
|
|
|
2278
2345
|
/**
|
|
2279
2346
|
* @brief A relatively accurate lower bound on the amount of memory consumed by the system.
|
|
2280
2347
|
* In practice it's error will be below 10%.
|
|
2348
|
+
*
|
|
2349
|
+
* @see `stream_length` for the length of the binary serialized representation.
|
|
2281
2350
|
*/
|
|
2282
2351
|
std::size_t memory_usage(std::size_t allocator_entry_bytes = default_allocator_entry_bytes()) const noexcept {
|
|
2283
2352
|
std::size_t total = 0;
|
|
@@ -2295,358 +2364,297 @@ class index_gt {
|
|
|
2295
2364
|
return total;
|
|
2296
2365
|
}
|
|
2297
2366
|
|
|
2298
|
-
std::size_t memory_usage_per_node(
|
|
2299
|
-
|
|
2300
|
-
void change_metric(metric_t const& m) noexcept {
|
|
2301
|
-
metric_ = m;
|
|
2302
|
-
for (std::size_t i = 0; i != limits_.threads(); ++i)
|
|
2303
|
-
contexts_[i].metric = metric_;
|
|
2304
|
-
}
|
|
2367
|
+
std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
|
|
2305
2368
|
|
|
2306
2369
|
#pragma endregion
|
|
2307
2370
|
|
|
2308
2371
|
#pragma region Serialization
|
|
2309
2372
|
|
|
2310
|
-
struct serialization_result_t {
|
|
2311
|
-
error_t error;
|
|
2312
|
-
|
|
2313
|
-
explicit operator bool() const noexcept { return !error; }
|
|
2314
|
-
serialization_result_t failed(error_t message) noexcept {
|
|
2315
|
-
error = std::move(message);
|
|
2316
|
-
return std::move(*this);
|
|
2317
|
-
}
|
|
2318
|
-
};
|
|
2319
|
-
|
|
2320
2373
|
/**
|
|
2321
|
-
* @brief Saves serialized binary index representation to disk
|
|
2322
|
-
* co-locating vectors and neighbors lists.
|
|
2323
|
-
* Available on Linux, MacOS, Windows.
|
|
2374
|
+
* @brief Saves serialized binary index representation to a file, generally on disk.
|
|
2324
2375
|
*/
|
|
2325
2376
|
template <typename progress_at = dummy_progress_t>
|
|
2326
|
-
serialization_result_t save(
|
|
2377
|
+
serialization_result_t save(output_file_t file, progress_at&& progress = {}) const noexcept {
|
|
2378
|
+
|
|
2379
|
+
serialization_result_t result = file.open_if_not();
|
|
2380
|
+
if (result)
|
|
2381
|
+
stream(
|
|
2382
|
+
[&](void* buffer, std::size_t length) {
|
|
2383
|
+
result = file.write(buffer, length);
|
|
2384
|
+
return !!result;
|
|
2385
|
+
},
|
|
2386
|
+
std::forward<progress_at>(progress));
|
|
2387
|
+
return result;
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
/**
|
|
2391
|
+
* @brief Saves serialized binary index representation to a stream.
|
|
2392
|
+
*/
|
|
2393
|
+
template <typename output_callback_at, typename progress_at = dummy_progress_t>
|
|
2394
|
+
serialization_result_t stream(output_callback_at&& callback, progress_at&& progress = {}) const noexcept {
|
|
2327
2395
|
|
|
2328
|
-
// Make sure we have right to write to that file
|
|
2329
2396
|
serialization_result_t result;
|
|
2330
|
-
std::FILE* file = std::fopen(file_path, "wb");
|
|
2331
|
-
if (!file)
|
|
2332
|
-
return result.failed(std::strerror(errno));
|
|
2333
2397
|
|
|
2334
|
-
//
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
//
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
state.scalar_kind = metric_.scalar_kind();
|
|
2353
|
-
state.size = size_;
|
|
2354
|
-
state.entry_idx = entry_id_;
|
|
2355
|
-
|
|
2356
|
-
// Augment with metadata
|
|
2357
|
-
std::size_t graphs_bytes = 0;
|
|
2358
|
-
std::size_t vectors_bytes = 0;
|
|
2359
|
-
for (std::size_t i = 0; i != state.size; ++i) {
|
|
2360
|
-
node_t node = node_with_id_(i);
|
|
2361
|
-
std::size_t node_bytes = node_bytes_(node);
|
|
2362
|
-
std::size_t node_vector_bytes = node_vector_bytes_(node);
|
|
2363
|
-
graphs_bytes += node_bytes - node_vector_bytes;
|
|
2364
|
-
vectors_bytes += node_vector_bytes;
|
|
2398
|
+
// Export some basic metadata
|
|
2399
|
+
index_serialized_header_t header;
|
|
2400
|
+
header.size = nodes_count_;
|
|
2401
|
+
header.connectivity = config_.connectivity;
|
|
2402
|
+
header.connectivity_base = config_.connectivity_base;
|
|
2403
|
+
header.max_level = max_level_;
|
|
2404
|
+
header.entry_slot = entry_slot_;
|
|
2405
|
+
if (!callback(&header, sizeof(header)))
|
|
2406
|
+
return result.failed("Failed to serialize into stream");
|
|
2407
|
+
|
|
2408
|
+
// Export the number of levels per node
|
|
2409
|
+
// That is both enough to estimate the overall memory consumption,
|
|
2410
|
+
// and to be able to estimate the offsets of every entry in the file.
|
|
2411
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2412
|
+
node_t node = node_at_(i);
|
|
2413
|
+
level_t level = node.level();
|
|
2414
|
+
if (!callback(&level, sizeof(level)))
|
|
2415
|
+
return result.failed("Failed to serialize into stream");
|
|
2365
2416
|
}
|
|
2366
|
-
state.bytes_for_graphs = graphs_bytes;
|
|
2367
|
-
state.bytes_for_vectors = vectors_bytes;
|
|
2368
|
-
state.bytes_checksum = 0;
|
|
2369
|
-
|
|
2370
|
-
// Perform serialization
|
|
2371
|
-
auto write_chunk = [&](void* begin, std::size_t length) {
|
|
2372
|
-
std::size_t written = std::fwrite(begin, length, 1, file);
|
|
2373
|
-
if (!written) {
|
|
2374
|
-
std::fclose(file);
|
|
2375
|
-
result.failed(std::strerror(errno));
|
|
2376
|
-
}
|
|
2377
|
-
};
|
|
2378
|
-
|
|
2379
|
-
// Write the header
|
|
2380
|
-
write_chunk(&state_buffer[0], sizeof(file_header_t));
|
|
2381
|
-
if (result.error)
|
|
2382
|
-
return result;
|
|
2383
2417
|
|
|
2384
|
-
//
|
|
2385
|
-
for (std::size_t i = 0; i !=
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
write_chunk(node.tape(), node_bytes - node_vector_bytes);
|
|
2391
|
-
if (result.error)
|
|
2392
|
-
return result;
|
|
2393
|
-
write_chunk(node.vector(), node_vector_bytes);
|
|
2394
|
-
if (result.error)
|
|
2395
|
-
return result;
|
|
2396
|
-
progress(i, state.size);
|
|
2418
|
+
// After that dump the nodes themselves
|
|
2419
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2420
|
+
span_bytes_t node_bytes = node_bytes_(node_at_(i));
|
|
2421
|
+
if (!callback(node_bytes.data(), node_bytes.size()))
|
|
2422
|
+
return result.failed("Failed to serialize into stream");
|
|
2423
|
+
progress(i, header.size);
|
|
2397
2424
|
}
|
|
2398
2425
|
|
|
2399
|
-
std::fclose(file);
|
|
2400
2426
|
return {};
|
|
2401
2427
|
}
|
|
2402
2428
|
|
|
2403
2429
|
/**
|
|
2404
|
-
* @brief
|
|
2405
|
-
* copying both vectors and neighbors lists into RAM.
|
|
2406
|
-
* Available on Linux, MacOS, Windows.
|
|
2430
|
+
* @brief Estimate the binary length (in bytes) of the serialized index.
|
|
2407
2431
|
*/
|
|
2408
|
-
|
|
2409
|
-
|
|
2432
|
+
std::size_t stream_length() const noexcept {
|
|
2433
|
+
std::size_t neighbors_length = 0;
|
|
2434
|
+
for (std::size_t i = 0; i != size(); ++i)
|
|
2435
|
+
neighbors_length += node_bytes_(node_at_(i).level());
|
|
2436
|
+
return sizeof(index_serialized_header_t) + neighbors_length;
|
|
2437
|
+
}
|
|
2410
2438
|
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2439
|
+
/**
|
|
2440
|
+
* @brief Loads the serialized binary index representation from disk to RAM.
|
|
2441
|
+
* Adjusts the configuration properties of the constructed index to
|
|
2442
|
+
* match the settings in the file.
|
|
2443
|
+
*/
|
|
2444
|
+
template <typename progress_at = dummy_progress_t>
|
|
2445
|
+
serialization_result_t load(input_file_t file, progress_at&& progress = {}) noexcept {
|
|
2416
2446
|
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
if (!read) {
|
|
2420
|
-
std::fclose(file);
|
|
2421
|
-
result.failed(std::strerror(errno));
|
|
2422
|
-
}
|
|
2423
|
-
};
|
|
2447
|
+
// Remove previously stored objects
|
|
2448
|
+
reset();
|
|
2424
2449
|
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
if (result.error)
|
|
2429
|
-
return result;
|
|
2450
|
+
serialization_result_t result = file.open_if_not();
|
|
2451
|
+
if (!result)
|
|
2452
|
+
return result;
|
|
2430
2453
|
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
if (state.bytes_per_id != sizeof(id_t)) {
|
|
2437
|
-
std::fclose(file);
|
|
2438
|
-
return result.failed("Incompatible ID type!");
|
|
2439
|
-
}
|
|
2454
|
+
// Pull basic metadata
|
|
2455
|
+
index_serialized_header_t header;
|
|
2456
|
+
result = file.read(&header, sizeof(header));
|
|
2457
|
+
if (!result)
|
|
2458
|
+
return result;
|
|
2440
2459
|
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2460
|
+
// Allocate some dynamic memory to read all the levels
|
|
2461
|
+
using levels_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<level_t>;
|
|
2462
|
+
buffer_gt<level_t, levels_allocator_t> levels(header.size);
|
|
2463
|
+
if (!levels)
|
|
2464
|
+
return result.failed("Out of memory");
|
|
2465
|
+
result = file.read(levels, header.size * sizeof(level_t));
|
|
2466
|
+
if (!result)
|
|
2467
|
+
return result;
|
|
2444
2468
|
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2469
|
+
// Submit metadata
|
|
2470
|
+
config_.connectivity = header.connectivity;
|
|
2471
|
+
config_.connectivity_base = header.connectivity_base;
|
|
2472
|
+
pre_ = precompute_(config_);
|
|
2473
|
+
index_limits_t limits;
|
|
2474
|
+
limits.members = header.size;
|
|
2475
|
+
if (!reserve(limits)) {
|
|
2476
|
+
reset();
|
|
2477
|
+
return result.failed("Out of memory");
|
|
2454
2478
|
}
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
if (result
|
|
2464
|
-
|
|
2465
|
-
read_chunk(&dim, sizeof(dim));
|
|
2466
|
-
if (result.error)
|
|
2479
|
+
nodes_count_ = header.size;
|
|
2480
|
+
max_level_ = static_cast<level_t>(header.max_level);
|
|
2481
|
+
entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
|
|
2482
|
+
|
|
2483
|
+
// Load the nodes
|
|
2484
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2485
|
+
span_bytes_t node_bytes = node_malloc_(levels[i]);
|
|
2486
|
+
result = file.read(node_bytes.data(), node_bytes.size());
|
|
2487
|
+
if (!result) {
|
|
2488
|
+
reset();
|
|
2467
2489
|
return result;
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
std::size_t node_bytes = node_bytes_(dim, level);
|
|
2473
|
-
node_t node = node_malloc_(dim, level);
|
|
2474
|
-
node.label(label);
|
|
2475
|
-
node.dim(dim);
|
|
2476
|
-
node.level(level);
|
|
2477
|
-
read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
|
|
2478
|
-
if (result.error)
|
|
2479
|
-
return result;
|
|
2480
|
-
nodes_[i] = node;
|
|
2481
|
-
progress(i, size);
|
|
2490
|
+
}
|
|
2491
|
+
nodes_[i] = node_t{node_bytes.data()};
|
|
2492
|
+
progress(i, header.size);
|
|
2482
2493
|
}
|
|
2483
|
-
|
|
2484
|
-
std::fclose(file);
|
|
2485
|
-
reset_view_();
|
|
2486
2494
|
return {};
|
|
2487
2495
|
}
|
|
2488
2496
|
|
|
2489
2497
|
/**
|
|
2490
2498
|
* @brief Memory-maps the serialized binary index representation from disk,
|
|
2491
|
-
* @b without copying
|
|
2492
|
-
* Available on Linux, MacOS, but @b not on Windows.
|
|
2499
|
+
* @b without copying data into RAM, and fetching it on-demand.
|
|
2493
2500
|
*/
|
|
2494
2501
|
template <typename progress_at = dummy_progress_t>
|
|
2495
|
-
serialization_result_t view(
|
|
2496
|
-
|
|
2502
|
+
serialization_result_t view(memory_mapped_file_t file, std::size_t offset = 0,
|
|
2503
|
+
progress_at&& progress = {}) noexcept {
|
|
2497
2504
|
|
|
2498
|
-
|
|
2505
|
+
// Remove previously stored objects
|
|
2506
|
+
reset();
|
|
2499
2507
|
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
return result.failed("Opening file failed!");
|
|
2508
|
+
serialization_result_t result = file.open_if_not();
|
|
2509
|
+
if (!result)
|
|
2510
|
+
return result;
|
|
2504
2511
|
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
if (
|
|
2508
|
-
|
|
2509
|
-
|
|
2512
|
+
// Pull basic metadata
|
|
2513
|
+
index_serialized_header_t header;
|
|
2514
|
+
if (file.size() - offset < sizeof(header))
|
|
2515
|
+
return result.failed("File is corrupted and lacks a header");
|
|
2516
|
+
std::memcpy(&header, file.data() + offset, sizeof(header));
|
|
2517
|
+
|
|
2518
|
+
// Precompute offsets of every node, but before that we need to update the configs
|
|
2519
|
+
// This could have been done with `std::exclusive_scan`, but it's only available from C++17.
|
|
2520
|
+
using offsets_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
|
|
2521
|
+
buffer_gt<std::size_t, offsets_allocator_t> offsets(header.size);
|
|
2522
|
+
if (!offsets)
|
|
2523
|
+
return result.failed("Out of memory");
|
|
2524
|
+
|
|
2525
|
+
config_.connectivity = header.connectivity;
|
|
2526
|
+
config_.connectivity_base = header.connectivity_base;
|
|
2527
|
+
pre_ = precompute_(config_);
|
|
2528
|
+
misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
|
|
2529
|
+
offsets[0] = offset + sizeof(header) + sizeof(level_t) * header.size;
|
|
2530
|
+
for (std::size_t i = 1; i < header.size; ++i)
|
|
2531
|
+
offsets[i] = offsets[i - 1] + node_bytes_(levels[i - 1]);
|
|
2532
|
+
|
|
2533
|
+
std::size_t total_bytes = offsets[header.size - 1] + node_bytes_(levels[header.size - 1]);
|
|
2534
|
+
if (file.size() < total_bytes) {
|
|
2535
|
+
reset();
|
|
2536
|
+
return result.failed("File is corrupted and can't fit all the nodes");
|
|
2510
2537
|
}
|
|
2511
2538
|
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2539
|
+
// Submit metadata and reserve memory
|
|
2540
|
+
index_limits_t limits;
|
|
2541
|
+
limits.members = header.size;
|
|
2542
|
+
if (!reserve(limits)) {
|
|
2543
|
+
reset();
|
|
2544
|
+
return result.failed("Out of memory");
|
|
2517
2545
|
}
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
#else
|
|
2527
|
-
int descriptor = open(file_path, O_RDONLY);
|
|
2528
|
-
#endif
|
|
2529
|
-
|
|
2530
|
-
// Estimate the file size
|
|
2531
|
-
struct stat file_stat;
|
|
2532
|
-
int fstat_status = fstat(descriptor, &file_stat);
|
|
2533
|
-
if (fstat_status < 0) {
|
|
2534
|
-
close(descriptor);
|
|
2535
|
-
return result.failed(std::strerror(errno));
|
|
2546
|
+
nodes_count_ = header.size;
|
|
2547
|
+
max_level_ = static_cast<level_t>(header.max_level);
|
|
2548
|
+
entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
|
|
2549
|
+
|
|
2550
|
+
// Rapidly address all the nodes
|
|
2551
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2552
|
+
nodes_[i] = node_t{(byte_t*)file.data() + offsets[i]};
|
|
2553
|
+
progress(i, header.size);
|
|
2536
2554
|
}
|
|
2555
|
+
viewed_file_ = std::move(file);
|
|
2556
|
+
return {};
|
|
2557
|
+
}
|
|
2537
2558
|
|
|
2538
|
-
|
|
2539
|
-
byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_PRIVATE, descriptor, 0);
|
|
2540
|
-
if (file == MAP_FAILED) {
|
|
2541
|
-
close(descriptor);
|
|
2542
|
-
return result.failed(std::strerror(errno));
|
|
2543
|
-
}
|
|
2544
|
-
viewed_file_.file_descriptor = descriptor;
|
|
2545
|
-
viewed_file_.ptr = file;
|
|
2546
|
-
viewed_file_.length = file_stat.st_size;
|
|
2547
|
-
#endif // Platform specific code
|
|
2559
|
+
#pragma endregion
|
|
2548
2560
|
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2561
|
+
/**
|
|
2562
|
+
* @brief Performs compaction on the whole HNSW index, purging some entries
|
|
2563
|
+
* and links to them, while also generating a more efficient mapping,
|
|
2564
|
+
* putting the more frequently used entries closer together.
|
|
2565
|
+
*
|
|
2566
|
+
*
|
|
2567
|
+
* Scans the whole collection, removing the links leading towards
|
|
2568
|
+
* banned entries. This essentially isolates some nodes from the rest
|
|
2569
|
+
* of the graph, while keeping their outgoing links, in case the node
|
|
2570
|
+
* is structurally relevant and has a crucial role in the index.
|
|
2571
|
+
* It won't reclaim the memory.
|
|
2572
|
+
*
|
|
2573
|
+
* @param[in] allow_member Predicate to mark nodes for isolation.
|
|
2574
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
2575
|
+
* @param[in] progress Callback to report the execution progress.
|
|
2576
|
+
*/
|
|
2577
|
+
template <typename values_at, typename metric_at, //
|
|
2578
|
+
typename slot_transition_at = dummy_label_to_label_mapping_t, //
|
|
2579
|
+
typename executor_at = dummy_executor_t, //
|
|
2580
|
+
typename progress_at = dummy_progress_t, //
|
|
2581
|
+
typename prefetch_at = dummy_prefetch_t>
|
|
2582
|
+
void compact( //
|
|
2583
|
+
values_at&& values, //
|
|
2584
|
+
metric_at&& metric, //
|
|
2585
|
+
slot_transition_at&& slot_transition, //
|
|
2560
2586
|
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2587
|
+
executor_at&& executor = executor_at{}, //
|
|
2588
|
+
progress_at&& progress = progress_at{}, //
|
|
2589
|
+
prefetch_at&& prefetch = prefetch_at{}) noexcept {
|
|
2590
|
+
|
|
2591
|
+
// Export all the keys, slots, and levels.
|
|
2592
|
+
// Partition them with the predicate.
|
|
2593
|
+
// Sort the allowed entries in descending order of their level.
|
|
2594
|
+
// Create a new array mapping old slots to the new ones (INT_MAX for deleted items).
|
|
2595
|
+
struct slot_level_t {
|
|
2596
|
+
compressed_slot_t old_slot;
|
|
2597
|
+
compressed_slot_t cluster;
|
|
2598
|
+
level_t level;
|
|
2599
|
+
};
|
|
2600
|
+
using slot_level_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<slot_level_t>;
|
|
2601
|
+
buffer_gt<slot_level_t, slot_level_allocator_t> slots_and_levels(size());
|
|
2602
|
+
|
|
2603
|
+
// For every bottom level node, determine its parent cluster
|
|
2604
|
+
executor.execute_bulk(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
|
|
2605
|
+
context_t& context = contexts_[thread_idx];
|
|
2606
|
+
std::size_t cluster = search_for_one_( //
|
|
2607
|
+
values[citerator_at(old_slot)], //
|
|
2608
|
+
metric, prefetch, //
|
|
2609
|
+
entry_slot_, max_level_, 0, context);
|
|
2610
|
+
slots_and_levels[old_slot] = { //
|
|
2611
|
+
static_cast<compressed_slot_t>(old_slot), //
|
|
2612
|
+
static_cast<compressed_slot_t>(cluster), //
|
|
2613
|
+
node_at_(old_slot).level()};
|
|
2614
|
+
});
|
|
2564
2615
|
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
return result.failed("Out of memory!");
|
|
2616
|
+
// Where the actual permutation happens:
|
|
2617
|
+
std::sort(slots_and_levels.begin(), slots_and_levels.end(), [](slot_level_t const& a, slot_level_t const& b) {
|
|
2618
|
+
return a.level == b.level ? a.cluster < b.cluster : a.level > b.level;
|
|
2619
|
+
});
|
|
2570
2620
|
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2621
|
+
using size_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
|
|
2622
|
+
buffer_gt<std::size_t, size_allocator_t> old_slot_to_new(slots_and_levels.size());
|
|
2623
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot)
|
|
2624
|
+
old_slot_to_new[slots_and_levels[new_slot].old_slot] = new_slot;
|
|
2575
2625
|
|
|
2576
|
-
//
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
for (std::size_t i = 0; i != size; ++i) {
|
|
2580
|
-
byte_t* tape = (byte_t*)(file + progress_bytes);
|
|
2581
|
-
dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
|
|
2582
|
-
level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
|
|
2583
|
-
|
|
2584
|
-
std::size_t node_bytes = node_bytes_(dim, level);
|
|
2585
|
-
std::size_t node_vector_bytes = dim * sizeof(scalar_t);
|
|
2586
|
-
nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
|
|
2587
|
-
progress_bytes += node_bytes;
|
|
2588
|
-
progress(i, size);
|
|
2589
|
-
}
|
|
2626
|
+
// Erase all the incoming links
|
|
2627
|
+
buffer_gt<node_t, nodes_allocator_t> reordered_nodes(slots_and_levels.size());
|
|
2628
|
+
tape_allocator_t reordered_tape;
|
|
2590
2629
|
|
|
2591
|
-
|
|
2592
|
-
|
|
2630
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
|
|
2631
|
+
std::size_t old_slot = slots_and_levels[new_slot].old_slot;
|
|
2632
|
+
node_t old_node = node_at_(old_slot);
|
|
2593
2633
|
|
|
2594
|
-
|
|
2634
|
+
std::size_t node_bytes = node_bytes_(old_node.level());
|
|
2635
|
+
byte_t* new_data = (byte_t*)reordered_tape.allocate(node_bytes);
|
|
2636
|
+
node_t new_node{new_data};
|
|
2637
|
+
std::memcpy(new_data, old_node.tape(), node_bytes);
|
|
2595
2638
|
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
std::size_t engagements{};
|
|
2600
|
-
std::size_t cycles{};
|
|
2601
|
-
std::size_t measurements{};
|
|
2639
|
+
for (level_t level = 0; level <= old_node.level(); ++level)
|
|
2640
|
+
for (misaligned_ref_gt<compressed_slot_t> neighbor : neighbors_(new_node, level))
|
|
2641
|
+
neighbor = static_cast<compressed_slot_t>(old_slot_to_new[compressed_slot_t(neighbor)]);
|
|
2602
2642
|
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
return std::move(*this);
|
|
2643
|
+
reordered_nodes[new_slot] = new_node;
|
|
2644
|
+
|
|
2645
|
+
progress(new_slot, slots_and_levels.size());
|
|
2607
2646
|
}
|
|
2608
|
-
};
|
|
2609
2647
|
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
* @param[inout] second_to_first Container to map ::second labels to ::first.
|
|
2617
|
-
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
2618
|
-
* @param[in] progress Callback to report the execution progress.
|
|
2619
|
-
*/
|
|
2620
|
-
template < //
|
|
2621
|
-
typename first_to_second_at = dummy_label_to_label_mapping_t, //
|
|
2622
|
-
typename second_to_first_at = dummy_label_to_label_mapping_t, //
|
|
2623
|
-
typename executor_at = dummy_executor_t, //
|
|
2624
|
-
typename progress_at = dummy_progress_t //
|
|
2625
|
-
>
|
|
2626
|
-
static join_result_t join( //
|
|
2627
|
-
index_gt const& first, index_gt const& second, //
|
|
2628
|
-
join_config_t config = {}, //
|
|
2629
|
-
first_to_second_at&& first_to_second = first_to_second_at{}, //
|
|
2630
|
-
second_to_first_at&& second_to_first = second_to_first_at{}, //
|
|
2631
|
-
executor_at&& executor = executor_at{}, //
|
|
2632
|
-
progress_at&& progress = progress_at{}) noexcept {
|
|
2648
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
|
|
2649
|
+
std::size_t old_slot = slots_and_levels[new_slot].old_slot;
|
|
2650
|
+
slot_transition(node_at_(old_slot).ckey(), //
|
|
2651
|
+
static_cast<compressed_slot_t>(old_slot), //
|
|
2652
|
+
static_cast<compressed_slot_t>(new_slot));
|
|
2653
|
+
}
|
|
2633
2654
|
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
config, //
|
|
2638
|
-
std::forward<second_to_first_at>(second_to_first), //
|
|
2639
|
-
std::forward<first_to_second_at>(first_to_second), //
|
|
2640
|
-
std::forward<executor_at>(executor), //
|
|
2641
|
-
std::forward<progress_at>(progress));
|
|
2642
|
-
else
|
|
2643
|
-
return join_small_and_big_( //
|
|
2644
|
-
first, second, //
|
|
2645
|
-
config, //
|
|
2646
|
-
std::forward<first_to_second_at>(first_to_second), //
|
|
2647
|
-
std::forward<second_to_first_at>(second_to_first), //
|
|
2648
|
-
std::forward<executor_at>(executor), //
|
|
2649
|
-
std::forward<progress_at>(progress));
|
|
2655
|
+
nodes_ = std::move(reordered_nodes);
|
|
2656
|
+
tape_allocator_ = std::move(reordered_tape);
|
|
2657
|
+
entry_slot_ = old_slot_to_new[entry_slot_];
|
|
2650
2658
|
}
|
|
2651
2659
|
|
|
2652
2660
|
/**
|
|
@@ -2671,293 +2679,80 @@ class index_gt {
|
|
|
2671
2679
|
progress_at&& progress = progress_at{}) noexcept {
|
|
2672
2680
|
|
|
2673
2681
|
// Erase all the incoming links
|
|
2674
|
-
|
|
2675
|
-
|
|
2682
|
+
std::size_t nodes_count = size();
|
|
2683
|
+
executor.execute_bulk(nodes_count, [&](std::size_t, std::size_t node_idx) {
|
|
2684
|
+
node_t node = node_at_(node_idx);
|
|
2676
2685
|
for (level_t level = 0; level <= node.level(); ++level) {
|
|
2677
2686
|
neighbors_ref_t neighbors = neighbors_(node, level);
|
|
2678
2687
|
std::size_t old_size = neighbors.size();
|
|
2679
2688
|
neighbors.clear();
|
|
2680
2689
|
for (std::size_t i = 0; i != old_size; ++i) {
|
|
2681
|
-
|
|
2682
|
-
node_t neighbor =
|
|
2683
|
-
if (allow_member(member_cref_t{neighbor.
|
|
2684
|
-
neighbors.push_back(
|
|
2690
|
+
compressed_slot_t neighbor_slot = neighbors[i];
|
|
2691
|
+
node_t neighbor = node_at_(neighbor_slot);
|
|
2692
|
+
if (allow_member(member_cref_t{neighbor.ckey(), neighbor_slot}))
|
|
2693
|
+
neighbors.push_back(neighbor_slot);
|
|
2685
2694
|
}
|
|
2686
2695
|
}
|
|
2696
|
+
progress(node_idx, nodes_count);
|
|
2687
2697
|
});
|
|
2688
2698
|
}
|
|
2689
2699
|
|
|
2690
2700
|
private:
|
|
2691
|
-
template <typename first_to_second_at, typename second_to_first_at, typename executor_at, typename progress_at>
|
|
2692
|
-
static join_result_t join_small_and_big_( //
|
|
2693
|
-
index_gt const& men, index_gt const& women, //
|
|
2694
|
-
join_config_t config, //
|
|
2695
|
-
first_to_second_at&& man_to_woman, //
|
|
2696
|
-
second_to_first_at&& woman_to_man, //
|
|
2697
|
-
executor_at&& executor, //
|
|
2698
|
-
progress_at&& progress) noexcept {
|
|
2699
|
-
|
|
2700
|
-
join_result_t result;
|
|
2701
|
-
|
|
2702
|
-
// Sanity checks and argument validation:
|
|
2703
|
-
if (&men == &women)
|
|
2704
|
-
return result.failed("Can't join with itself, consider copying");
|
|
2705
|
-
if (config.max_proposals == 0)
|
|
2706
|
-
config.max_proposals = std::log(men.size()) + executor.size();
|
|
2707
|
-
config.max_proposals = (std::min)(men.size(), config.max_proposals);
|
|
2708
|
-
|
|
2709
|
-
using proposals_count_t = std::uint16_t;
|
|
2710
|
-
using ids_allocator_t = typename allocator_traits_t::template rebind_alloc<id_t>;
|
|
2711
|
-
allocator_t& alloc = men.allocator_;
|
|
2712
|
-
|
|
2713
|
-
// Create an atomic queue, as a ring structure, from/to which
|
|
2714
|
-
// free men will be added/pulled.
|
|
2715
|
-
std::mutex free_men_mutex{};
|
|
2716
|
-
ring_gt<id_t, ids_allocator_t> free_men;
|
|
2717
|
-
free_men.reserve(men.size());
|
|
2718
|
-
for (std::size_t i = 0; i != men.size(); ++i)
|
|
2719
|
-
free_men.push(static_cast<id_t>(i));
|
|
2720
|
-
|
|
2721
|
-
// We are gonna need some temporary memory.
|
|
2722
|
-
proposals_count_t* proposal_counts = (proposals_count_t*)alloc.allocate(sizeof(proposals_count_t) * men.size());
|
|
2723
|
-
id_t* man_to_woman_ids = (id_t*)alloc.allocate(sizeof(id_t) * men.size());
|
|
2724
|
-
id_t* woman_to_man_ids = (id_t*)alloc.allocate(sizeof(id_t) * women.size());
|
|
2725
|
-
if (!proposal_counts || !man_to_woman_ids || !woman_to_man_ids)
|
|
2726
|
-
return result.failed("Can't temporary mappings");
|
|
2727
|
-
|
|
2728
|
-
id_t missing_id;
|
|
2729
|
-
std::memset((void*)&missing_id, 0xFF, sizeof(id_t));
|
|
2730
|
-
std::memset((void*)man_to_woman_ids, 0xFF, sizeof(id_t) * men.size());
|
|
2731
|
-
std::memset((void*)woman_to_man_ids, 0xFF, sizeof(id_t) * women.size());
|
|
2732
|
-
std::memset(proposal_counts, 0, sizeof(proposals_count_t) * men.size());
|
|
2733
|
-
|
|
2734
|
-
// Define locks, to limit concurrent accesses to `man_to_woman_ids` and `woman_to_man_ids`.
|
|
2735
|
-
visits_bitset_t men_locks, women_locks;
|
|
2736
|
-
if (!men_locks.resize(men.size()) || !women_locks.resize(women.size()))
|
|
2737
|
-
return result.failed("Can't allocate locks");
|
|
2738
|
-
|
|
2739
|
-
// Accumulate statistics from all the contexts,
|
|
2740
|
-
// to have a baseline to compare with, by the time the `join` is finished.
|
|
2741
|
-
std::size_t old_measurements{};
|
|
2742
|
-
std::size_t old_cycles{};
|
|
2743
|
-
for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
|
|
2744
|
-
old_measurements += women.contexts_[thread_idx].measurements_count;
|
|
2745
|
-
old_cycles += women.contexts_[thread_idx].iteration_cycles;
|
|
2746
|
-
}
|
|
2747
|
-
std::atomic<std::size_t> rounds{0};
|
|
2748
|
-
std::atomic<std::size_t> engagements{0};
|
|
2749
|
-
|
|
2750
|
-
// Concurrently process all the men
|
|
2751
|
-
executor.execute_bulk([&](std::size_t thread_idx) {
|
|
2752
|
-
context_t& context = women.contexts_[thread_idx];
|
|
2753
|
-
search_config_t search_config;
|
|
2754
|
-
search_config.expansion = config.expansion;
|
|
2755
|
-
search_config.exact = config.exact;
|
|
2756
|
-
search_config.thread = thread_idx;
|
|
2757
|
-
id_t free_man_id;
|
|
2758
|
-
|
|
2759
|
-
// While there exist a free man who still has a woman to propose to.
|
|
2760
|
-
while (true) {
|
|
2761
|
-
std::size_t passed_rounds = 0;
|
|
2762
|
-
std::size_t total_rounds = 0;
|
|
2763
|
-
{
|
|
2764
|
-
std::unique_lock<std::mutex> pop_lock(free_men_mutex);
|
|
2765
|
-
if (!free_men.try_pop(free_man_id))
|
|
2766
|
-
// Primary exit path, we have exhausted the list of candidates
|
|
2767
|
-
break;
|
|
2768
|
-
passed_rounds = ++rounds;
|
|
2769
|
-
total_rounds = passed_rounds + free_men.size();
|
|
2770
|
-
}
|
|
2771
|
-
progress(passed_rounds, total_rounds);
|
|
2772
|
-
while (men_locks.atomic_set(free_man_id))
|
|
2773
|
-
;
|
|
2774
|
-
|
|
2775
|
-
node_t free_man = men.node_with_id_(free_man_id);
|
|
2776
|
-
proposals_count_t& free_man_proposals = proposal_counts[free_man_id];
|
|
2777
|
-
if (free_man_proposals >= config.max_proposals)
|
|
2778
|
-
continue;
|
|
2779
|
-
|
|
2780
|
-
// Find the closest woman, to whom this man hasn't proposed yet.
|
|
2781
|
-
free_man_proposals++;
|
|
2782
|
-
search_result_t candidates = women.search(free_man.vector_view(), free_man_proposals, search_config);
|
|
2783
|
-
if (!candidates) {
|
|
2784
|
-
// TODO:
|
|
2785
|
-
}
|
|
2786
|
-
|
|
2787
|
-
match_t match = candidates.back();
|
|
2788
|
-
member_cref_t woman = match.member;
|
|
2789
|
-
while (women_locks.atomic_set(woman.id))
|
|
2790
|
-
;
|
|
2791
|
-
|
|
2792
|
-
id_t husband_id = woman_to_man_ids[woman.id];
|
|
2793
|
-
bool woman_is_free = husband_id == missing_id;
|
|
2794
|
-
if (woman_is_free) {
|
|
2795
|
-
// Engagement
|
|
2796
|
-
man_to_woman_ids[free_man_id] = woman.id;
|
|
2797
|
-
woman_to_man_ids[woman.id] = free_man_id;
|
|
2798
|
-
engagements++;
|
|
2799
|
-
} else {
|
|
2800
|
-
distance_t distance_from_husband = context.measure(woman.vector, men.node_with_id_(husband_id));
|
|
2801
|
-
distance_t distance_from_candidate = match.distance;
|
|
2802
|
-
if (distance_from_husband > distance_from_candidate) {
|
|
2803
|
-
// Break-up
|
|
2804
|
-
while (men_locks.atomic_set(husband_id))
|
|
2805
|
-
;
|
|
2806
|
-
man_to_woman_ids[husband_id] = missing_id;
|
|
2807
|
-
men_locks.atomic_reset(husband_id);
|
|
2808
|
-
|
|
2809
|
-
// New Engagement
|
|
2810
|
-
man_to_woman_ids[free_man_id] = woman.id;
|
|
2811
|
-
woman_to_man_ids[woman.id] = free_man_id;
|
|
2812
|
-
engagements++;
|
|
2813
|
-
|
|
2814
|
-
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
2815
|
-
free_men.push(husband_id);
|
|
2816
|
-
} else {
|
|
2817
|
-
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
2818
|
-
free_men.push(free_man_id);
|
|
2819
|
-
}
|
|
2820
|
-
}
|
|
2821
|
-
|
|
2822
|
-
men_locks.atomic_reset(free_man_id);
|
|
2823
|
-
women_locks.atomic_reset(woman.id);
|
|
2824
|
-
}
|
|
2825
|
-
});
|
|
2826
|
-
|
|
2827
|
-
// Export the IDs into labels:
|
|
2828
|
-
std::size_t intersection_size = 0;
|
|
2829
|
-
for (std::size_t i = 0; i != men.size(); ++i) {
|
|
2830
|
-
id_t woman_id = man_to_woman_ids[i];
|
|
2831
|
-
if (woman_id != missing_id) {
|
|
2832
|
-
label_t man = men.node_with_id_(i).label();
|
|
2833
|
-
label_t woman = women.node_with_id_(woman_id).label();
|
|
2834
|
-
man_to_woman[man] = woman;
|
|
2835
|
-
woman_to_man[woman] = man;
|
|
2836
|
-
intersection_size++;
|
|
2837
|
-
}
|
|
2838
|
-
}
|
|
2839
|
-
|
|
2840
|
-
// Deallocate memory
|
|
2841
|
-
alloc.deallocate((byte_t*)proposal_counts, sizeof(proposals_count_t) * men.size());
|
|
2842
|
-
alloc.deallocate((byte_t*)man_to_woman_ids, sizeof(id_t) * men.size());
|
|
2843
|
-
alloc.deallocate((byte_t*)woman_to_man_ids, sizeof(id_t) * women.size());
|
|
2844
|
-
|
|
2845
|
-
// Export stats
|
|
2846
|
-
result.engagements = engagements;
|
|
2847
|
-
result.intersection_size = intersection_size;
|
|
2848
|
-
for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
|
|
2849
|
-
result.measurements += women.contexts_[thread_idx].measurements_count;
|
|
2850
|
-
result.cycles += women.contexts_[thread_idx].iteration_cycles;
|
|
2851
|
-
}
|
|
2852
|
-
result.measurements -= old_measurements;
|
|
2853
|
-
result.cycles -= old_cycles;
|
|
2854
|
-
return result;
|
|
2855
|
-
}
|
|
2856
|
-
|
|
2857
|
-
void reset_view_() noexcept {
|
|
2858
|
-
if (!viewed_file_)
|
|
2859
|
-
return;
|
|
2860
|
-
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
2861
|
-
UnmapViewOfFile(viewed_file_.ptr);
|
|
2862
|
-
CloseHandle(viewed_file_.mapping_handle);
|
|
2863
|
-
CloseHandle(viewed_file_.file_handle);
|
|
2864
|
-
|
|
2865
|
-
#else
|
|
2866
|
-
munmap(viewed_file_.ptr, viewed_file_.length);
|
|
2867
|
-
close(viewed_file_.file_descriptor);
|
|
2868
|
-
#endif
|
|
2869
|
-
viewed_file_ = {};
|
|
2870
|
-
}
|
|
2871
|
-
|
|
2872
2701
|
inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
|
|
2873
2702
|
precomputed_constants_t pre;
|
|
2874
|
-
pre.connectivity_max_base = config.connectivity * base_level_multiple_();
|
|
2875
2703
|
pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
|
|
2876
|
-
pre.neighbors_bytes = config.connectivity * sizeof(
|
|
2877
|
-
pre.neighbors_base_bytes =
|
|
2704
|
+
pre.neighbors_bytes = config.connectivity * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
|
|
2705
|
+
pre.neighbors_base_bytes = config.connectivity_base * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
|
|
2878
2706
|
return pre;
|
|
2879
2707
|
}
|
|
2880
2708
|
|
|
2881
|
-
inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
|
|
2882
|
-
inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
|
|
2883
|
-
return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
|
|
2884
|
-
}
|
|
2885
|
-
|
|
2886
2709
|
using span_bytes_t = span_gt<byte_t>;
|
|
2887
|
-
struct node_bytes_split_t {
|
|
2888
|
-
span_bytes_t tape{};
|
|
2889
|
-
span_bytes_t vector{};
|
|
2890
2710
|
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
inline node_bytes_split_t node_bytes_split_(node_t node) const noexcept {
|
|
2900
|
-
std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
|
|
2901
|
-
std::size_t bytes_in_tape = node_head_bytes_() + levels_bytes;
|
|
2902
|
-
return {{node.tape(), bytes_in_tape}, {(byte_t*)node.vector(), node_vector_bytes_(node)}};
|
|
2711
|
+
inline span_bytes_t node_bytes_(node_t node) const noexcept { return {node.tape(), node_bytes_(node.level())}; }
|
|
2712
|
+
inline std::size_t node_bytes_(level_t level) const noexcept {
|
|
2713
|
+
return node_head_bytes_() + node_neighbors_bytes_(level);
|
|
2714
|
+
}
|
|
2715
|
+
inline std::size_t node_neighbors_bytes_(node_t node) const noexcept { return node_neighbors_bytes_(node.level()); }
|
|
2716
|
+
inline std::size_t node_neighbors_bytes_(level_t level) const noexcept {
|
|
2717
|
+
return pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
|
|
2903
2718
|
}
|
|
2904
2719
|
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
std::size_t vector_bytes = node_vector_bytes_(dims_to_store);
|
|
2911
|
-
std::size_t node_bytes = node_bytes_(dims_to_store, level);
|
|
2912
|
-
std::size_t non_vector_bytes = node_bytes - vector_bytes;
|
|
2913
|
-
|
|
2914
|
-
byte_t* data = (byte_t*)point_allocator_.allocate(node_bytes);
|
|
2915
|
-
if (!data)
|
|
2916
|
-
return node_bytes_split_t{};
|
|
2917
|
-
return {{data, non_vector_bytes}, {data + non_vector_bytes, vector_bytes}};
|
|
2720
|
+
span_bytes_t node_malloc_(level_t level) noexcept {
|
|
2721
|
+
std::size_t node_bytes = node_bytes_(level);
|
|
2722
|
+
byte_t* data = (byte_t*)tape_allocator_.allocate(node_bytes);
|
|
2723
|
+
return data ? span_bytes_t{data, node_bytes} : span_bytes_t{};
|
|
2918
2724
|
}
|
|
2919
2725
|
|
|
2920
|
-
node_t node_make_(
|
|
2921
|
-
|
|
2922
|
-
if (
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
node_t node = node_bytes;
|
|
2929
|
-
node.label(label);
|
|
2930
|
-
node.dim(static_cast<dim_t>(vector.size()));
|
|
2726
|
+
node_t node_make_(key_t key, level_t level) noexcept {
|
|
2727
|
+
span_bytes_t node_bytes = node_malloc_(level);
|
|
2728
|
+
if (!node_bytes)
|
|
2729
|
+
return {};
|
|
2730
|
+
|
|
2731
|
+
std::memset(node_bytes.data(), 0, node_bytes.size());
|
|
2732
|
+
node_t node{(byte_t*)node_bytes.data()};
|
|
2733
|
+
node.key(key);
|
|
2931
2734
|
node.level(level);
|
|
2932
2735
|
return node;
|
|
2933
2736
|
}
|
|
2934
2737
|
|
|
2935
|
-
node_t node_make_copy_(
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
node_t old_node = old_bytes;
|
|
2942
|
-
node_bytes_split_t node_bytes = node_malloc_(old_node.vector_view().size(), old_node.level());
|
|
2943
|
-
std::memcpy(node_bytes.tape.data(), old_bytes.tape.data(), old_bytes.tape.size());
|
|
2944
|
-
std::memcpy(node_bytes.vector.data(), old_bytes.vector.data(), old_bytes.vector.size());
|
|
2945
|
-
return node_bytes;
|
|
2946
|
-
}
|
|
2738
|
+
node_t node_make_copy_(span_bytes_t old_bytes) noexcept {
|
|
2739
|
+
byte_t* data = (byte_t*)tape_allocator_.allocate(old_bytes.size());
|
|
2740
|
+
if (!data)
|
|
2741
|
+
return {};
|
|
2742
|
+
std::memcpy(data, old_bytes.data(), old_bytes.size());
|
|
2743
|
+
return node_t{data};
|
|
2947
2744
|
}
|
|
2948
2745
|
|
|
2949
|
-
void node_free_(std::size_t
|
|
2950
|
-
|
|
2746
|
+
void node_free_(std::size_t idx) noexcept {
|
|
2951
2747
|
if (viewed_file_)
|
|
2952
2748
|
return;
|
|
2953
2749
|
|
|
2954
|
-
node_t& node = nodes_[
|
|
2955
|
-
|
|
2956
|
-
point_allocator_.deallocate(node.tape(), node_bytes);
|
|
2750
|
+
node_t& node = nodes_[idx];
|
|
2751
|
+
tape_allocator_.deallocate(node.tape(), node_bytes_(node).size());
|
|
2957
2752
|
node = node_t{};
|
|
2958
2753
|
}
|
|
2959
2754
|
|
|
2960
|
-
inline node_t
|
|
2755
|
+
inline node_t node_at_(std::size_t idx) const noexcept { return nodes_[idx]; }
|
|
2961
2756
|
inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
|
|
2962
2757
|
|
|
2963
2758
|
inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
|
|
@@ -2970,91 +2765,102 @@ class index_gt {
|
|
|
2970
2765
|
|
|
2971
2766
|
struct node_lock_t {
|
|
2972
2767
|
visits_bitset_t& bitset;
|
|
2973
|
-
std::size_t
|
|
2974
|
-
|
|
2975
|
-
inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
|
|
2768
|
+
std::size_t slot;
|
|
2769
|
+
inline ~node_lock_t() noexcept { bitset.atomic_reset(slot); }
|
|
2976
2770
|
};
|
|
2977
2771
|
|
|
2978
|
-
inline node_lock_t node_lock_(std::size_t
|
|
2979
|
-
while (nodes_mutexes_.atomic_set(
|
|
2772
|
+
inline node_lock_t node_lock_(std::size_t slot) const noexcept {
|
|
2773
|
+
while (nodes_mutexes_.atomic_set(slot))
|
|
2980
2774
|
;
|
|
2981
|
-
return {nodes_mutexes_,
|
|
2775
|
+
return {nodes_mutexes_, slot};
|
|
2982
2776
|
}
|
|
2983
2777
|
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2778
|
+
template <typename value_at, typename metric_at, typename prefetch_at>
|
|
2779
|
+
void connect_node_across_levels_( //
|
|
2780
|
+
value_at&& value, metric_at&& metric, prefetch_at&& prefetch, //
|
|
2781
|
+
std::size_t node_slot, std::size_t entry_slot, level_t max_level, level_t target_level, //
|
|
2782
|
+
index_update_config_t const& config, context_t& context) usearch_noexcept_m {
|
|
2988
2783
|
|
|
2989
2784
|
// Go down the level, tracking only the closest match
|
|
2990
|
-
|
|
2785
|
+
std::size_t closest_slot = search_for_one_( //
|
|
2786
|
+
value, metric, prefetch, //
|
|
2787
|
+
entry_slot, max_level, target_level, context);
|
|
2991
2788
|
|
|
2992
2789
|
// From `target_level` down perform proper extensive search
|
|
2993
2790
|
for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
|
|
2994
2791
|
// TODO: Handle out of memory conditions
|
|
2995
|
-
search_to_insert_(
|
|
2996
|
-
|
|
2997
|
-
reconnect_neighbor_nodes_(
|
|
2792
|
+
search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
|
|
2793
|
+
closest_slot = connect_new_node_(metric, node_slot, level, context);
|
|
2794
|
+
reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
|
|
2998
2795
|
}
|
|
2999
2796
|
}
|
|
3000
2797
|
|
|
3001
|
-
|
|
2798
|
+
template <typename metric_at>
|
|
2799
|
+
std::size_t connect_new_node_( //
|
|
2800
|
+
metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
|
|
3002
2801
|
|
|
3003
|
-
node_t new_node =
|
|
2802
|
+
node_t new_node = node_at_(new_slot);
|
|
3004
2803
|
top_candidates_t& top = context.top_candidates;
|
|
3005
2804
|
|
|
3006
|
-
// Outgoing links from `
|
|
2805
|
+
// Outgoing links from `new_slot`:
|
|
3007
2806
|
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3008
2807
|
{
|
|
3009
2808
|
usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
|
|
3010
|
-
candidates_view_t top_view = refine_(
|
|
2809
|
+
candidates_view_t top_view = refine_(metric, config_.connectivity, top, context);
|
|
3011
2810
|
|
|
3012
2811
|
for (std::size_t idx = 0; idx != top_view.size(); idx++) {
|
|
3013
2812
|
usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
3014
|
-
usearch_assert_m(level <=
|
|
3015
|
-
new_neighbors.push_back(top_view[idx].
|
|
2813
|
+
usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
|
|
2814
|
+
new_neighbors.push_back(top_view[idx].slot);
|
|
3016
2815
|
}
|
|
3017
2816
|
}
|
|
3018
2817
|
|
|
3019
2818
|
return new_neighbors[0];
|
|
3020
2819
|
}
|
|
3021
2820
|
|
|
3022
|
-
|
|
2821
|
+
template <typename value_at, typename metric_at>
|
|
2822
|
+
void reconnect_neighbor_nodes_( //
|
|
2823
|
+
metric_at&& metric, std::size_t new_slot, value_at&& value, level_t level,
|
|
2824
|
+
context_t& context) usearch_noexcept_m {
|
|
3023
2825
|
|
|
3024
|
-
node_t new_node =
|
|
2826
|
+
node_t new_node = node_at_(new_slot);
|
|
3025
2827
|
top_candidates_t& top = context.top_candidates;
|
|
3026
2828
|
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3027
2829
|
|
|
3028
2830
|
// Reverse links from the neighbors:
|
|
3029
|
-
std::size_t const connectivity_max = level ? config_.connectivity :
|
|
3030
|
-
for (
|
|
3031
|
-
|
|
3032
|
-
|
|
2831
|
+
std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
|
|
2832
|
+
for (compressed_slot_t close_slot : new_neighbors) {
|
|
2833
|
+
if (close_slot == new_slot)
|
|
2834
|
+
continue;
|
|
2835
|
+
node_lock_t close_lock = node_lock_(close_slot);
|
|
2836
|
+
node_t close_node = node_at_(close_slot);
|
|
3033
2837
|
|
|
3034
2838
|
neighbors_ref_t close_header = neighbors_(close_node, level);
|
|
3035
2839
|
usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
|
|
3036
|
-
usearch_assert_m(
|
|
2840
|
+
usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
|
|
3037
2841
|
usearch_assert_m(level <= close_node.level(), "Linking to missing level");
|
|
3038
2842
|
|
|
3039
|
-
// If `
|
|
2843
|
+
// If `new_slot` is already present in the neighboring connections of `close_slot`
|
|
3040
2844
|
// then no need to modify any connections or run the heuristics.
|
|
3041
2845
|
if (close_header.size() < connectivity_max) {
|
|
3042
|
-
close_header.push_back(
|
|
2846
|
+
close_header.push_back(static_cast<compressed_slot_t>(new_slot));
|
|
3043
2847
|
continue;
|
|
3044
2848
|
}
|
|
3045
2849
|
|
|
3046
2850
|
// To fit a new connection we need to drop an existing one.
|
|
3047
2851
|
top.clear();
|
|
3048
2852
|
usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
|
|
3049
|
-
top.insert_reserved(
|
|
3050
|
-
|
|
3051
|
-
|
|
2853
|
+
top.insert_reserved(
|
|
2854
|
+
{context.measure(value, citerator_at(close_slot), metric), static_cast<compressed_slot_t>(new_slot)});
|
|
2855
|
+
for (compressed_slot_t successor_slot : close_header)
|
|
2856
|
+
top.insert_reserved(
|
|
2857
|
+
{context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
|
|
3052
2858
|
|
|
3053
2859
|
// Export the results:
|
|
3054
2860
|
close_header.clear();
|
|
3055
|
-
candidates_view_t top_view = refine_(
|
|
2861
|
+
candidates_view_t top_view = refine_(metric, connectivity_max, top, context);
|
|
3056
2862
|
for (std::size_t idx = 0; idx != top_view.size(); idx++)
|
|
3057
|
-
close_header.push_back(top_view[idx].
|
|
2863
|
+
close_header.push_back(top_view[idx].slot);
|
|
3058
2864
|
}
|
|
3059
2865
|
}
|
|
3060
2866
|
|
|
@@ -3064,31 +2870,105 @@ class index_gt {
|
|
|
3064
2870
|
return (level_t)r;
|
|
3065
2871
|
}
|
|
3066
2872
|
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
2873
|
+
struct candidates_range_t;
|
|
2874
|
+
class candidates_iterator_t {
|
|
2875
|
+
friend struct candidates_range_t;
|
|
2876
|
+
|
|
2877
|
+
index_gt const& index_;
|
|
2878
|
+
neighbors_ref_t neighbors_;
|
|
2879
|
+
visits_bitset_t& visits_;
|
|
2880
|
+
std::size_t current_;
|
|
2881
|
+
|
|
2882
|
+
candidates_iterator_t& skip_missing() noexcept {
|
|
2883
|
+
while (current_ != neighbors_.size()) {
|
|
2884
|
+
compressed_slot_t neighbor_slot = neighbors_[current_];
|
|
2885
|
+
if (visits_.test(neighbor_slot))
|
|
2886
|
+
current_++;
|
|
2887
|
+
else
|
|
2888
|
+
break;
|
|
2889
|
+
}
|
|
2890
|
+
return *this;
|
|
2891
|
+
}
|
|
3071
2892
|
|
|
3072
|
-
|
|
2893
|
+
public:
|
|
2894
|
+
using element_t = compressed_slot_t;
|
|
2895
|
+
using iterator_category = std::forward_iterator_tag;
|
|
2896
|
+
using value_type = element_t;
|
|
2897
|
+
using difference_type = std::ptrdiff_t;
|
|
2898
|
+
using pointer = misaligned_ptr_gt<element_t>;
|
|
2899
|
+
using reference = misaligned_ref_gt<element_t>;
|
|
2900
|
+
|
|
2901
|
+
reference operator*() const noexcept { return slot(); }
|
|
2902
|
+
candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_bitset_t& visits,
|
|
2903
|
+
std::size_t progress) noexcept
|
|
2904
|
+
: index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
|
|
2905
|
+
candidates_iterator_t operator++(int) noexcept {
|
|
2906
|
+
return candidates_iterator_t(index_, visits_, neighbors_, current_ + 1).skip_missing();
|
|
2907
|
+
}
|
|
2908
|
+
candidates_iterator_t& operator++() noexcept {
|
|
2909
|
+
++current_;
|
|
2910
|
+
skip_missing();
|
|
2911
|
+
return *this;
|
|
2912
|
+
}
|
|
2913
|
+
bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
|
|
2914
|
+
bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
|
|
2915
|
+
|
|
2916
|
+
key_t key() const noexcept { return index_->node_at_(slot()).key(); }
|
|
2917
|
+
compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
|
|
2918
|
+
friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
|
|
2919
|
+
friend inline key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
|
|
2920
|
+
};
|
|
2921
|
+
|
|
2922
|
+
struct candidates_range_t {
|
|
2923
|
+
index_gt const& index;
|
|
2924
|
+
neighbors_ref_t neighbors;
|
|
2925
|
+
visits_bitset_t& visits;
|
|
2926
|
+
|
|
2927
|
+
candidates_iterator_t begin() const noexcept {
|
|
2928
|
+
return candidates_iterator_t{index, neighbors, visits, 0}.skip_missing();
|
|
2929
|
+
}
|
|
2930
|
+
candidates_iterator_t end() const noexcept { return {index, neighbors, visits, neighbors.size()}; }
|
|
2931
|
+
};
|
|
2932
|
+
|
|
2933
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
2934
|
+
std::size_t search_for_one_( //
|
|
2935
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
2936
|
+
std::size_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
|
|
2937
|
+
|
|
2938
|
+
visits_bitset_t& visits = context.visits;
|
|
2939
|
+
visits.clear();
|
|
2940
|
+
|
|
2941
|
+
// Optional prefetching
|
|
2942
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
2943
|
+
prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
|
|
2944
|
+
|
|
2945
|
+
distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
|
|
3073
2946
|
for (level_t level = begin_level; level > end_level; --level) {
|
|
3074
2947
|
bool changed;
|
|
3075
2948
|
do {
|
|
3076
2949
|
changed = false;
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
2950
|
+
node_lock_t closest_lock = node_lock_(closest_slot);
|
|
2951
|
+
neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
|
|
2952
|
+
|
|
2953
|
+
// Optional prefetching
|
|
2954
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
2955
|
+
candidates_range_t missing_candidates{*this, closest_neighbors, visits};
|
|
2956
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
2957
|
+
}
|
|
2958
|
+
|
|
2959
|
+
// Actual traversal
|
|
2960
|
+
for (compressed_slot_t candidate_slot : closest_neighbors) {
|
|
2961
|
+
distance_t candidate_dist = context.measure(query, citerator_at(candidate_slot), metric);
|
|
3082
2962
|
if (candidate_dist < closest_dist) {
|
|
3083
2963
|
closest_dist = candidate_dist;
|
|
3084
|
-
|
|
2964
|
+
closest_slot = candidate_slot;
|
|
3085
2965
|
changed = true;
|
|
3086
2966
|
}
|
|
3087
2967
|
}
|
|
3088
2968
|
context.iteration_cycles++;
|
|
3089
2969
|
} while (changed);
|
|
3090
2970
|
}
|
|
3091
|
-
return
|
|
2971
|
+
return closest_slot;
|
|
3092
2972
|
}
|
|
3093
2973
|
|
|
3094
2974
|
/**
|
|
@@ -3096,8 +2976,11 @@ class index_gt {
|
|
|
3096
2976
|
* Locks the nodes in the process, assuming other threads are updating neighbors lists.
|
|
3097
2977
|
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3098
2978
|
*/
|
|
3099
|
-
|
|
3100
|
-
|
|
2979
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
2980
|
+
bool search_to_insert_( //
|
|
2981
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
2982
|
+
std::size_t start_slot, std::size_t new_slot, level_t level, std::size_t top_limit,
|
|
2983
|
+
context_t& context) noexcept {
|
|
3101
2984
|
|
|
3102
2985
|
visits_bitset_t& visits = context.visits;
|
|
3103
2986
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
@@ -3107,10 +2990,14 @@ class index_gt {
|
|
|
3107
2990
|
next.clear();
|
|
3108
2991
|
top.clear();
|
|
3109
2992
|
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
2993
|
+
// Optional prefetching
|
|
2994
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
2995
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
2996
|
+
|
|
2997
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
2998
|
+
next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
|
|
2999
|
+
top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3000
|
+
visits.set(start_slot);
|
|
3114
3001
|
|
|
3115
3002
|
while (!next.empty()) {
|
|
3116
3003
|
|
|
@@ -3121,24 +3008,30 @@ class index_gt {
|
|
|
3121
3008
|
next.pop();
|
|
3122
3009
|
context.iteration_cycles++;
|
|
3123
3010
|
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3011
|
+
compressed_slot_t candidate_slot = candidacy.slot;
|
|
3012
|
+
if (new_slot == candidate_slot)
|
|
3013
|
+
continue;
|
|
3014
|
+
node_t candidate_ref = node_at_(candidate_slot);
|
|
3015
|
+
node_lock_t candidate_lock = node_lock_(candidate_slot);
|
|
3127
3016
|
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
3128
3017
|
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3018
|
+
// Optional prefetching
|
|
3019
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3020
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3021
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3022
|
+
}
|
|
3133
3023
|
|
|
3134
|
-
|
|
3135
|
-
|
|
3024
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
3025
|
+
if (visits.set(successor_slot))
|
|
3026
|
+
continue;
|
|
3136
3027
|
|
|
3028
|
+
// node_lock_t successor_lock = node_lock_(successor_slot);
|
|
3029
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3137
3030
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
3138
3031
|
// This can substantially grow our priority queue:
|
|
3139
|
-
next.insert({-successor_dist,
|
|
3032
|
+
next.insert({-successor_dist, successor_slot});
|
|
3140
3033
|
// This will automatically evict poor matches:
|
|
3141
|
-
top.insert({successor_dist,
|
|
3034
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
3142
3035
|
radius = top.top().distance;
|
|
3143
3036
|
}
|
|
3144
3037
|
}
|
|
@@ -3151,10 +3044,10 @@ class index_gt {
|
|
|
3151
3044
|
* Doesn't lock any nodes, assuming read-only simultaneous access.
|
|
3152
3045
|
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3153
3046
|
*/
|
|
3154
|
-
template <typename predicate_at>
|
|
3155
|
-
bool search_to_find_in_base_(
|
|
3156
|
-
|
|
3157
|
-
|
|
3047
|
+
template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
|
|
3048
|
+
bool search_to_find_in_base_( //
|
|
3049
|
+
value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
|
|
3050
|
+
std::size_t start_slot, std::size_t expansion, context_t& context) const noexcept {
|
|
3158
3051
|
|
|
3159
3052
|
visits_bitset_t& visits = context.visits;
|
|
3160
3053
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
@@ -3165,10 +3058,14 @@ class index_gt {
|
|
|
3165
3058
|
next.clear();
|
|
3166
3059
|
top.clear();
|
|
3167
3060
|
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3061
|
+
// Optional prefetching
|
|
3062
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
3063
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
3064
|
+
|
|
3065
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
3066
|
+
next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3067
|
+
top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3068
|
+
visits.set(start_slot);
|
|
3172
3069
|
|
|
3173
3070
|
while (!next.empty()) {
|
|
3174
3071
|
|
|
@@ -3179,28 +3076,29 @@ class index_gt {
|
|
|
3179
3076
|
next.pop();
|
|
3180
3077
|
context.iteration_cycles++;
|
|
3181
3078
|
|
|
3182
|
-
|
|
3183
|
-
neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
|
|
3079
|
+
neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
|
|
3184
3080
|
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3081
|
+
// Optional prefetching
|
|
3082
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3083
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3084
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3085
|
+
}
|
|
3189
3086
|
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3087
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
3088
|
+
if (visits.set(successor_slot))
|
|
3089
|
+
continue;
|
|
3193
3090
|
|
|
3091
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3194
3092
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
3195
3093
|
// This can substantially grow our priority queue:
|
|
3196
|
-
next.insert({-successor_dist,
|
|
3197
|
-
if (
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3094
|
+
next.insert({-successor_dist, successor_slot});
|
|
3095
|
+
if (!is_dummy<predicate_at>())
|
|
3096
|
+
if (!predicate(member_cref_t{node_at_(successor_slot).ckey(), successor_slot}))
|
|
3097
|
+
continue;
|
|
3098
|
+
|
|
3099
|
+
// This will automatically evict poor matches:
|
|
3100
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
3101
|
+
radius = top.top().distance;
|
|
3204
3102
|
}
|
|
3205
3103
|
}
|
|
3206
3104
|
}
|
|
@@ -3209,33 +3107,35 @@ class index_gt {
|
|
|
3209
3107
|
}
|
|
3210
3108
|
|
|
3211
3109
|
/**
|
|
3212
|
-
* @brief Iterates through all
|
|
3110
|
+
* @brief Iterates through all members, without actually touching the index.
|
|
3213
3111
|
*/
|
|
3214
|
-
template <typename predicate_at>
|
|
3215
|
-
void search_exact_(
|
|
3216
|
-
|
|
3217
|
-
|
|
3112
|
+
template <typename value_at, typename metric_at, typename predicate_at>
|
|
3113
|
+
void search_exact_( //
|
|
3114
|
+
value_at&& query, metric_at&& metric, predicate_at&& predicate, //
|
|
3115
|
+
std::size_t count, context_t& context) const noexcept {
|
|
3218
3116
|
|
|
3219
3117
|
top_candidates_t& top = context.top_candidates;
|
|
3220
3118
|
top.clear();
|
|
3221
3119
|
top.reserve(count);
|
|
3222
3120
|
for (std::size_t i = 0; i != size(); ++i) {
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3121
|
+
if (!is_dummy<predicate_at>())
|
|
3122
|
+
if (!predicate(at(i)))
|
|
3123
|
+
continue;
|
|
3124
|
+
|
|
3125
|
+
distance_t distance = context.measure(query, citerator_at(i), metric);
|
|
3126
|
+
top.insert(candidate_t{distance, static_cast<compressed_slot_t>(i)}, count);
|
|
3228
3127
|
}
|
|
3229
3128
|
}
|
|
3230
3129
|
|
|
3231
|
-
void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
|
|
3232
|
-
|
|
3233
3130
|
/**
|
|
3234
3131
|
* @brief This algorithm from the original paper implements a heuristic,
|
|
3235
3132
|
* that massively reduces the number of connections a point has,
|
|
3236
3133
|
* to keep only the neighbors, that are from each other.
|
|
3237
3134
|
*/
|
|
3238
|
-
|
|
3135
|
+
template <typename metric_at>
|
|
3136
|
+
candidates_view_t refine_( //
|
|
3137
|
+
metric_at&& metric, //
|
|
3138
|
+
std::size_t needed, top_candidates_t& top, context_t& context) const noexcept {
|
|
3239
3139
|
|
|
3240
3140
|
top.sort_ascending();
|
|
3241
3141
|
candidate_t* top_data = top.data();
|
|
@@ -3247,14 +3147,14 @@ class index_gt {
|
|
|
3247
3147
|
std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
|
|
3248
3148
|
while (submitted_count < needed && consumed_count < top_count) {
|
|
3249
3149
|
candidate_t candidate = top_data[consumed_count];
|
|
3250
|
-
node_t candidate_ref = node_with_id_(candidate.id);
|
|
3251
|
-
distance_t candidate_dist = candidate.distance;
|
|
3252
3150
|
bool good = true;
|
|
3253
3151
|
for (std::size_t idx = 0; idx < submitted_count; idx++) {
|
|
3254
3152
|
candidate_t submitted = top_data[idx];
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3153
|
+
distance_t inter_result_dist = context.measure( //
|
|
3154
|
+
citerator_at(candidate.slot), //
|
|
3155
|
+
citerator_at(submitted.slot), //
|
|
3156
|
+
metric);
|
|
3157
|
+
if (inter_result_dist < candidate.distance) {
|
|
3258
3158
|
good = false;
|
|
3259
3159
|
break;
|
|
3260
3160
|
}
|
|
@@ -3272,44 +3172,218 @@ class index_gt {
|
|
|
3272
3172
|
}
|
|
3273
3173
|
};
|
|
3274
3174
|
|
|
3175
|
+
struct join_result_t {
|
|
3176
|
+
error_t error{};
|
|
3177
|
+
std::size_t intersection_size{};
|
|
3178
|
+
std::size_t engagements{};
|
|
3179
|
+
std::size_t visited_members{};
|
|
3180
|
+
std::size_t computed_distances{};
|
|
3181
|
+
|
|
3182
|
+
explicit operator bool() const noexcept { return !error; }
|
|
3183
|
+
join_result_t failed(error_t message) noexcept {
|
|
3184
|
+
error = std::move(message);
|
|
3185
|
+
return std::move(*this);
|
|
3186
|
+
}
|
|
3187
|
+
};
|
|
3188
|
+
|
|
3275
3189
|
/**
|
|
3276
|
-
* @brief
|
|
3277
|
-
*
|
|
3190
|
+
* @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
|
|
3191
|
+
* to perform fast one-to-one matching between two large collections
|
|
3192
|
+
* of vectors, using approximate nearest neighbors search.
|
|
3193
|
+
*
|
|
3194
|
+
* @param[inout] man_to_woman Container to map ::first keys to ::second.
|
|
3195
|
+
* @param[inout] woman_to_man Container to map ::second keys to ::first.
|
|
3196
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
3197
|
+
* @param[in] progress Callback to report the execution progress.
|
|
3278
3198
|
*/
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3199
|
+
template < //
|
|
3200
|
+
|
|
3201
|
+
typename men_at, //
|
|
3202
|
+
typename women_at, //
|
|
3203
|
+
typename men_values_at, //
|
|
3204
|
+
typename women_values_at, //
|
|
3205
|
+
typename men_metric_at, //
|
|
3206
|
+
typename women_metric_at, //
|
|
3207
|
+
|
|
3208
|
+
typename man_to_woman_at = dummy_label_to_label_mapping_t, //
|
|
3209
|
+
typename woman_to_man_at = dummy_label_to_label_mapping_t, //
|
|
3210
|
+
typename executor_at = dummy_executor_t, //
|
|
3211
|
+
typename progress_at = dummy_progress_t //
|
|
3212
|
+
>
|
|
3213
|
+
static join_result_t join( //
|
|
3214
|
+
men_at const& men, //
|
|
3215
|
+
women_at const& women, //
|
|
3216
|
+
men_values_at const& men_values, //
|
|
3217
|
+
women_values_at const& women_values, //
|
|
3218
|
+
men_metric_at&& men_metric, //
|
|
3219
|
+
women_metric_at&& women_metric, //
|
|
3220
|
+
|
|
3221
|
+
index_join_config_t config = {}, //
|
|
3222
|
+
man_to_woman_at&& man_to_woman = man_to_woman_at{}, //
|
|
3223
|
+
woman_to_man_at&& woman_to_man = woman_to_man_at{}, //
|
|
3224
|
+
executor_at&& executor = executor_at{}, //
|
|
3225
|
+
progress_at&& progress = progress_at{}) noexcept {
|
|
3226
|
+
|
|
3227
|
+
if (women.size() < men.size())
|
|
3228
|
+
return unum::usearch::join( //
|
|
3229
|
+
women, men, //
|
|
3230
|
+
women_values, men_values, //
|
|
3231
|
+
std::forward<women_metric_at>(women_metric), std::forward<men_metric_at>(men_metric), //
|
|
3232
|
+
|
|
3233
|
+
config, //
|
|
3234
|
+
std::forward<woman_to_man_at>(woman_to_man), //
|
|
3235
|
+
std::forward<man_to_woman_at>(man_to_woman), //
|
|
3236
|
+
std::forward<executor_at>(executor), //
|
|
3237
|
+
std::forward<progress_at>(progress));
|
|
3238
|
+
|
|
3239
|
+
join_result_t result;
|
|
3240
|
+
|
|
3241
|
+
// Sanity checks and argument validation:
|
|
3242
|
+
if (&men == &women)
|
|
3243
|
+
return result.failed("Can't join with itself, consider copying");
|
|
3244
|
+
|
|
3245
|
+
if (config.max_proposals == 0)
|
|
3246
|
+
config.max_proposals = std::log(men.size()) + executor.size();
|
|
3247
|
+
|
|
3248
|
+
using proposals_count_t = std::uint16_t;
|
|
3249
|
+
config.max_proposals = (std::min)(men.size(), config.max_proposals);
|
|
3250
|
+
|
|
3251
|
+
using distance_t = typename men_at::distance_t;
|
|
3252
|
+
using dynamic_allocator_traits_t = typename men_at::dynamic_allocator_traits_t;
|
|
3253
|
+
using man_key_t = typename men_at::key_t;
|
|
3254
|
+
using woman_key_t = typename women_at::key_t;
|
|
3255
|
+
|
|
3256
|
+
// Use the `compressed_slot_t` type of the larger collection
|
|
3257
|
+
using compressed_slot_t = typename women_at::compressed_slot_t;
|
|
3258
|
+
using compressed_slot_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<compressed_slot_t>;
|
|
3259
|
+
using proposals_count_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<proposals_count_t>;
|
|
3260
|
+
|
|
3261
|
+
// Create an atomic queue, as a ring structure, from/to which
|
|
3262
|
+
// free men will be added/pulled.
|
|
3263
|
+
std::mutex free_men_mutex{};
|
|
3264
|
+
ring_gt<compressed_slot_t, compressed_slot_allocator_t> free_men;
|
|
3265
|
+
free_men.reserve(men.size());
|
|
3266
|
+
for (std::size_t i = 0; i != men.size(); ++i)
|
|
3267
|
+
free_men.push(static_cast<compressed_slot_t>(i));
|
|
3268
|
+
|
|
3269
|
+
// We are gonna need some temporary memory.
|
|
3270
|
+
buffer_gt<proposals_count_t, proposals_count_allocator_t> proposal_counts(men.size());
|
|
3271
|
+
buffer_gt<compressed_slot_t, compressed_slot_allocator_t> man_to_woman_slots(men.size());
|
|
3272
|
+
buffer_gt<compressed_slot_t, compressed_slot_allocator_t> woman_to_man_slots(women.size());
|
|
3273
|
+
if (!proposal_counts || !man_to_woman_slots || !woman_to_man_slots)
|
|
3274
|
+
return result.failed("Can't temporary mappings");
|
|
3275
|
+
|
|
3276
|
+
compressed_slot_t missing_slot;
|
|
3277
|
+
std::memset((void*)&missing_slot, 0xFF, sizeof(compressed_slot_t));
|
|
3278
|
+
std::memset((void*)man_to_woman_slots.data(), 0xFF, sizeof(compressed_slot_t) * men.size());
|
|
3279
|
+
std::memset((void*)woman_to_man_slots.data(), 0xFF, sizeof(compressed_slot_t) * women.size());
|
|
3280
|
+
std::memset(proposal_counts.data(), 0, sizeof(proposals_count_t) * men.size());
|
|
3281
|
+
|
|
3282
|
+
// Define locks, to limit concurrent accesses to `man_to_woman_slots` and `woman_to_man_slots`.
|
|
3283
|
+
visits_bitset_t men_locks(men.size()), women_locks(women.size());
|
|
3284
|
+
if (!men_locks || !women_locks)
|
|
3285
|
+
return result.failed("Can't allocate locks");
|
|
3286
|
+
|
|
3287
|
+
std::atomic<std::size_t> rounds{0};
|
|
3288
|
+
std::atomic<std::size_t> engagements{0};
|
|
3289
|
+
std::atomic<std::size_t> computed_distances{0};
|
|
3290
|
+
std::atomic<std::size_t> visited_members{0};
|
|
3291
|
+
|
|
3292
|
+
// Concurrently process all the men
|
|
3293
|
+
executor.execute_bulk([&](std::size_t thread_idx) {
|
|
3294
|
+
index_search_config_t search_config;
|
|
3295
|
+
search_config.expansion = config.expansion;
|
|
3296
|
+
search_config.exact = config.exact;
|
|
3297
|
+
search_config.thread = thread_idx;
|
|
3298
|
+
compressed_slot_t free_man_slot;
|
|
3299
|
+
|
|
3300
|
+
// While there exist a free man who still has a woman to propose to.
|
|
3301
|
+
while (true) {
|
|
3302
|
+
std::size_t passed_rounds = 0;
|
|
3303
|
+
std::size_t total_rounds = 0;
|
|
3304
|
+
{
|
|
3305
|
+
std::unique_lock<std::mutex> pop_lock(free_men_mutex);
|
|
3306
|
+
if (!free_men.try_pop(free_man_slot))
|
|
3307
|
+
// Primary exit path, we have exhausted the list of candidates
|
|
3308
|
+
break;
|
|
3309
|
+
passed_rounds = ++rounds;
|
|
3310
|
+
total_rounds = passed_rounds + free_men.size();
|
|
3311
|
+
}
|
|
3312
|
+
progress(passed_rounds, total_rounds);
|
|
3313
|
+
while (men_locks.atomic_set(free_man_slot))
|
|
3314
|
+
;
|
|
3315
|
+
|
|
3316
|
+
proposals_count_t& free_man_proposals = proposal_counts[free_man_slot];
|
|
3317
|
+
if (free_man_proposals >= config.max_proposals)
|
|
3318
|
+
continue;
|
|
3319
|
+
|
|
3320
|
+
// Find the closest woman, to whom this man hasn't proposed yet.
|
|
3321
|
+
++free_man_proposals;
|
|
3322
|
+
auto candidates = women.search(men_values[free_man_slot], free_man_proposals, women_metric, search_config);
|
|
3323
|
+
visited_members += candidates.visited_members;
|
|
3324
|
+
computed_distances += candidates.computed_distances;
|
|
3325
|
+
if (!candidates) {
|
|
3326
|
+
// TODO:
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
auto match = candidates.back();
|
|
3330
|
+
auto woman = match.member;
|
|
3331
|
+
while (women_locks.atomic_set(woman.slot))
|
|
3332
|
+
;
|
|
3333
|
+
|
|
3334
|
+
compressed_slot_t husband_slot = woman_to_man_slots[woman.slot];
|
|
3335
|
+
bool woman_is_free = husband_slot == missing_slot;
|
|
3336
|
+
if (woman_is_free) {
|
|
3337
|
+
// Engagement
|
|
3338
|
+
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
3339
|
+
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3340
|
+
engagements++;
|
|
3341
|
+
} else {
|
|
3342
|
+
distance_t distance_from_husband = women_metric(women_values[woman.slot], men_values[husband_slot]);
|
|
3343
|
+
distance_t distance_from_candidate = match.distance;
|
|
3344
|
+
if (distance_from_husband > distance_from_candidate) {
|
|
3345
|
+
// Break-up
|
|
3346
|
+
while (men_locks.atomic_set(husband_slot))
|
|
3347
|
+
;
|
|
3348
|
+
man_to_woman_slots[husband_slot] = missing_slot;
|
|
3349
|
+
men_locks.atomic_reset(husband_slot);
|
|
3350
|
+
|
|
3351
|
+
// New Engagement
|
|
3352
|
+
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
3353
|
+
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3354
|
+
engagements++;
|
|
3355
|
+
|
|
3356
|
+
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
3357
|
+
free_men.push(husband_slot);
|
|
3358
|
+
} else {
|
|
3359
|
+
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
3360
|
+
free_men.push(free_man_slot);
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
|
|
3364
|
+
men_locks.atomic_reset(free_man_slot);
|
|
3365
|
+
women_locks.atomic_reset(woman.slot);
|
|
3366
|
+
}
|
|
3367
|
+
});
|
|
3368
|
+
|
|
3369
|
+
// Export the "slots" into keys:
|
|
3370
|
+
std::size_t intersection_size = 0;
|
|
3371
|
+
for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
|
|
3372
|
+
compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
|
|
3373
|
+
if (woman_slot != missing_slot) {
|
|
3374
|
+
man_key_t man = men.at(man_slot).key;
|
|
3375
|
+
woman_key_t woman = women.at(woman_slot).key;
|
|
3376
|
+
man_to_woman[man] = woman;
|
|
3377
|
+
woman_to_man[woman] = man;
|
|
3378
|
+
intersection_size++;
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
|
|
3382
|
+
// Export stats
|
|
3383
|
+
result.engagements = engagements;
|
|
3384
|
+
result.intersection_size = intersection_size;
|
|
3385
|
+
result.computed_distances = computed_distances;
|
|
3386
|
+
result.visited_members = visited_members;
|
|
3313
3387
|
return result;
|
|
3314
3388
|
}
|
|
3315
3389
|
|