usearch 0.22.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.releaserc +10 -2
- package/.vscode/launch.json +10 -10
- package/.vscode/settings.json +9 -1
- package/.vscode/tasks.json +4 -4
- package/CMakeLists.txt +6 -1
- package/Cargo.toml +2 -2
- package/README.md +35 -41
- package/build.rs +2 -2
- package/c/CMakeLists.txt +48 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +83 -72
- package/c/test.c +20 -20
- package/c/usearch.h +132 -32
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +3 -1
- package/cpp/README.md +16 -16
- package/cpp/bench.cpp +50 -126
- package/cpp/test.cpp +164 -80
- package/docs/benchmarks.md +26 -5
- package/docs/c/reference.rst +3 -3
- package/docs/compilation.md +31 -49
- package/docs/cpp/index.rst +1 -1
- package/docs/cpp/reference.rst +4 -4
- package/golang/lib.go +275 -141
- package/golang/lib_test.go +105 -0
- package/include/usearch/index.hpp +1681 -1656
- package/include/usearch/index_dense.hpp +1400 -0
- package/include/usearch/index_plugins.hpp +1263 -0
- package/java/README.md +1 -1
- package/java/cloud/unum/usearch/Index.java +10 -10
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
- package/java/test/IndexTest.java +1 -1
- package/javascript/README.md +6 -6
- package/javascript/lib.cpp +156 -55
- package/javascript/test.js +26 -11
- package/javascript/usearch.d.ts +102 -0
- package/javascript/usearch.js +2 -118
- package/objc/USearchObjective.mm +57 -31
- package/objc/include/USearchObjective.h +32 -22
- package/package.json +5 -3
- package/python/README.md +23 -14
- package/python/lib.cpp +468 -444
- package/python/scripts/bench.ipynb +3 -2
- package/python/scripts/bench.py +4 -4
- package/python/scripts/index_faiss.py +5 -5
- package/python/scripts/join.py +4 -4
- package/python/scripts/test.py +152 -61
- package/python/scripts/test_jit.py +10 -12
- package/python/scripts/test_sparse.py +32 -0
- package/python/usearch/client.py +26 -28
- package/python/usearch/eval.py +73 -23
- package/python/usearch/index.py +520 -228
- package/python/usearch/io.py +57 -25
- package/python/usearch/numba.py +4 -4
- package/python/usearch/server.py +10 -10
- package/rust/lib.cpp +34 -30
- package/rust/lib.hpp +14 -12
- package/rust/lib.rs +8 -7
- package/setup.py +5 -2
- package/simsimd/.github/workflows/prerelease.yml +73 -0
- package/simsimd/.github/workflows/release.yml +110 -0
- package/simsimd/.github/workflows/update_version.sh +3 -0
- package/simsimd/.releaserc +77 -0
- package/simsimd/.vscode/launch.json +27 -1
- package/simsimd/.vscode/settings.json +22 -1
- package/simsimd/.vscode/tasks.json +6 -0
- package/simsimd/CMakeLists.txt +3 -2
- package/simsimd/README.md +82 -46
- package/simsimd/bench.cxx +63 -24
- package/simsimd/include/simsimd/simsimd.h +88 -66
- package/simsimd/include/simsimd/simsimd_chem.h +126 -0
- package/simsimd/package.json +8 -0
- package/simsimd/pyproject.toml +42 -0
- package/simsimd/python/python.c +81 -0
- package/simsimd/python/run.sh +1 -0
- package/simsimd/python/test.c +109 -0
- package/simsimd/python/test.py +23 -0
- package/simsimd/setup.py +44 -0
- package/swift/Index+Sugar.swift +75 -23
- package/swift/README.md +26 -2
- package/swift/Test.swift +19 -7
- package/swift/USearch.swift +2 -2
- package/tsconfig.json +8 -0
- package/wasm/CMakeLists.txt +1 -0
- package/wasmer.toml +1 -1
- package/wolfram/CMakeLists.txt +1 -0
- package/wolfram/lib.cpp +11 -13
- package/golang/usearch.h +0 -9
- package/golang/usearch_par_test.go +0 -40
- package/golang/usearch_test.go +0 -82
- package/include/usearch/index_punned_dense.hpp +0 -1001
- package/include/usearch/index_punned_helpers.hpp +0 -676
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file index.hpp
|
|
3
|
-
* @author
|
|
3
|
+
* @author Ash Vardanian
|
|
4
4
|
* @brief Single-header Vector Search.
|
|
5
5
|
* @date 2023-04-26
|
|
6
6
|
*
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
#define USEARCH_VERSION_PATCH 0
|
|
15
15
|
|
|
16
16
|
// Inferring C++ version
|
|
17
|
+
// https://stackoverflow.com/a/61552074
|
|
17
18
|
#if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
|
|
18
19
|
#define USEARCH_DEFINED_CPP17
|
|
19
20
|
#endif
|
|
@@ -42,10 +43,13 @@
|
|
|
42
43
|
#endif
|
|
43
44
|
|
|
44
45
|
// Inferring hardware bitness: 32 vs 64
|
|
45
|
-
|
|
46
|
+
// https://stackoverflow.com/a/5273354
|
|
47
|
+
#if INTPTR_MAX == INT64_MAX
|
|
46
48
|
#define USEARCH_64BIT_ENV
|
|
47
|
-
#
|
|
49
|
+
#elif INTPTR_MAX == INT32_MAX
|
|
48
50
|
#define USEARCH_32BIT_ENV
|
|
51
|
+
#else
|
|
52
|
+
#error Unknown pointer size or missing size macros!
|
|
49
53
|
#endif
|
|
50
54
|
|
|
51
55
|
#if !defined(USEARCH_USE_OPENMP)
|
|
@@ -119,51 +123,7 @@
|
|
|
119
123
|
namespace unum {
|
|
120
124
|
namespace usearch {
|
|
121
125
|
|
|
122
|
-
using f64_t = double;
|
|
123
|
-
using f32_t = float;
|
|
124
126
|
using byte_t = char;
|
|
125
|
-
enum b1x8_t : unsigned char {};
|
|
126
|
-
|
|
127
|
-
enum class metric_kind_t : std::uint8_t {
|
|
128
|
-
unknown_k = 0,
|
|
129
|
-
// Classics:
|
|
130
|
-
ip_k = 'i',
|
|
131
|
-
cos_k = 'c',
|
|
132
|
-
l2sq_k = 'e',
|
|
133
|
-
|
|
134
|
-
// Custom:
|
|
135
|
-
pearson_k = 'p',
|
|
136
|
-
haversine_k = 'h',
|
|
137
|
-
|
|
138
|
-
// Sets:
|
|
139
|
-
jaccard_k = 'j',
|
|
140
|
-
hamming_k = 'b',
|
|
141
|
-
tanimoto_k = 't',
|
|
142
|
-
sorensen_k = 's',
|
|
143
|
-
};
|
|
144
|
-
|
|
145
|
-
enum class scalar_kind_t : std::uint8_t {
|
|
146
|
-
unknown_k = 0,
|
|
147
|
-
f64_k,
|
|
148
|
-
f32_k,
|
|
149
|
-
f16_k,
|
|
150
|
-
f8_k,
|
|
151
|
-
b1x8_k,
|
|
152
|
-
};
|
|
153
|
-
|
|
154
|
-
template <typename scalar_at> scalar_kind_t common_scalar_kind() noexcept {
|
|
155
|
-
if (std::is_same<scalar_at, f32_t>())
|
|
156
|
-
return scalar_kind_t::f32_k;
|
|
157
|
-
if (std::is_same<scalar_at, f64_t>())
|
|
158
|
-
return scalar_kind_t::f64_k;
|
|
159
|
-
if (std::is_same<scalar_at, b1x8_t>())
|
|
160
|
-
return scalar_kind_t::b1x8_k;
|
|
161
|
-
return scalar_kind_t::unknown_k;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
|
|
165
|
-
|
|
166
|
-
template <typename at> at square(at value) noexcept { return value * value; }
|
|
167
127
|
|
|
168
128
|
template <std::size_t multiple_ak> std::size_t divide_round_up(std::size_t num) noexcept {
|
|
169
129
|
return (num + multiple_ak - 1) / multiple_ak;
|
|
@@ -187,6 +147,7 @@ inline std::size_t ceil2(std::size_t v) noexcept {
|
|
|
187
147
|
return v;
|
|
188
148
|
}
|
|
189
149
|
|
|
150
|
+
/// @brief Simply dereferencing misaligned pointers can be dangerous.
|
|
190
151
|
template <typename at> void misaligned_store(void* ptr, at v) noexcept {
|
|
191
152
|
static_assert(!std::is_reference<at>::value, "Can't store a reference");
|
|
192
153
|
std::memcpy(ptr, &v, sizeof(at));
|
|
@@ -207,54 +168,80 @@ template <typename at, typename other_at = at> at exchange(at& obj, other_at&& n
|
|
|
207
168
|
return old_value;
|
|
208
169
|
}
|
|
209
170
|
|
|
171
|
+
/// @brief The `std::destroy_at` alternative for C++11.
|
|
172
|
+
template <typename at, typename sfinae_at = at>
|
|
173
|
+
typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
|
|
174
|
+
template <typename at, typename sfinae_at = at>
|
|
175
|
+
typename std::enable_if<!std::is_pod<sfinae_at>::value>::type destroy_at(at* obj) {
|
|
176
|
+
obj->~sfinae_at();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/// @brief The `std::construct_at` alternative for C++11.
|
|
180
|
+
template <typename at, typename sfinae_at = at>
|
|
181
|
+
typename std::enable_if<std::is_pod<sfinae_at>::value>::type construct_at(at*) {}
|
|
182
|
+
template <typename at, typename sfinae_at = at>
|
|
183
|
+
typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* obj) {
|
|
184
|
+
new (obj) at();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* @brief A reference to a misaligned memory location with a specific type.
|
|
189
|
+
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
190
|
+
* indivisible by `sizeof(at)`.
|
|
191
|
+
*/
|
|
210
192
|
template <typename at> class misaligned_ref_gt {
|
|
193
|
+
using element_t = at;
|
|
194
|
+
using mutable_t = typename std::remove_const<element_t>::type;
|
|
211
195
|
byte_t* ptr_;
|
|
212
196
|
|
|
213
197
|
public:
|
|
214
198
|
misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
215
|
-
operator
|
|
216
|
-
misaligned_ref_gt& operator=(
|
|
217
|
-
misaligned_store<
|
|
199
|
+
operator mutable_t() const noexcept { return misaligned_load<mutable_t>(ptr_); }
|
|
200
|
+
misaligned_ref_gt& operator=(mutable_t const& v) noexcept {
|
|
201
|
+
misaligned_store<mutable_t>(ptr_, v);
|
|
218
202
|
return *this;
|
|
219
203
|
}
|
|
220
204
|
|
|
221
205
|
void reset(byte_t* ptr) noexcept { ptr_ = ptr; }
|
|
206
|
+
byte_t* ptr() const noexcept { return ptr_; }
|
|
222
207
|
};
|
|
223
208
|
|
|
209
|
+
/**
|
|
210
|
+
* @brief A pointer to a misaligned memory location with a specific type.
|
|
211
|
+
* It is needed to avoid Undefined Behavior when dereferencing addresses
|
|
212
|
+
* indivisible by `sizeof(at)`.
|
|
213
|
+
*/
|
|
224
214
|
template <typename at> class misaligned_ptr_gt {
|
|
215
|
+
using element_t = at;
|
|
216
|
+
using mutable_t = typename std::remove_const<element_t>::type;
|
|
225
217
|
byte_t* ptr_;
|
|
226
218
|
|
|
227
219
|
public:
|
|
228
220
|
using iterator_category = std::random_access_iterator_tag;
|
|
229
|
-
using value_type =
|
|
221
|
+
using value_type = element_t;
|
|
230
222
|
using difference_type = std::ptrdiff_t;
|
|
231
|
-
using pointer = misaligned_ptr_gt<
|
|
232
|
-
using reference = misaligned_ref_gt<
|
|
223
|
+
using pointer = misaligned_ptr_gt<element_t>;
|
|
224
|
+
using reference = misaligned_ref_gt<element_t>;
|
|
233
225
|
|
|
234
226
|
reference operator*() const noexcept { return {ptr_}; }
|
|
227
|
+
reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
|
|
228
|
+
value_type operator[](std::size_t i) const noexcept {
|
|
229
|
+
return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
|
|
230
|
+
}
|
|
235
231
|
|
|
236
232
|
misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
|
|
237
|
-
misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(
|
|
238
|
-
misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(
|
|
239
|
-
misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(
|
|
240
|
-
misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(
|
|
233
|
+
misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(element_t)); }
|
|
234
|
+
misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(element_t)); }
|
|
235
|
+
misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(element_t)); }
|
|
236
|
+
misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(element_t)); }
|
|
237
|
+
|
|
238
|
+
// clang-format off
|
|
239
|
+
misaligned_ptr_gt& operator++() noexcept { ptr_ += sizeof(element_t); return *this; }
|
|
240
|
+
misaligned_ptr_gt& operator--() noexcept { ptr_ -= sizeof(element_t); return *this; }
|
|
241
|
+
misaligned_ptr_gt& operator+=(difference_type d) noexcept { ptr_ += d * sizeof(element_t); return *this; }
|
|
242
|
+
misaligned_ptr_gt& operator-=(difference_type d) noexcept { ptr_ -= d * sizeof(element_t); return *this; }
|
|
243
|
+
// clang-format on
|
|
241
244
|
|
|
242
|
-
misaligned_ptr_gt& operator++() noexcept {
|
|
243
|
-
ptr_ += sizeof(at);
|
|
244
|
-
return *this;
|
|
245
|
-
}
|
|
246
|
-
misaligned_ptr_gt& operator--() noexcept {
|
|
247
|
-
ptr_ -= sizeof(at);
|
|
248
|
-
return *this;
|
|
249
|
-
}
|
|
250
|
-
misaligned_ptr_gt& operator+=(difference_type d) noexcept {
|
|
251
|
-
ptr_ += d * sizeof(at);
|
|
252
|
-
return *this;
|
|
253
|
-
}
|
|
254
|
-
misaligned_ptr_gt& operator-=(difference_type d) noexcept {
|
|
255
|
-
ptr_ -= d * sizeof(at);
|
|
256
|
-
return *this;
|
|
257
|
-
}
|
|
258
245
|
bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
|
|
259
246
|
bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
|
|
260
247
|
};
|
|
@@ -278,307 +265,56 @@ template <typename scalar_at> class span_gt {
|
|
|
278
265
|
};
|
|
279
266
|
|
|
280
267
|
/**
|
|
281
|
-
* @brief
|
|
282
|
-
|
|
283
|
-
template <typename scalar_at = float, typename result_at = scalar_at> struct ip_gt {
|
|
284
|
-
using scalar_t = scalar_at;
|
|
285
|
-
using view_t = span_gt<scalar_t const>;
|
|
286
|
-
using result_t = result_at;
|
|
287
|
-
using result_type = result_t;
|
|
288
|
-
|
|
289
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::ip_k; }
|
|
290
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
291
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
292
|
-
|
|
293
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
294
|
-
result_type ab{};
|
|
295
|
-
#if USEARCH_USE_OPENMP
|
|
296
|
-
#pragma omp simd reduction(+ : ab)
|
|
297
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
298
|
-
#pragma clang loop vectorize(enable)
|
|
299
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
300
|
-
#pragma GCC ivdep
|
|
301
|
-
#endif
|
|
302
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
303
|
-
ab += result_t(a[i]) * result_t(b[i]);
|
|
304
|
-
return 1 - ab;
|
|
305
|
-
}
|
|
306
|
-
};
|
|
307
|
-
|
|
308
|
-
/**
|
|
309
|
-
* @brief Cosine (Angular) distance.
|
|
310
|
-
* Identical to the Inner Product of normalized vectors.
|
|
311
|
-
* Unless you are running on an tiny embedded platform, this metric
|
|
312
|
-
* is recommended over `::ip_gt` for low-precision scalars.
|
|
313
|
-
*/
|
|
314
|
-
template <typename scalar_at = float, typename result_at = scalar_at> struct cos_gt {
|
|
315
|
-
using scalar_t = scalar_at;
|
|
316
|
-
using view_t = span_gt<scalar_t const>;
|
|
317
|
-
using result_t = result_at;
|
|
318
|
-
using result_type = result_t;
|
|
319
|
-
|
|
320
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::cos_k; }
|
|
321
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
322
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
323
|
-
|
|
324
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
325
|
-
result_t ab{}, a2{}, b2{};
|
|
326
|
-
#if USEARCH_USE_OPENMP
|
|
327
|
-
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
328
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
329
|
-
#pragma clang loop vectorize(enable)
|
|
330
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
331
|
-
#pragma GCC ivdep
|
|
332
|
-
#endif
|
|
333
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
334
|
-
ab += result_t(a[i]) * result_t(b[i]), //
|
|
335
|
-
a2 += square<result_t>(a[i]), //
|
|
336
|
-
b2 += square<result_t>(b[i]);
|
|
337
|
-
return (ab != 0) ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 1;
|
|
338
|
-
}
|
|
339
|
-
};
|
|
340
|
-
|
|
341
|
-
/**
|
|
342
|
-
* @brief Squared Euclidean (L2) distance.
|
|
343
|
-
* Square root is avoided at the end, as it won't affect the ordering.
|
|
268
|
+
* @brief Similar to `std::vector`, but doesn't support dynamic resizing.
|
|
269
|
+
* On the bright side, this can't throw exceptions.
|
|
344
270
|
*/
|
|
345
|
-
template <typename scalar_at
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
using result_t = result_at;
|
|
349
|
-
using result_type = result_t;
|
|
350
|
-
|
|
351
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::l2sq_k; }
|
|
352
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
353
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
354
|
-
|
|
355
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
356
|
-
result_t ab_deltas_sq{};
|
|
357
|
-
#if USEARCH_USE_OPENMP
|
|
358
|
-
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
359
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
360
|
-
#pragma clang loop vectorize(enable)
|
|
361
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
362
|
-
#pragma GCC ivdep
|
|
363
|
-
#endif
|
|
364
|
-
for (std::size_t i = 0; i != dim; ++i)
|
|
365
|
-
ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
|
|
366
|
-
return ab_deltas_sq;
|
|
367
|
-
}
|
|
368
|
-
};
|
|
369
|
-
|
|
370
|
-
/**
|
|
371
|
-
* @brief Hamming distance computes the number of differing bits in
|
|
372
|
-
* two arrays of integers. An example would be a textual document,
|
|
373
|
-
* tokenized and hashed into a fixed-capacity bitset.
|
|
374
|
-
*/
|
|
375
|
-
template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct hamming_gt {
|
|
376
|
-
using scalar_t = scalar_at;
|
|
377
|
-
using view_t = span_gt<scalar_t const>;
|
|
378
|
-
using result_t = result_at;
|
|
379
|
-
using result_type = result_t;
|
|
380
|
-
static_assert( //
|
|
381
|
-
std::is_unsigned<scalar_t>::value ||
|
|
382
|
-
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
383
|
-
"Hamming distance requires unsigned integral words");
|
|
384
|
-
|
|
385
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::hamming_k; }
|
|
386
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
387
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
388
|
-
|
|
389
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
390
|
-
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
391
|
-
result_t matches{};
|
|
392
|
-
#if USEARCH_USE_OPENMP
|
|
393
|
-
#pragma omp simd reduction(+ : matches)
|
|
394
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
395
|
-
#pragma clang loop vectorize(enable)
|
|
396
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
397
|
-
#pragma GCC ivdep
|
|
398
|
-
#endif
|
|
399
|
-
for (std::size_t i = 0; i != words; ++i)
|
|
400
|
-
matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
|
|
401
|
-
return matches;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
271
|
+
template <typename scalar_at, typename allocator_at> class buffer_gt {
|
|
272
|
+
scalar_at* data_;
|
|
273
|
+
std::size_t size_;
|
|
404
274
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::tanimoto_k; }
|
|
421
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
422
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
423
|
-
|
|
424
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
425
|
-
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
426
|
-
result_t and_count{};
|
|
427
|
-
result_t or_count{};
|
|
428
|
-
#if USEARCH_USE_OPENMP
|
|
429
|
-
#pragma omp simd reduction(+ : and_count, or_count)
|
|
430
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
431
|
-
#pragma clang loop vectorize(enable)
|
|
432
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
433
|
-
#pragma GCC ivdep
|
|
434
|
-
#endif
|
|
435
|
-
for (std::size_t i = 0; i != words; ++i)
|
|
436
|
-
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
437
|
-
or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
|
|
438
|
-
return 1 - result_t(and_count) / or_count;
|
|
275
|
+
public:
|
|
276
|
+
buffer_gt() noexcept : data_(nullptr), size_(0u) {}
|
|
277
|
+
buffer_gt(std::size_t size) noexcept : data_(allocator_at{}.allocate(size)), size_(data_ ? size : 0u) {
|
|
278
|
+
if (!std::is_trivially_default_constructible<scalar_at>::value)
|
|
279
|
+
for (std::size_t i = 0; i != size_; ++i)
|
|
280
|
+
construct_at(data_ + i);
|
|
281
|
+
}
|
|
282
|
+
~buffer_gt() noexcept {
|
|
283
|
+
if (!std::is_trivially_destructible<scalar_at>::value)
|
|
284
|
+
for (std::size_t i = 0; i != size_; ++i)
|
|
285
|
+
destroy_at(data_ + i);
|
|
286
|
+
allocator_at{}.deallocate(data_, size_);
|
|
287
|
+
data_ = nullptr;
|
|
288
|
+
size_ = 0;
|
|
439
289
|
}
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
*
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
static_assert( //
|
|
452
|
-
std::is_unsigned<scalar_t>::value ||
|
|
453
|
-
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
454
|
-
"Sorensen-Dice distance requires unsigned integral words");
|
|
455
|
-
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
456
|
-
|
|
457
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::sorensen_k; }
|
|
458
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
459
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
460
|
-
|
|
461
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
462
|
-
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
463
|
-
result_t and_count{};
|
|
464
|
-
result_t any_count{};
|
|
465
|
-
#if USEARCH_USE_OPENMP
|
|
466
|
-
#pragma omp simd reduction(+ : and_count, any_count)
|
|
467
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
468
|
-
#pragma clang loop vectorize(enable)
|
|
469
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
470
|
-
#pragma GCC ivdep
|
|
471
|
-
#endif
|
|
472
|
-
for (std::size_t i = 0; i != words; ++i)
|
|
473
|
-
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
474
|
-
any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
|
|
475
|
-
return 1 - 2 * result_t(and_count) / any_count;
|
|
290
|
+
scalar_at* data() const noexcept { return data_; }
|
|
291
|
+
std::size_t size() const noexcept { return size_; }
|
|
292
|
+
scalar_at* begin() const noexcept { return data_; }
|
|
293
|
+
scalar_at* end() const noexcept { return data_ + size_; }
|
|
294
|
+
operator scalar_at*() const noexcept { return data(); }
|
|
295
|
+
scalar_at& operator[](std::size_t i) noexcept { return data_[i]; }
|
|
296
|
+
scalar_at const& operator[](std::size_t i) const noexcept { return data_[i]; }
|
|
297
|
+
explicit operator bool() const noexcept { return data_; }
|
|
298
|
+
scalar_at* release() noexcept {
|
|
299
|
+
size_ = 0;
|
|
300
|
+
return exchange(data_, nullptr);
|
|
476
301
|
}
|
|
477
|
-
};
|
|
478
302
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
* Can be used to compute the similarity between two textual documents
|
|
482
|
-
* using the IDs of tokens present in them.
|
|
483
|
-
* Similar to `tanimoto_gt` for dense representations.
|
|
484
|
-
*/
|
|
485
|
-
template <typename scalar_at = std::int32_t, typename result_at = float> struct jaccard_gt {
|
|
486
|
-
using scalar_t = scalar_at;
|
|
487
|
-
using view_t = span_gt<scalar_t const>;
|
|
488
|
-
using result_t = result_at;
|
|
489
|
-
using result_type = result_t;
|
|
490
|
-
static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
|
|
491
|
-
|
|
492
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::jaccard_k; }
|
|
493
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
494
|
-
inline result_t operator()(view_t a, view_t b) const noexcept {
|
|
495
|
-
return operator()(a.data(), b.data(), a.size(), b.size());
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
inline result_t operator()( //
|
|
499
|
-
scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
|
|
500
|
-
result_t intersection{};
|
|
501
|
-
std::size_t i{};
|
|
502
|
-
std::size_t j{};
|
|
503
|
-
while (i != a_length && j != b_length) {
|
|
504
|
-
intersection += a[i] == b[j];
|
|
505
|
-
i += a[i] < b[j];
|
|
506
|
-
j += a[i] >= b[j];
|
|
507
|
-
}
|
|
508
|
-
return 1 - intersection / (a_length + b_length - intersection);
|
|
509
|
-
}
|
|
510
|
-
};
|
|
303
|
+
buffer_gt(buffer_gt const&) = delete;
|
|
304
|
+
buffer_gt& operator=(buffer_gt const&) = delete;
|
|
511
305
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
template <typename scalar_at = float, typename result_at = float> struct pearson_correlation_gt {
|
|
518
|
-
using scalar_t = scalar_at;
|
|
519
|
-
using view_t = span_gt<scalar_t const>;
|
|
520
|
-
using result_t = result_at;
|
|
521
|
-
using result_type = result_t;
|
|
522
|
-
|
|
523
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::pearson_k; }
|
|
524
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
525
|
-
inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
|
|
526
|
-
|
|
527
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
528
|
-
result_t a_sum{}, b_sum{}, ab_sum{};
|
|
529
|
-
result_t a_sq_sum{}, b_sq_sum{};
|
|
530
|
-
#if USEARCH_USE_OPENMP
|
|
531
|
-
#pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
|
|
532
|
-
#elif defined(USEARCH_DEFINED_CLANG)
|
|
533
|
-
#pragma clang loop vectorize(enable)
|
|
534
|
-
#elif defined(USEARCH_DEFINED_GCC)
|
|
535
|
-
#pragma GCC ivdep
|
|
536
|
-
#endif
|
|
537
|
-
for (std::size_t i = 0; i != dim; ++i) {
|
|
538
|
-
a_sum += result_t(a[i]);
|
|
539
|
-
b_sum += result_t(b[i]);
|
|
540
|
-
ab_sum += result_t(a[i]) * result_t(b[i]);
|
|
541
|
-
a_sq_sum += result_t(a[i]) * result_t(a[i]);
|
|
542
|
-
b_sq_sum += result_t(b[i]) * result_t(b[i]);
|
|
543
|
-
}
|
|
544
|
-
result_t denom = std::sqrt((dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum));
|
|
545
|
-
result_t corr = (dim * ab_sum - a_sum * b_sum) / denom;
|
|
546
|
-
return -corr;
|
|
306
|
+
buffer_gt(buffer_gt&& other) noexcept : data_(exchange(other.data_, nullptr)), size_(exchange(other.size_, 0)) {}
|
|
307
|
+
buffer_gt& operator=(buffer_gt&& other) noexcept {
|
|
308
|
+
std::swap(data_, other.data_);
|
|
309
|
+
std::swap(size_, other.size_);
|
|
310
|
+
return *this;
|
|
547
311
|
}
|
|
548
312
|
};
|
|
549
313
|
|
|
550
314
|
/**
|
|
551
|
-
* @brief
|
|
552
|
-
*
|
|
315
|
+
* @brief A lightweight error class for handling error messages,
|
|
316
|
+
* which are expected to be allocated in static memory.
|
|
553
317
|
*/
|
|
554
|
-
template <typename scalar_at = float, typename result_at = scalar_at> struct haversine_gt {
|
|
555
|
-
using scalar_t = scalar_at;
|
|
556
|
-
using view_t = span_gt<scalar_t const>;
|
|
557
|
-
using result_t = result_at;
|
|
558
|
-
using result_type = result_t;
|
|
559
|
-
static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
|
|
560
|
-
|
|
561
|
-
inline metric_kind_t kind() const noexcept { return metric_kind_t::haversine_k; }
|
|
562
|
-
inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
|
|
563
|
-
|
|
564
|
-
inline result_t operator()(scalar_t const* a, scalar_t const* b) const noexcept {
|
|
565
|
-
result_t lat_a = a[0], lon_a = a[1];
|
|
566
|
-
result_t lat_b = b[0], lon_b = b[1];
|
|
567
|
-
|
|
568
|
-
result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a);
|
|
569
|
-
result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a);
|
|
570
|
-
|
|
571
|
-
result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
|
|
572
|
-
result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
|
|
573
|
-
|
|
574
|
-
result_t x = //
|
|
575
|
-
square(std::sin(lat_delta / 2.f)) +
|
|
576
|
-
std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta / 2.f));
|
|
577
|
-
|
|
578
|
-
return std::atan2(std::sqrt(x), std::sqrt(1.f - x));
|
|
579
|
-
}
|
|
580
|
-
};
|
|
581
|
-
|
|
582
318
|
class error_t {
|
|
583
319
|
char const* message_{};
|
|
584
320
|
|
|
@@ -596,11 +332,14 @@ class error_t {
|
|
|
596
332
|
explicit operator bool() const noexcept { return message_ != nullptr; }
|
|
597
333
|
char const* what() const noexcept { return message_; }
|
|
598
334
|
|
|
599
|
-
#if defined(__cpp_exceptions)
|
|
335
|
+
#if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
|
|
600
336
|
~error_t() noexcept(false) {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
337
|
+
#if defined(USEARCH_DEFINED_CPP17)
|
|
338
|
+
if (message_ && std::uncaught_exceptions() == 0)
|
|
339
|
+
#else
|
|
340
|
+
if (message_ && std::uncaught_exception() == 0)
|
|
341
|
+
#endif
|
|
342
|
+
raise();
|
|
604
343
|
}
|
|
605
344
|
void raise() noexcept(false) {
|
|
606
345
|
if (message_)
|
|
@@ -615,6 +354,13 @@ class error_t {
|
|
|
615
354
|
#endif
|
|
616
355
|
};
|
|
617
356
|
|
|
357
|
+
/**
|
|
358
|
+
* @brief Similar to `std::expected` in C++23, wraps a statement evaluation result,
|
|
359
|
+
* or an error. It's used to avoid raising exception, and gracefully propagate
|
|
360
|
+
* the error.
|
|
361
|
+
*
|
|
362
|
+
* @tparam result_at The type of the expected result.
|
|
363
|
+
*/
|
|
618
364
|
template <typename result_at> struct expected_gt {
|
|
619
365
|
result_at result;
|
|
620
366
|
error_t error;
|
|
@@ -637,49 +383,41 @@ template <typename result_at> struct expected_gt {
|
|
|
637
383
|
|
|
638
384
|
/**
|
|
639
385
|
* @brief Light-weight bitset implementation to track visited nodes during graph traversal.
|
|
640
|
-
* Extends basic functionality with atomic operations.
|
|
386
|
+
* Extends basic functionality with @b atomic operations.
|
|
641
387
|
*/
|
|
642
|
-
template <typename allocator_at = std::allocator<
|
|
388
|
+
template <typename allocator_at = std::allocator<byte_t>> class visits_bitset_gt {
|
|
643
389
|
using allocator_t = allocator_at;
|
|
644
390
|
using byte_t = typename allocator_t::value_type;
|
|
645
391
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
646
392
|
|
|
647
|
-
using
|
|
393
|
+
using compressed_slot_t = unsigned long;
|
|
648
394
|
|
|
649
|
-
static constexpr std::size_t bits_per_slot() { return sizeof(
|
|
650
|
-
static constexpr
|
|
395
|
+
static constexpr std::size_t bits_per_slot() { return sizeof(compressed_slot_t) * CHAR_BIT; }
|
|
396
|
+
static constexpr compressed_slot_t bits_mask() { return sizeof(compressed_slot_t) * CHAR_BIT - 1; }
|
|
397
|
+
static constexpr std::size_t slots(std::size_t bits) { return divide_round_up<bits_per_slot()>(bits); }
|
|
651
398
|
|
|
652
|
-
|
|
399
|
+
compressed_slot_t* slots_{};
|
|
653
400
|
/// @brief Number of slots.
|
|
654
401
|
std::size_t count_{};
|
|
655
402
|
|
|
656
403
|
public:
|
|
657
404
|
visits_bitset_gt() noexcept {}
|
|
658
405
|
~visits_bitset_gt() noexcept { reset(); }
|
|
659
|
-
|
|
406
|
+
|
|
407
|
+
explicit operator bool() const noexcept { return slots_; }
|
|
408
|
+
void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(compressed_slot_t)); }
|
|
660
409
|
|
|
661
410
|
void reset() noexcept {
|
|
662
411
|
if (slots_)
|
|
663
|
-
allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(
|
|
412
|
+
allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(compressed_slot_t));
|
|
664
413
|
slots_ = nullptr;
|
|
665
414
|
count_ = 0;
|
|
666
415
|
}
|
|
667
416
|
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
if (count <= count_)
|
|
672
|
-
return true;
|
|
673
|
-
|
|
674
|
-
slot_t* slots = (slot_t*)allocator_t{}.allocate(count * sizeof(slot_t));
|
|
675
|
-
if (!slots)
|
|
676
|
-
return false;
|
|
677
|
-
|
|
678
|
-
reset();
|
|
679
|
-
count_ = count;
|
|
680
|
-
slots_ = slots;
|
|
417
|
+
visits_bitset_gt(std::size_t capacity) noexcept
|
|
418
|
+
: slots_((compressed_slot_t*)allocator_t{}.allocate(slots(capacity) * sizeof(compressed_slot_t))),
|
|
419
|
+
count_(slots_ ? slots(capacity) : 0u) {
|
|
681
420
|
clear();
|
|
682
|
-
return true;
|
|
683
421
|
}
|
|
684
422
|
|
|
685
423
|
visits_bitset_gt(visits_bitset_gt&& other) noexcept {
|
|
@@ -697,33 +435,54 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
|
|
|
697
435
|
visits_bitset_gt& operator=(visits_bitset_gt const&) = delete;
|
|
698
436
|
|
|
699
437
|
inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
|
|
700
|
-
inline
|
|
438
|
+
inline bool set(std::size_t i) noexcept {
|
|
439
|
+
compressed_slot_t& slot = slots_[i / bits_per_slot()];
|
|
440
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
441
|
+
bool value = slot & mask;
|
|
442
|
+
slot |= mask;
|
|
443
|
+
return value;
|
|
444
|
+
}
|
|
701
445
|
|
|
702
446
|
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
703
447
|
|
|
704
448
|
inline bool atomic_set(std::size_t i) noexcept {
|
|
705
|
-
|
|
449
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
706
450
|
return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
|
|
707
451
|
}
|
|
708
452
|
|
|
709
453
|
inline void atomic_reset(std::size_t i) noexcept {
|
|
710
|
-
|
|
454
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
711
455
|
InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
|
|
712
456
|
}
|
|
713
457
|
|
|
714
458
|
#else
|
|
715
459
|
|
|
716
460
|
inline bool atomic_set(std::size_t i) noexcept {
|
|
717
|
-
|
|
461
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
718
462
|
return __atomic_fetch_or(&slots_[i / bits_per_slot()], mask, __ATOMIC_ACQUIRE) & mask;
|
|
719
463
|
}
|
|
720
464
|
|
|
721
465
|
inline void atomic_reset(std::size_t i) noexcept {
|
|
722
|
-
|
|
466
|
+
compressed_slot_t mask{1ul << (i & bits_mask())};
|
|
723
467
|
__atomic_fetch_and(&slots_[i / bits_per_slot()], ~mask, __ATOMIC_RELEASE);
|
|
724
468
|
}
|
|
725
469
|
|
|
726
470
|
#endif
|
|
471
|
+
|
|
472
|
+
class lock_t {
|
|
473
|
+
visits_bitset_gt& bitset_;
|
|
474
|
+
std::size_t bit_offset_;
|
|
475
|
+
|
|
476
|
+
public:
|
|
477
|
+
inline ~lock_t() noexcept { bitset_.atomic_reset(bit_offset_); }
|
|
478
|
+
inline lock_t(visits_bitset_gt& bitset, std::size_t bit_offset) noexcept
|
|
479
|
+
: bitset_(bitset), bit_offset_(bit_offset) {
|
|
480
|
+
while (bitset_.atomic_set(bit_offset_))
|
|
481
|
+
;
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
|
|
485
|
+
inline lock_t lock(std::size_t i) noexcept { return {*this, i}; }
|
|
727
486
|
};
|
|
728
487
|
|
|
729
488
|
using visits_bitset_t = visits_bitset_gt<>;
|
|
@@ -1006,10 +765,10 @@ class usearch_pack_m uint40_t {
|
|
|
1006
765
|
|
|
1007
766
|
public:
|
|
1008
767
|
inline uint40_t() noexcept { broadcast(0); }
|
|
1009
|
-
inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets,
|
|
1010
|
-
|
|
1011
|
-
#
|
|
1012
|
-
inline uint40_t(std::
|
|
768
|
+
inline uint40_t(std::uint32_t n) noexcept { std::memcpy(&octets[1], &n, 4); }
|
|
769
|
+
|
|
770
|
+
#ifdef USEARCH_64BIT_ENV
|
|
771
|
+
inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
|
|
1013
772
|
#endif
|
|
1014
773
|
|
|
1015
774
|
uint40_t(uint40_t&&) = default;
|
|
@@ -1017,47 +776,26 @@ class usearch_pack_m uint40_t {
|
|
|
1017
776
|
uint40_t& operator=(uint40_t&&) = default;
|
|
1018
777
|
uint40_t& operator=(uint40_t const&) = default;
|
|
1019
778
|
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
inline uint40_t& operator+=(std::size_t n) noexcept {
|
|
1028
|
-
unsigned char* n_octets = reinterpret_cast<unsigned char*>(&n);
|
|
1029
|
-
std::uint32_t& n_tail = *reinterpret_cast<std::uint32_t*>(n_octets);
|
|
1030
|
-
std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
|
|
1031
|
-
octets[4] += static_cast<unsigned char>((tail + n_tail) < tail);
|
|
1032
|
-
tail += n_tail;
|
|
1033
|
-
octets[4] += n_octets[4];
|
|
1034
|
-
return *this;
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
inline uint40_t operator+(std::size_t n) noexcept {
|
|
1038
|
-
uint40_t other(*this);
|
|
1039
|
-
other += n;
|
|
1040
|
-
return other;
|
|
779
|
+
#if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
|
|
780
|
+
inline uint40_t(std::size_t n) noexcept {
|
|
781
|
+
#ifdef USEARCH_64BIT_ENV
|
|
782
|
+
std::memcpy(octets, &n, 5);
|
|
783
|
+
#else
|
|
784
|
+
std::memcpy(octets, &n, 4);
|
|
785
|
+
#endif
|
|
1041
786
|
}
|
|
787
|
+
#endif
|
|
1042
788
|
|
|
1043
789
|
inline operator std::size_t() const noexcept {
|
|
1044
790
|
std::size_t result = 0;
|
|
1045
791
|
#ifdef USEARCH_64BIT_ENV
|
|
1046
|
-
std::memcpy(
|
|
792
|
+
std::memcpy(&result, octets, 5);
|
|
1047
793
|
#else
|
|
1048
|
-
std::memcpy(
|
|
794
|
+
std::memcpy(&result, octets + 1, 4);
|
|
1049
795
|
#endif
|
|
1050
796
|
return result;
|
|
1051
797
|
}
|
|
1052
798
|
|
|
1053
|
-
inline uint40_t& operator++() noexcept { return *this += 1u; }
|
|
1054
|
-
|
|
1055
|
-
inline uint40_t operator++(int) noexcept {
|
|
1056
|
-
uint40_t old = *this;
|
|
1057
|
-
*this += 1u;
|
|
1058
|
-
return old;
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
799
|
inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
|
|
1062
800
|
inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
|
|
1063
801
|
};
|
|
@@ -1068,6 +806,9 @@ class usearch_pack_m uint40_t {
|
|
|
1068
806
|
|
|
1069
807
|
static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
|
|
1070
808
|
|
|
809
|
+
/**
|
|
810
|
+
* @brief Basic single-threaded @b ring class, used for all kinds of task queues.
|
|
811
|
+
*/
|
|
1071
812
|
template <typename element_at, typename allocator_at = std::allocator<element_at>> //
|
|
1072
813
|
class ring_gt {
|
|
1073
814
|
public:
|
|
@@ -1202,12 +943,6 @@ constexpr std::size_t default_expansion_search() { return 64; }
|
|
|
1202
943
|
|
|
1203
944
|
constexpr std::size_t default_allocator_entry_bytes() { return 64; }
|
|
1204
945
|
|
|
1205
|
-
/**
|
|
1206
|
-
* @brief The "magic" sequence helps infer the type of the file.
|
|
1207
|
-
* USearch indexes start with the "usearch" string.
|
|
1208
|
-
*/
|
|
1209
|
-
constexpr char const* default_magic() { return "usearch"; }
|
|
1210
|
-
|
|
1211
946
|
/**
|
|
1212
947
|
* @brief Configuration settings for the index construction.
|
|
1213
948
|
* Includes the main `::connectivity` parameter (`M` in the paper)
|
|
@@ -1219,14 +954,16 @@ struct index_config_t {
|
|
|
1219
954
|
/// > It is called `M` in the paper.
|
|
1220
955
|
std::size_t connectivity = default_connectivity();
|
|
1221
956
|
|
|
1222
|
-
/// @brief
|
|
1223
|
-
///
|
|
1224
|
-
///
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
957
|
+
/// @brief Number of neighbors per graph node in base level graph.
|
|
958
|
+
/// Defaults to double of the other levels, so 64 in FAISS and 32 in hnswlib.
|
|
959
|
+
/// > It is called `M0` in the paper.
|
|
960
|
+
std::size_t connectivity_base = default_connectivity() * 2;
|
|
961
|
+
|
|
962
|
+
inline index_config_t() = default;
|
|
963
|
+
inline index_config_t(std::size_t c) noexcept
|
|
964
|
+
: connectivity(c ? c : default_connectivity()), connectivity_base(c ? c * 2 : default_connectivity() * 2) {}
|
|
965
|
+
inline index_config_t(std::size_t c, std::size_t cb) noexcept
|
|
966
|
+
: connectivity(c), connectivity_base((std::max)(c, cb)) {}
|
|
1230
967
|
};
|
|
1231
968
|
|
|
1232
969
|
struct index_limits_t {
|
|
@@ -1234,13 +971,13 @@ struct index_limits_t {
|
|
|
1234
971
|
std::size_t threads_add = std::thread::hardware_concurrency();
|
|
1235
972
|
std::size_t threads_search = std::thread::hardware_concurrency();
|
|
1236
973
|
|
|
1237
|
-
index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
|
|
1238
|
-
index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
1239
|
-
std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
1240
|
-
std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
974
|
+
inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
|
|
975
|
+
inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
976
|
+
inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
977
|
+
inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
1241
978
|
};
|
|
1242
979
|
|
|
1243
|
-
struct
|
|
980
|
+
struct index_update_config_t {
|
|
1244
981
|
/// @brief Hyper-parameter controlling the quality of indexing.
|
|
1245
982
|
/// Defaults to 40 in FAISS and 200 in hnswlib.
|
|
1246
983
|
/// > It is called `efConstruction` in the paper.
|
|
@@ -1248,12 +985,9 @@ struct add_config_t {
|
|
|
1248
985
|
|
|
1249
986
|
/// @brief Optional thread identifier for multi-threaded construction.
|
|
1250
987
|
std::size_t thread = 0;
|
|
1251
|
-
|
|
1252
|
-
/// @brief Don't copy the ::vector, if it's persisted elsewhere.
|
|
1253
|
-
bool store_vector = true;
|
|
1254
988
|
};
|
|
1255
989
|
|
|
1256
|
-
struct
|
|
990
|
+
struct index_search_config_t {
|
|
1257
991
|
/// @brief Hyper-parameter controlling the quality of search.
|
|
1258
992
|
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1259
993
|
/// > It is called `ef` in the paper.
|
|
@@ -1266,133 +1000,21 @@ struct search_config_t {
|
|
|
1266
1000
|
bool exact = false;
|
|
1267
1001
|
};
|
|
1268
1002
|
|
|
1269
|
-
struct
|
|
1270
|
-
bool copy_vectors = true;
|
|
1271
|
-
};
|
|
1003
|
+
struct index_copy_config_t {};
|
|
1272
1004
|
|
|
1273
|
-
struct
|
|
1005
|
+
struct index_join_config_t {
|
|
1274
1006
|
/// @brief Controls maximum number of proposals per man during stable marriage.
|
|
1275
1007
|
std::size_t max_proposals = 0;
|
|
1008
|
+
|
|
1276
1009
|
/// @brief Hyper-parameter controlling the quality of search.
|
|
1277
1010
|
/// Defaults to 16 in FAISS and 10 in hnswlib.
|
|
1278
1011
|
/// > It is called `ef` in the paper.
|
|
1279
1012
|
std::size_t expansion = default_expansion_search();
|
|
1013
|
+
|
|
1280
1014
|
/// @brief Brute-forces exhaustive search over all entries in the index.
|
|
1281
1015
|
bool exact = false;
|
|
1282
1016
|
};
|
|
1283
1017
|
|
|
1284
|
-
using file_header_t = byte_t[64];
|
|
1285
|
-
|
|
1286
|
-
/**
|
|
1287
|
-
* @brief Serialized binary representations of the USearch index start with metadata.
|
|
1288
|
-
* Metadata is parsed into a `file_head_t`, containing the USearch package version,
|
|
1289
|
-
* and the properties of the index.
|
|
1290
|
-
*
|
|
1291
|
-
* It uses: 13 bytes for file versioning, 22 bytes for structural information = 35 bytes.
|
|
1292
|
-
* The following 24 bytes contain binary size of the graph, of the vectors, and the checksum,
|
|
1293
|
-
* leaving 5 bytes at the end vacant.
|
|
1294
|
-
*/
|
|
1295
|
-
struct file_head_t {
|
|
1296
|
-
|
|
1297
|
-
// Versioning: 7 + 2 * 3 = 13 bytes
|
|
1298
|
-
using magic_t = char[7];
|
|
1299
|
-
using version_major_t = std::uint16_t;
|
|
1300
|
-
using version_minor_t = std::uint16_t;
|
|
1301
|
-
using version_patch_t = std::uint16_t;
|
|
1302
|
-
|
|
1303
|
-
// Structural: 1 * 6 + 8 * 2 = 22 bytes
|
|
1304
|
-
using connectivity_t = std::uint8_t;
|
|
1305
|
-
using max_level_t = std::uint8_t;
|
|
1306
|
-
using vector_alignment_t = std::uint8_t;
|
|
1307
|
-
using bytes_per_label_t = std::uint8_t;
|
|
1308
|
-
using bytes_per_id_t = std::uint8_t;
|
|
1309
|
-
using size_t = std::uint64_t;
|
|
1310
|
-
using entry_idx_t = std::uint64_t;
|
|
1311
|
-
|
|
1312
|
-
// Versioning:
|
|
1313
|
-
char const* magic;
|
|
1314
|
-
misaligned_ref_gt<version_major_t> version_major;
|
|
1315
|
-
misaligned_ref_gt<version_minor_t> version_minor;
|
|
1316
|
-
misaligned_ref_gt<version_patch_t> version_patch;
|
|
1317
|
-
|
|
1318
|
-
// Structural:
|
|
1319
|
-
misaligned_ref_gt<metric_kind_t> metric;
|
|
1320
|
-
misaligned_ref_gt<connectivity_t> connectivity;
|
|
1321
|
-
misaligned_ref_gt<max_level_t> max_level;
|
|
1322
|
-
misaligned_ref_gt<vector_alignment_t> vector_alignment;
|
|
1323
|
-
misaligned_ref_gt<bytes_per_label_t> bytes_per_label;
|
|
1324
|
-
misaligned_ref_gt<bytes_per_id_t> bytes_per_id;
|
|
1325
|
-
misaligned_ref_gt<scalar_kind_t> scalar_kind;
|
|
1326
|
-
misaligned_ref_gt<size_t> size;
|
|
1327
|
-
misaligned_ref_gt<entry_idx_t> entry_idx;
|
|
1328
|
-
|
|
1329
|
-
// Additional:
|
|
1330
|
-
misaligned_ref_gt<size_t> bytes_for_graphs;
|
|
1331
|
-
misaligned_ref_gt<size_t> bytes_for_vectors;
|
|
1332
|
-
misaligned_ref_gt<size_t> bytes_checksum;
|
|
1333
|
-
|
|
1334
|
-
file_head_t(byte_t* ptr) noexcept
|
|
1335
|
-
: magic((char const*)exchange(ptr, ptr + sizeof(magic_t))),
|
|
1336
|
-
version_major(exchange(ptr, ptr + sizeof(version_major_t))),
|
|
1337
|
-
version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
|
|
1338
|
-
version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
|
|
1339
|
-
metric(exchange(ptr, ptr + sizeof(metric_kind_t))), connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
|
|
1340
|
-
max_level(exchange(ptr, ptr + sizeof(max_level_t))),
|
|
1341
|
-
vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
|
|
1342
|
-
bytes_per_label(exchange(ptr, ptr + sizeof(bytes_per_label_t))),
|
|
1343
|
-
bytes_per_id(exchange(ptr, ptr + sizeof(bytes_per_id_t))),
|
|
1344
|
-
scalar_kind(exchange(ptr, ptr + sizeof(scalar_kind_t))), size(exchange(ptr, ptr + sizeof(size_t))),
|
|
1345
|
-
entry_idx(exchange(ptr, ptr + sizeof(entry_idx_t))), bytes_for_graphs(exchange(ptr, ptr + sizeof(size_t))),
|
|
1346
|
-
bytes_for_vectors(exchange(ptr, ptr + sizeof(size_t))), bytes_checksum(exchange(ptr, ptr + sizeof(size_t))) {}
|
|
1347
|
-
};
|
|
1348
|
-
|
|
1349
|
-
struct file_head_result_t {
|
|
1350
|
-
|
|
1351
|
-
using magic_t = file_head_t::magic_t;
|
|
1352
|
-
using version_major_t = file_head_t::version_major_t;
|
|
1353
|
-
using version_minor_t = file_head_t::version_minor_t;
|
|
1354
|
-
using version_patch_t = file_head_t::version_patch_t;
|
|
1355
|
-
|
|
1356
|
-
using connectivity_t = file_head_t::connectivity_t;
|
|
1357
|
-
using max_level_t = file_head_t::max_level_t;
|
|
1358
|
-
using vector_alignment_t = file_head_t::vector_alignment_t;
|
|
1359
|
-
using bytes_per_label_t = file_head_t::bytes_per_label_t;
|
|
1360
|
-
using bytes_per_id_t = file_head_t::bytes_per_id_t;
|
|
1361
|
-
using size_t = file_head_t::size_t;
|
|
1362
|
-
using entry_idx_t = file_head_t::entry_idx_t;
|
|
1363
|
-
|
|
1364
|
-
// Versioning:
|
|
1365
|
-
version_major_t version_major;
|
|
1366
|
-
version_minor_t version_minor;
|
|
1367
|
-
version_patch_t version_patch;
|
|
1368
|
-
|
|
1369
|
-
// Structural:
|
|
1370
|
-
metric_kind_t metric;
|
|
1371
|
-
connectivity_t connectivity;
|
|
1372
|
-
max_level_t max_level;
|
|
1373
|
-
vector_alignment_t vector_alignment;
|
|
1374
|
-
bytes_per_label_t bytes_per_label;
|
|
1375
|
-
bytes_per_id_t bytes_per_id;
|
|
1376
|
-
scalar_kind_t scalar_kind;
|
|
1377
|
-
size_t size;
|
|
1378
|
-
entry_idx_t entry_idx;
|
|
1379
|
-
|
|
1380
|
-
// Additional:
|
|
1381
|
-
size_t bytes_for_graphs;
|
|
1382
|
-
size_t bytes_for_vectors;
|
|
1383
|
-
size_t bytes_checksum;
|
|
1384
|
-
|
|
1385
|
-
error_t error;
|
|
1386
|
-
|
|
1387
|
-
explicit operator bool() const noexcept { return !error; }
|
|
1388
|
-
file_head_result_t failed(error_t message) noexcept {
|
|
1389
|
-
error = std::move(message);
|
|
1390
|
-
return std::move(*this);
|
|
1391
|
-
}
|
|
1392
|
-
};
|
|
1393
|
-
|
|
1394
|
-
static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 bytes");
|
|
1395
|
-
|
|
1396
1018
|
/// @brief C++17 and newer version deprecate the `std::result_of`
|
|
1397
1019
|
template <typename metric_at, typename... args_at>
|
|
1398
1020
|
using return_type_gt =
|
|
@@ -1402,32 +1024,65 @@ using return_type_gt =
|
|
|
1402
1024
|
typename std::result_of<metric_at(args_at...)>::type;
|
|
1403
1025
|
#endif
|
|
1404
1026
|
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
};
|
|
1414
|
-
#else
|
|
1415
|
-
struct viewed_file_t {
|
|
1416
|
-
int file_descriptor{};
|
|
1417
|
-
void* ptr{};
|
|
1418
|
-
size_t length{};
|
|
1419
|
-
explicit operator bool() const noexcept { return file_descriptor != 0; }
|
|
1027
|
+
/**
|
|
1028
|
+
* @brief An example of what a USearch-compatible ad-hoc filter would look like.
|
|
1029
|
+
*
|
|
1030
|
+
* A similar function object can be passed to search queries to further filter entries
|
|
1031
|
+
* on their auxiliary properties, such as some categorical labels stored in an external DBMS.
|
|
1032
|
+
*/
|
|
1033
|
+
struct dummy_predicate_t {
|
|
1034
|
+
template <typename member_at> constexpr bool operator()(member_at&&) const noexcept { return true; }
|
|
1420
1035
|
};
|
|
1421
|
-
#endif
|
|
1422
1036
|
|
|
1423
|
-
|
|
1424
|
-
|
|
1037
|
+
/**
|
|
1038
|
+
* @brief An example of what a USearch-compatible ad-hoc operation on in-flight entries.
|
|
1039
|
+
*
|
|
1040
|
+
* This kind of callbacks is used when the engine is being updated and you want to patch
|
|
1041
|
+
* the entries, while their are still under locks - limiting concurrent access and providing
|
|
1042
|
+
* consistency.
|
|
1043
|
+
*/
|
|
1044
|
+
struct dummy_callback_t {
|
|
1045
|
+
template <typename member_at> void operator()(member_at&&) const noexcept {}
|
|
1425
1046
|
};
|
|
1426
1047
|
|
|
1048
|
+
/**
|
|
1049
|
+
* @brief An example of what a USearch-compatible progress-bar should look like.
|
|
1050
|
+
*
|
|
1051
|
+
* This is particularly helpful when handling long-running tasks, like serialization,
|
|
1052
|
+
* saving, and loading from disk, or index-level joins.
|
|
1053
|
+
*/
|
|
1427
1054
|
struct dummy_progress_t {
|
|
1428
1055
|
inline void operator()(std::size_t /*progress*/, std::size_t /*total*/) const noexcept {}
|
|
1429
1056
|
};
|
|
1430
1057
|
|
|
1058
|
+
/**
|
|
1059
|
+
* @brief An example of what a USearch-compatible values prefetching mechanism should look like.
|
|
1060
|
+
*
|
|
1061
|
+
* USearch is designed to handle very large datasets, that may not fir into RAM. Fetching from
|
|
1062
|
+
* external memory is very expensive, so we've added a pre-fetching mechanism, that accepts
|
|
1063
|
+
* multiple objects at once, to cache in RAM ahead of the computation.
|
|
1064
|
+
* The received iterators support both `get_slot` and `get_key` operations.
|
|
1065
|
+
* An example usage may look like this:
|
|
1066
|
+
*
|
|
1067
|
+
* template <typename member_citerator_like_at>
|
|
1068
|
+
* inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {
|
|
1069
|
+
* for (; begin != end; ++begin)
|
|
1070
|
+
* io_uring_prefetch(offset_in_file(get_key(begin)));
|
|
1071
|
+
* }
|
|
1072
|
+
*/
|
|
1073
|
+
struct dummy_prefetch_t {
|
|
1074
|
+
template <typename member_citerator_like_at>
|
|
1075
|
+
inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {}
|
|
1076
|
+
};
|
|
1077
|
+
|
|
1078
|
+
/**
|
|
1079
|
+
* @brief An example of what a USearch-compatible executor (thread-pool) should look like.
|
|
1080
|
+
*
|
|
1081
|
+
* It's expected to have `execute_bulk(callback)` API to schedule one task per thread;
|
|
1082
|
+
* an identical `execute_bulk(count, callback)` overload that also accepts the number
|
|
1083
|
+
* of tasks, and somehow schedules them between threads; as well as `size()` to
|
|
1084
|
+
* determine the number of available threads.
|
|
1085
|
+
*/
|
|
1431
1086
|
struct dummy_executor_t {
|
|
1432
1087
|
dummy_executor_t() noexcept {}
|
|
1433
1088
|
std::size_t size() const noexcept { return 1; }
|
|
@@ -1444,13 +1099,34 @@ struct dummy_executor_t {
|
|
|
1444
1099
|
}
|
|
1445
1100
|
};
|
|
1446
1101
|
|
|
1102
|
+
/**
|
|
1103
|
+
* @brief An example of what a USearch-compatible label-to-label mapping should look like.
|
|
1104
|
+
*
|
|
1105
|
+
* This is particularly helpful for "Semantic Joins", where we map entries of one collection
|
|
1106
|
+
* to entries of another. In assymetric setups, where A -> B is needed, but B -> A is not,
|
|
1107
|
+
* this can be passed to minimize memory usage.
|
|
1108
|
+
*/
|
|
1447
1109
|
struct dummy_label_to_label_mapping_t {
|
|
1448
1110
|
struct member_ref_t {
|
|
1449
|
-
template <typename
|
|
1111
|
+
template <typename key_at> member_ref_t& operator=(key_at&&) noexcept { return *this; }
|
|
1450
1112
|
};
|
|
1451
|
-
template <typename
|
|
1113
|
+
template <typename key_at> member_ref_t operator[](key_at&&) const noexcept { return {}; }
|
|
1452
1114
|
};
|
|
1453
1115
|
|
|
1116
|
+
/**
|
|
1117
|
+
* @brief Checks if the provided object has a dummy type, emulating an interface,
|
|
1118
|
+
* but performaing no real computation.
|
|
1119
|
+
*/
|
|
1120
|
+
template <typename object_at> static constexpr bool is_dummy() {
|
|
1121
|
+
using object_t = typename std::remove_all_extents<object_at>::type;
|
|
1122
|
+
return std::is_same<object_t, dummy_predicate_t>::value || //
|
|
1123
|
+
std::is_same<object_t, dummy_callback_t>::value || //
|
|
1124
|
+
std::is_same<object_t, dummy_progress_t>::value || //
|
|
1125
|
+
std::is_same<object_t, dummy_prefetch_t>::value || //
|
|
1126
|
+
std::is_same<object_t, dummy_executor_t>::value || //
|
|
1127
|
+
std::is_same<object_t, dummy_label_to_label_mapping_t>::value;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1454
1130
|
template <typename, typename at> struct has_reset_gt {
|
|
1455
1131
|
static_assert(std::integral_constant<at, false>::value, "Second template parameter needs to be of function type.");
|
|
1456
1132
|
};
|
|
@@ -1474,116 +1150,388 @@ struct has_reset_gt<check_at, return_at(args_at...)> {
|
|
|
1474
1150
|
*/
|
|
1475
1151
|
template <typename at> constexpr bool has_reset() { return has_reset_gt<at, void()>::value; }
|
|
1476
1152
|
|
|
1153
|
+
struct serialization_result_t {
|
|
1154
|
+
error_t error;
|
|
1155
|
+
|
|
1156
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1157
|
+
serialization_result_t failed(error_t message) noexcept {
|
|
1158
|
+
error = std::move(message);
|
|
1159
|
+
return std::move(*this);
|
|
1160
|
+
}
|
|
1161
|
+
};
|
|
1162
|
+
|
|
1477
1163
|
/**
|
|
1478
|
-
* @brief
|
|
1479
|
-
* Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
|
|
1480
|
-
*
|
|
1481
|
-
* @section Features
|
|
1482
|
-
* - Thread-safe for concurrent construction.
|
|
1483
|
-
* - Doesn't allocate new threads, and reuses the ones its called from.
|
|
1484
|
-
* - Search for vectors of different dimensionality, if ::`metric_at` supports that.
|
|
1485
|
-
*
|
|
1486
|
-
* @tparam metric_at
|
|
1487
|
-
* A function responsible for computing the distance (dis-similarity) between two vectors.
|
|
1488
|
-
* Can have following signatures:
|
|
1489
|
-
* - `distance_t (*) (span_gt<scalar_t const>, span_gt<scalar_t const>)`
|
|
1490
|
-
* - `distance_t (*) (scalar_t const *, scalar_t const *)`
|
|
1491
|
-
*
|
|
1492
|
-
* @tparam label_at
|
|
1493
|
-
* The type of unique labels to assign to vectors.
|
|
1494
|
-
*
|
|
1495
|
-
* @tparam id_at
|
|
1496
|
-
* The smallest unsigned integer type to address indexed elements.
|
|
1497
|
-
* Can be a built-in `uint32_t`, `uint64_t`, or our custom `uint40_t`.
|
|
1498
|
-
*
|
|
1499
|
-
* @tparam dynamic_allocator_at
|
|
1500
|
-
* Dynamic memory allocator for temporary buffers, visits indicators, and
|
|
1501
|
-
* priority queues, needed during construction and traversals of graphs.
|
|
1502
|
-
* The allocated buffers may be uninitialized.
|
|
1503
|
-
*
|
|
1504
|
-
* @tparam tape_allocator_at
|
|
1505
|
-
* Potentially different memory allocator for primary allocations of nodes and vectors.
|
|
1506
|
-
* It would never `deallocate` separate entries, and would only free all the space at once.
|
|
1507
|
-
* The allocated buffers may be uninitialized.
|
|
1508
|
-
*
|
|
1509
|
-
* @section Usage
|
|
1510
|
-
*
|
|
1511
|
-
* @subsection Exceptions
|
|
1512
|
-
*
|
|
1513
|
-
* None of the methods throw exceptions in the "Release" compilation mode.
|
|
1514
|
-
* It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
|
|
1515
|
-
* safe to copy.
|
|
1516
|
-
*
|
|
1517
|
-
* @section Implementation Details
|
|
1518
|
-
*
|
|
1519
|
-
* Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
|
|
1520
|
-
* Every added vector forms a node in one or more levels of the graph.
|
|
1521
|
-
* Every node is present in the base level. Every following level contains a smaller
|
|
1522
|
-
* fraction of nodes. During search, the operation starts with the smaller levels
|
|
1523
|
-
* and zooms-in on every following iteration of larger graph traversals.
|
|
1524
|
-
*
|
|
1525
|
-
* Just one memory allocation is performed regardless of the number of levels.
|
|
1526
|
-
* The adjacency lists across all levels are concatenated into that single buffer.
|
|
1527
|
-
* That buffer starts with a "head", that stores the metadata, such as the
|
|
1528
|
-
* tallest "level" of the graph that it belongs to, the external "label", and the
|
|
1529
|
-
* number of "dimensions" in the vector.
|
|
1530
|
-
*
|
|
1531
|
-
* @subsection Colocated Nodes
|
|
1532
|
-
*
|
|
1533
|
-
* Every node contains some graph-related data, as well as the original vector.
|
|
1534
|
-
* Assuming the vectors can get large, we allow separating the index from the vectors.
|
|
1535
|
-
* They can either be stored in a single continuous block of memory, or in two separate
|
|
1536
|
-
* locations. Vector storage can also be detached from `index_gt` and managed by the user.
|
|
1537
|
-
* To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
|
|
1538
|
-
* overloads.
|
|
1539
|
-
*
|
|
1540
|
-
* @subsection Missing Functionality
|
|
1541
|
-
*
|
|
1542
|
-
* To simplify the implementation, the `index_gt` lacks endpoints to remove existing
|
|
1543
|
-
* vectors. That, however, is solved by `index_punned_dense_gt`, which also adds automatic casting.
|
|
1164
|
+
* @brief Smart-pointer wrapping the LibC @b `FILE` for binary file @b outputs.
|
|
1544
1165
|
*
|
|
1166
|
+
* This class raises no exceptions and corresponds errors through `serialization_result_t`.
|
|
1167
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1545
1168
|
*/
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
typename dynamic_allocator_at = std::allocator<char>, //
|
|
1550
|
-
typename tape_allocator_at = dynamic_allocator_at> //
|
|
1551
|
-
class index_gt {
|
|
1552
|
-
public:
|
|
1553
|
-
using metric_t = metric_at;
|
|
1554
|
-
using label_t = label_at;
|
|
1555
|
-
using id_t = id_at;
|
|
1556
|
-
using dynamic_allocator_t = dynamic_allocator_at;
|
|
1557
|
-
using tape_allocator_t = tape_allocator_at;
|
|
1558
|
-
static_assert(sizeof(label_t) >= sizeof(id_t), "Having tiny labels doesn't make sense.");
|
|
1169
|
+
class output_file_t {
|
|
1170
|
+
char const* path_ = nullptr;
|
|
1171
|
+
std::FILE* file_ = nullptr;
|
|
1559
1172
|
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1173
|
+
public:
|
|
1174
|
+
output_file_t(char const* path) noexcept : path_(path) {}
|
|
1175
|
+
~output_file_t() noexcept { close(); }
|
|
1176
|
+
output_file_t(output_file_t&& other) noexcept
|
|
1177
|
+
: path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
|
|
1178
|
+
output_file_t& operator=(output_file_t&& other) noexcept {
|
|
1179
|
+
std::swap(path_, other.path_);
|
|
1180
|
+
std::swap(file_, other.file_);
|
|
1181
|
+
return *this;
|
|
1182
|
+
}
|
|
1183
|
+
serialization_result_t open_if_not() noexcept {
|
|
1184
|
+
serialization_result_t result;
|
|
1185
|
+
if (!file_)
|
|
1186
|
+
file_ = std::fopen(path_, "wb");
|
|
1187
|
+
if (!file_)
|
|
1188
|
+
return result.failed(std::strerror(errno));
|
|
1189
|
+
return result;
|
|
1190
|
+
}
|
|
1191
|
+
serialization_result_t write(void* begin, std::size_t length) noexcept {
|
|
1192
|
+
serialization_result_t result;
|
|
1193
|
+
std::size_t written = std::fwrite(begin, length, 1, file_);
|
|
1194
|
+
if (!written)
|
|
1195
|
+
result.failed(std::strerror(errno));
|
|
1196
|
+
return result;
|
|
1197
|
+
}
|
|
1198
|
+
void close() noexcept {
|
|
1199
|
+
if (file_)
|
|
1200
|
+
std::fclose(exchange(file_, nullptr));
|
|
1201
|
+
}
|
|
1202
|
+
};
|
|
1563
1203
|
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1204
|
+
/**
|
|
1205
|
+
* @brief Smart-pointer wrapping the LibC @b `FILE` for binary files @b inputs.
|
|
1206
|
+
*
|
|
1207
|
+
* This class raises no exceptions and corresponds errors through `serialization_result_t`.
|
|
1208
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1209
|
+
*/
|
|
1210
|
+
class input_file_t {
|
|
1211
|
+
char const* path_ = nullptr;
|
|
1212
|
+
std::FILE* file_ = nullptr;
|
|
1569
1213
|
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1214
|
+
public:
|
|
1215
|
+
input_file_t(char const* path) noexcept : path_(path) {}
|
|
1216
|
+
~input_file_t() noexcept { close(); }
|
|
1217
|
+
input_file_t(input_file_t&& other) noexcept
|
|
1218
|
+
: path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
|
|
1219
|
+
input_file_t& operator=(input_file_t&& other) noexcept {
|
|
1220
|
+
std::swap(path_, other.path_);
|
|
1221
|
+
std::swap(file_, other.file_);
|
|
1222
|
+
return *this;
|
|
1223
|
+
}
|
|
1574
1224
|
|
|
1575
|
-
|
|
1576
|
-
|
|
1225
|
+
serialization_result_t open_if_not() noexcept {
|
|
1226
|
+
serialization_result_t result;
|
|
1227
|
+
if (!file_)
|
|
1228
|
+
file_ = std::fopen(path_, "rb");
|
|
1229
|
+
if (!file_)
|
|
1230
|
+
return result.failed(std::strerror(errno));
|
|
1231
|
+
return result;
|
|
1232
|
+
}
|
|
1233
|
+
serialization_result_t read(void* begin, std::size_t length) noexcept {
|
|
1234
|
+
serialization_result_t result;
|
|
1235
|
+
std::size_t read = std::fread(begin, length, 1, file_);
|
|
1236
|
+
if (!read)
|
|
1237
|
+
result.failed(std::feof(file_) ? "End of file reached!" : std::strerror(errno));
|
|
1238
|
+
return result;
|
|
1239
|
+
}
|
|
1240
|
+
void close() noexcept {
|
|
1241
|
+
if (file_)
|
|
1242
|
+
std::fclose(exchange(file_, nullptr));
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
explicit operator bool() const noexcept { return file_; }
|
|
1246
|
+
bool seek_to(std::size_t progress) noexcept { return std::fseek(file_, progress, SEEK_SET) == 0; }
|
|
1247
|
+
bool seek_to_end() noexcept { return std::fseek(file_, 0L, SEEK_END) == 0; }
|
|
1248
|
+
bool infer_progress(std::size_t& progress) noexcept {
|
|
1249
|
+
long int result = std::ftell(file_);
|
|
1250
|
+
if (result == -1L)
|
|
1251
|
+
return false;
|
|
1252
|
+
progress = static_cast<std::size_t>(result);
|
|
1253
|
+
return true;
|
|
1254
|
+
}
|
|
1255
|
+
};
|
|
1256
|
+
|
|
1257
|
+
/**
|
|
1258
|
+
* @brief Represents a memory-mapped file or a pre-allocated anonymous memory region.
|
|
1259
|
+
*
|
|
1260
|
+
* This class provides a convenient way to memory-map a file and access its contents as a block of
|
|
1261
|
+
* memory. The class handles platform-specific memory-mapping operations on Windows, Linux, and MacOS.
|
|
1262
|
+
* The class automatically closes the file when the object is destroyed.
|
|
1263
|
+
*/
|
|
1264
|
+
class memory_mapped_file_t {
|
|
1265
|
+
char const* path_{}; /**< The path to the file to be memory-mapped. */
|
|
1266
|
+
void* ptr_{}; /**< A pointer to the memory-mapping. */
|
|
1267
|
+
size_t length_{}; /**< The length of the memory-mapped file in bytes. */
|
|
1268
|
+
|
|
1269
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1270
|
+
HANDLE file_handle_{}; /**< The file handle on Windows. */
|
|
1271
|
+
HANDLE mapping_handle_{}; /**< The mapping handle on Windows. */
|
|
1272
|
+
#else
|
|
1273
|
+
int file_descriptor_{}; /**< The file descriptor on Linux and MacOS. */
|
|
1274
|
+
#endif
|
|
1275
|
+
|
|
1276
|
+
public:
|
|
1277
|
+
explicit operator bool() const noexcept { return ptr_ != nullptr; }
|
|
1278
|
+
byte_t const* data() const noexcept { return reinterpret_cast<byte_t const*>(ptr_); }
|
|
1279
|
+
std::size_t size() const noexcept { return static_cast<std::size_t>(length_); }
|
|
1280
|
+
|
|
1281
|
+
memory_mapped_file_t() noexcept {}
|
|
1282
|
+
memory_mapped_file_t(char const* path) noexcept : path_(path) {}
|
|
1283
|
+
~memory_mapped_file_t() noexcept { close(); }
|
|
1284
|
+
memory_mapped_file_t(memory_mapped_file_t&& other) noexcept
|
|
1285
|
+
: path_(exchange(other.path_, nullptr)), ptr_(exchange(other.ptr_, nullptr)),
|
|
1286
|
+
length_(exchange(other.length_, 0)),
|
|
1287
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1288
|
+
file_handle_(exchange(other.file_handle_, nullptr)), mapping_handle_(exchange(other.mapping_handle_, nullptr))
|
|
1289
|
+
#else
|
|
1290
|
+
file_descriptor_(exchange(other.file_descriptor_, 0))
|
|
1291
|
+
#endif
|
|
1292
|
+
{
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
memory_mapped_file_t(byte_t* data, std::size_t length) noexcept : ptr_(data), length_(length) {}
|
|
1296
|
+
|
|
1297
|
+
memory_mapped_file_t& operator=(memory_mapped_file_t&& other) noexcept {
|
|
1298
|
+
std::swap(path_, other.path_);
|
|
1299
|
+
std::swap(ptr_, other.ptr_);
|
|
1300
|
+
std::swap(length_, other.length_);
|
|
1301
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1302
|
+
std::swap(file_handle_, other.file_handle_);
|
|
1303
|
+
std::swap(mapping_handle_, other.mapping_handle_);
|
|
1304
|
+
#else
|
|
1305
|
+
std::swap(file_descriptor_, other.file_descriptor_);
|
|
1306
|
+
#endif
|
|
1307
|
+
return *this;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
serialization_result_t open_if_not() noexcept {
|
|
1311
|
+
serialization_result_t result;
|
|
1312
|
+
if (!path_ || ptr_)
|
|
1313
|
+
return result;
|
|
1314
|
+
|
|
1315
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1316
|
+
|
|
1317
|
+
HANDLE file_handle =
|
|
1318
|
+
CreateFile(path_, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
|
|
1319
|
+
if (file_handle == INVALID_HANDLE_VALUE)
|
|
1320
|
+
return result.failed("Opening file failed!");
|
|
1321
|
+
|
|
1322
|
+
std::size_t file_length = GetFileSize(file_handle, 0);
|
|
1323
|
+
HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
|
|
1324
|
+
if (mapping_handle == 0) {
|
|
1325
|
+
CloseHandle(file_handle);
|
|
1326
|
+
return result.failed("Mapping file failed!");
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
|
|
1330
|
+
if (file == 0) {
|
|
1331
|
+
CloseHandle(mapping_handle);
|
|
1332
|
+
CloseHandle(file_handle);
|
|
1333
|
+
return result.failed("View the map failed!");
|
|
1334
|
+
}
|
|
1335
|
+
file_handle_ = file_handle;
|
|
1336
|
+
mapping_handle_ = mapping_handle;
|
|
1337
|
+
ptr_ = file;
|
|
1338
|
+
length_ = file_length;
|
|
1339
|
+
#else
|
|
1340
|
+
|
|
1341
|
+
#if defined(USEARCH_DEFINED_LINUX)
|
|
1342
|
+
int descriptor = open(path_, O_RDONLY | O_NOATIME);
|
|
1343
|
+
#else
|
|
1344
|
+
int descriptor = open(path_, O_RDONLY);
|
|
1345
|
+
#endif
|
|
1346
|
+
|
|
1347
|
+
// Estimate the file size
|
|
1348
|
+
struct stat file_stat;
|
|
1349
|
+
int fstat_status = fstat(descriptor, &file_stat);
|
|
1350
|
+
if (fstat_status < 0) {
|
|
1351
|
+
::close(descriptor);
|
|
1352
|
+
return result.failed(std::strerror(errno));
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
// Map the entire file
|
|
1356
|
+
byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_SHARED, descriptor, 0);
|
|
1357
|
+
if (file == MAP_FAILED) {
|
|
1358
|
+
::close(descriptor);
|
|
1359
|
+
return result.failed(std::strerror(errno));
|
|
1360
|
+
}
|
|
1361
|
+
file_descriptor_ = descriptor;
|
|
1362
|
+
ptr_ = file;
|
|
1363
|
+
length_ = file_stat.st_size;
|
|
1364
|
+
#endif // Platform specific code
|
|
1365
|
+
return result;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
void close() noexcept {
|
|
1369
|
+
if (!path_) {
|
|
1370
|
+
ptr_ = nullptr;
|
|
1371
|
+
length_ = 0;
|
|
1372
|
+
return;
|
|
1373
|
+
}
|
|
1374
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
1375
|
+
UnmapViewOfFile(ptr_);
|
|
1376
|
+
CloseHandle(mapping_handle_);
|
|
1377
|
+
CloseHandle(file_handle_);
|
|
1378
|
+
mapping_handle_ = nullptr;
|
|
1379
|
+
file_handle_ = nullptr;
|
|
1380
|
+
#else
|
|
1381
|
+
munmap(ptr_, length_);
|
|
1382
|
+
::close(file_descriptor_);
|
|
1383
|
+
file_descriptor_ = 0;
|
|
1384
|
+
#endif
|
|
1385
|
+
ptr_ = nullptr;
|
|
1386
|
+
length_ = 0;
|
|
1387
|
+
}
|
|
1388
|
+
};
|
|
1389
|
+
|
|
1390
|
+
struct index_serialized_header_t {
|
|
1391
|
+
std::uint64_t size = 0;
|
|
1392
|
+
std::uint64_t connectivity = 0;
|
|
1393
|
+
std::uint64_t connectivity_base = 0;
|
|
1394
|
+
std::uint64_t max_level = 0;
|
|
1395
|
+
std::uint64_t entry_slot = 0;
|
|
1396
|
+
};
|
|
1397
|
+
|
|
1398
|
+
using default_key_t = std::uint64_t;
|
|
1399
|
+
using default_slot_t = std::uint32_t;
|
|
1400
|
+
using default_distance_t = float;
|
|
1401
|
+
|
|
1402
|
+
template <typename key_at = default_key_t> struct member_gt {
|
|
1403
|
+
key_at key;
|
|
1404
|
+
std::size_t slot;
|
|
1405
|
+
};
|
|
1406
|
+
|
|
1407
|
+
template <typename key_at> inline std::size_t get_slot(member_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1408
|
+
template <typename key_at> inline key_at get_key(member_gt<key_at> const& m) noexcept { return m.key; }
|
|
1409
|
+
|
|
1410
|
+
template <typename key_at = default_key_t> struct member_cref_gt {
|
|
1411
|
+
misaligned_ref_gt<key_at const> key;
|
|
1412
|
+
std::size_t slot;
|
|
1413
|
+
};
|
|
1414
|
+
|
|
1415
|
+
template <typename key_at> inline std::size_t get_slot(member_cref_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1416
|
+
template <typename key_at> inline key_at get_key(member_cref_gt<key_at> const& m) noexcept { return m.key; }
|
|
1417
|
+
|
|
1418
|
+
template <typename key_at = default_key_t> struct member_ref_gt {
|
|
1419
|
+
misaligned_ref_gt<key_at> key;
|
|
1420
|
+
std::size_t slot;
|
|
1421
|
+
|
|
1422
|
+
inline operator member_cref_gt<key_at>() const noexcept { return {key.ptr(), slot}; }
|
|
1423
|
+
};
|
|
1424
|
+
|
|
1425
|
+
template <typename key_at> inline std::size_t get_slot(member_ref_gt<key_at> const& m) noexcept { return m.slot; }
|
|
1426
|
+
template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m) noexcept { return m.key; }
|
|
1427
|
+
|
|
1428
|
+
/**
|
|
1429
|
+
* @brief Approximate Nearest Neighbors Search @b index-structure using the
|
|
1430
|
+
* Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
|
|
1431
|
+
* If classical containers store @b Key->Value mappings, this one can
|
|
1432
|
+
* be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
|
|
1433
|
+
*
|
|
1434
|
+
* Unlike most implementations, this one is generic anc can be used for any search,
|
|
1435
|
+
* not just within equi-dimensional vectors. Examples range from texts to similar Chess
|
|
1436
|
+
* positions.
|
|
1437
|
+
*
|
|
1438
|
+
* @tparam key_at
|
|
1439
|
+
* The type of primary objects stored in the index.
|
|
1440
|
+
* The values, to which those map, are not managed by the same index structure.
|
|
1441
|
+
*
|
|
1442
|
+
* @tparam compressed_slot_at
|
|
1443
|
+
* The smallest unsigned integer type to address indexed elements.
|
|
1444
|
+
* It is used internally to maximize space-efficiency and is generally
|
|
1445
|
+
* up-casted to @b `std::size_t` in public interfaces.
|
|
1446
|
+
* Can be a built-in @b `uint32_t`, `uint64_t`, or our custom @b `uint40_t`.
|
|
1447
|
+
* Which makes the most sense for 4B+ entry indexes.
|
|
1448
|
+
*
|
|
1449
|
+
* @tparam dynamic_allocator_at
|
|
1450
|
+
* Dynamic memory allocator for temporary buffers, visits indicators, and
|
|
1451
|
+
* priority queues, needed during construction and traversals of graphs.
|
|
1452
|
+
* The allocated buffers may be uninitialized.
|
|
1453
|
+
*
|
|
1454
|
+
* @tparam tape_allocator_at
|
|
1455
|
+
* Potentially different memory allocator for primary allocations of nodes and vectors.
|
|
1456
|
+
* It would never `deallocate` separate entries, and would only free all the space at once.
|
|
1457
|
+
* The allocated buffers may be uninitialized.
|
|
1458
|
+
*
|
|
1459
|
+
* @section Features
|
|
1460
|
+
*
|
|
1461
|
+
* - Thread-safe for concurrent construction, search, and updates.
|
|
1462
|
+
* - Doesn't allocate new threads, and reuses the ones its called from.
|
|
1463
|
+
* - Allows storing value externally, managing just the similarity index.
|
|
1464
|
+
* - Joins.
|
|
1465
|
+
|
|
1466
|
+
* @section Usage
|
|
1467
|
+
*
|
|
1468
|
+
* @subsection Exceptions
|
|
1469
|
+
*
|
|
1470
|
+
* None of the methods throw exceptions in the "Release" compilation mode.
|
|
1471
|
+
* It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
|
|
1472
|
+
* safe to copy.
|
|
1473
|
+
*
|
|
1474
|
+
* @subsection Serialization
|
|
1475
|
+
*
|
|
1476
|
+
* When serialized, doesn't include any additional metadata.
|
|
1477
|
+
* It is just the multi-level proximity-graph. You may want to store metadata about
|
|
1478
|
+
* the used metric and key types somewhere else.
|
|
1479
|
+
*
|
|
1480
|
+
* @section Implementation Details
|
|
1481
|
+
*
|
|
1482
|
+
* Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
|
|
1483
|
+
* Every added vector forms a node in one or more levels of the graph.
|
|
1484
|
+
* Every node is present in the base level. Every following level contains a smaller
|
|
1485
|
+
* fraction of nodes. During search, the operation starts with the smaller levels
|
|
1486
|
+
* and zooms-in on every following iteration of larger graph traversals.
|
|
1487
|
+
*
|
|
1488
|
+
* Just one memory allocation is performed regardless of the number of levels.
|
|
1489
|
+
* The adjacency lists across all levels are concatenated into that single buffer.
|
|
1490
|
+
* That buffer starts with a "head", that stores the metadata, such as the
|
|
1491
|
+
* tallest "level" of the graph that it belongs to, the external "key", and the
|
|
1492
|
+
* number of "dimensions" in the vector.
|
|
1493
|
+
*
|
|
1494
|
+
* @section Metrics, Predicates and Callbacks
|
|
1495
|
+
*
|
|
1496
|
+
*
|
|
1497
|
+
* @section Smart References and Iterators
|
|
1498
|
+
*
|
|
1499
|
+
* - `member_citerator_t` and `member_iterator_t` have only slots, no indirections.
|
|
1500
|
+
*
|
|
1501
|
+
* - `member_cref_t` and `member_ref_t` contains the `slot` and a reference
|
|
1502
|
+
* to the key. So it passes through 1 level of visited_members in `nodes_`.
|
|
1503
|
+
* Retrieving the key via `get_key` will cause fetching yet another cache line.
|
|
1504
|
+
*
|
|
1505
|
+
* - `member_gt` contains an already prefetched copy of the key.
|
|
1506
|
+
*
|
|
1507
|
+
*/
|
|
1508
|
+
template <typename distance_at = default_distance_t, //
|
|
1509
|
+
typename key_at = default_key_t, //
|
|
1510
|
+
typename compressed_slot_at = default_slot_t, //
|
|
1511
|
+
typename dynamic_allocator_at = std::allocator<byte_t>, //
|
|
1512
|
+
typename tape_allocator_at = dynamic_allocator_at> //
|
|
1513
|
+
class index_gt {
|
|
1514
|
+
public:
|
|
1515
|
+
using distance_t = distance_at;
|
|
1516
|
+
using key_t = key_at;
|
|
1517
|
+
using compressed_slot_t = compressed_slot_at;
|
|
1518
|
+
using dynamic_allocator_t = dynamic_allocator_at;
|
|
1519
|
+
using tape_allocator_t = tape_allocator_at;
|
|
1520
|
+
static_assert(sizeof(key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
|
|
1521
|
+
|
|
1522
|
+
using member_cref_t = member_cref_gt<key_t>;
|
|
1523
|
+
using member_ref_t = member_ref_gt<key_t>;
|
|
1577
1524
|
|
|
1578
1525
|
template <typename ref_at, typename index_at> class member_iterator_gt {
|
|
1579
1526
|
using ref_t = ref_at;
|
|
1580
1527
|
using index_t = index_at;
|
|
1581
|
-
index_t* index_{};
|
|
1582
|
-
std::size_t offset_{};
|
|
1583
1528
|
|
|
1584
1529
|
friend class index_gt;
|
|
1585
1530
|
member_iterator_gt() noexcept {}
|
|
1586
|
-
member_iterator_gt(index_t* index, std::size_t
|
|
1531
|
+
member_iterator_gt(index_t* index, std::size_t slot) noexcept : index_(index), slot_(slot) {}
|
|
1532
|
+
|
|
1533
|
+
index_t* index_{};
|
|
1534
|
+
std::size_t slot_{};
|
|
1587
1535
|
|
|
1588
1536
|
public:
|
|
1589
1537
|
using iterator_category = std::random_access_iterator_tag;
|
|
@@ -1592,45 +1540,32 @@ class index_gt {
|
|
|
1592
1540
|
using pointer = void;
|
|
1593
1541
|
using reference = ref_t;
|
|
1594
1542
|
|
|
1595
|
-
reference operator*() const noexcept {
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
}
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
member_iterator_gt operator
|
|
1602
|
-
member_iterator_gt operator
|
|
1603
|
-
member_iterator_gt operator
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
}
|
|
1609
|
-
member_iterator_gt& operator
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
}
|
|
1613
|
-
|
|
1614
|
-
offset_ += d;
|
|
1615
|
-
return *this;
|
|
1616
|
-
}
|
|
1617
|
-
member_iterator_gt& operator-=(difference_type d) noexcept {
|
|
1618
|
-
offset_ -= d;
|
|
1619
|
-
return *this;
|
|
1620
|
-
}
|
|
1621
|
-
bool operator==(member_iterator_gt const& other) const noexcept {
|
|
1622
|
-
return index_ == other.index_ && offset_ == other.offset_;
|
|
1623
|
-
}
|
|
1624
|
-
bool operator!=(member_iterator_gt const& other) const noexcept {
|
|
1625
|
-
return index_ != other.index_ || offset_ != other.offset_;
|
|
1626
|
-
}
|
|
1543
|
+
reference operator*() const noexcept { return {index_->node_at_(slot_).key(), slot_}; }
|
|
1544
|
+
key_t key() const noexcept { return index_->node_at_(slot_).key(); }
|
|
1545
|
+
|
|
1546
|
+
friend inline std::size_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
|
|
1547
|
+
friend inline key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
|
|
1548
|
+
|
|
1549
|
+
member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
|
|
1550
|
+
member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
|
|
1551
|
+
member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
|
|
1552
|
+
member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
|
|
1553
|
+
|
|
1554
|
+
// clang-format off
|
|
1555
|
+
member_iterator_gt& operator++() noexcept { slot_ += 1; return *this; }
|
|
1556
|
+
member_iterator_gt& operator--() noexcept { slot_ -= 1; return *this; }
|
|
1557
|
+
member_iterator_gt& operator+=(difference_type d) noexcept { slot_ += d; return *this; }
|
|
1558
|
+
member_iterator_gt& operator-=(difference_type d) noexcept { slot_ -= d; return *this; }
|
|
1559
|
+
bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
|
|
1560
|
+
bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
|
|
1561
|
+
// clang-format on
|
|
1627
1562
|
};
|
|
1628
1563
|
|
|
1629
1564
|
using member_iterator_t = member_iterator_gt<member_ref_t, index_gt>;
|
|
1630
1565
|
using member_citerator_t = member_iterator_gt<member_cref_t, index_gt const>;
|
|
1631
1566
|
|
|
1632
1567
|
// STL compatibility:
|
|
1633
|
-
using value_type =
|
|
1568
|
+
using value_type = key_t;
|
|
1634
1569
|
using allocator_type = dynamic_allocator_t;
|
|
1635
1570
|
using size_type = std::size_t;
|
|
1636
1571
|
using difference_type = std::ptrdiff_t;
|
|
@@ -1643,108 +1578,113 @@ class index_gt {
|
|
|
1643
1578
|
using reverse_iterator = std::reverse_iterator<member_iterator_t>;
|
|
1644
1579
|
using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
|
|
1645
1580
|
|
|
1646
|
-
|
|
1647
|
-
using neighbors_count_t = std::uint32_t;
|
|
1648
|
-
using dim_t = std::uint32_t;
|
|
1649
|
-
using level_t = std::int32_t;
|
|
1650
|
-
|
|
1651
|
-
using allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
|
|
1581
|
+
using dynamic_allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
|
|
1652
1582
|
using byte_t = typename dynamic_allocator_t::value_type;
|
|
1653
|
-
static_assert(
|
|
1654
|
-
|
|
1583
|
+
static_assert( //
|
|
1584
|
+
sizeof(byte_t) == 1, //
|
|
1585
|
+
"Primary allocator must allocate separate addressable bytes");
|
|
1655
1586
|
|
|
1656
1587
|
using tape_allocator_traits_t = std::allocator_traits<tape_allocator_t>;
|
|
1657
|
-
static_assert(
|
|
1658
|
-
|
|
1588
|
+
static_assert( //
|
|
1589
|
+
sizeof(typename tape_allocator_traits_t::value_type) == 1, //
|
|
1590
|
+
"Tape allocator must allocate separate addressable bytes");
|
|
1659
1591
|
|
|
1592
|
+
private:
|
|
1660
1593
|
/**
|
|
1661
|
-
* @brief
|
|
1662
|
-
*
|
|
1594
|
+
* @brief Integer for the number of node neighbors at a specific level of the
|
|
1595
|
+
* multi-level graph. It's selected to be `std::uint32_t` to improve the
|
|
1596
|
+
* alignment in most common cases.
|
|
1663
1597
|
*/
|
|
1664
|
-
|
|
1598
|
+
using neighbors_count_t = std::uint32_t;
|
|
1599
|
+
using level_t = std::int16_t;
|
|
1665
1600
|
|
|
1666
1601
|
/**
|
|
1667
1602
|
* @brief How many bytes of memory are needed to form the "head" of the node.
|
|
1668
1603
|
*/
|
|
1669
|
-
static constexpr std::size_t node_head_bytes_() { return sizeof(
|
|
1604
|
+
static constexpr std::size_t node_head_bytes_() { return sizeof(key_t) + sizeof(level_t); }
|
|
1670
1605
|
|
|
1671
1606
|
using visits_bitset_t = visits_bitset_gt<dynamic_allocator_t>;
|
|
1672
1607
|
|
|
1673
1608
|
struct precomputed_constants_t {
|
|
1674
1609
|
double inverse_log_connectivity{};
|
|
1675
|
-
std::size_t connectivity_max_base{};
|
|
1676
1610
|
std::size_t neighbors_bytes{};
|
|
1677
1611
|
std::size_t neighbors_base_bytes{};
|
|
1678
1612
|
};
|
|
1613
|
+
/// @brief A space-efficient internal data-structure used in graph traversal queues.
|
|
1679
1614
|
struct candidate_t {
|
|
1680
1615
|
distance_t distance;
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
struct compare_by_distance_t {
|
|
1684
|
-
inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
|
|
1616
|
+
compressed_slot_t slot;
|
|
1617
|
+
inline bool operator<(candidate_t other) const noexcept { return distance < other.distance; }
|
|
1685
1618
|
};
|
|
1686
1619
|
|
|
1687
1620
|
using candidates_view_t = span_gt<candidate_t const>;
|
|
1688
|
-
using candidates_allocator_t = typename
|
|
1689
|
-
using top_candidates_t = sorted_buffer_gt<candidate_t,
|
|
1690
|
-
using next_candidates_t = max_heap_gt<candidate_t,
|
|
1621
|
+
using candidates_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<candidate_t>;
|
|
1622
|
+
using top_candidates_t = sorted_buffer_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
|
|
1623
|
+
using next_candidates_t = max_heap_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
|
|
1691
1624
|
|
|
1692
1625
|
/**
|
|
1693
|
-
* @brief A loosely-structured handle for every node. One such node is created for every
|
|
1626
|
+
* @brief A loosely-structured handle for every node. One such node is created for every member.
|
|
1694
1627
|
* To minimize memory usage and maximize the number of entries per cache-line, it only
|
|
1695
|
-
* stores to pointers.
|
|
1696
|
-
*
|
|
1628
|
+
* stores to pointers. The internal tape starts with a `key_t` @b key, then
|
|
1629
|
+
* a `level_t` for the number of graph @b levels in which this member appears,
|
|
1630
|
+
* then the { `neighbors_count_t`, `compressed_slot_t`, `compressed_slot_t` ... } sequences
|
|
1631
|
+
* for @b each-level.
|
|
1697
1632
|
*/
|
|
1698
1633
|
class node_t {
|
|
1699
1634
|
byte_t* tape_{};
|
|
1700
|
-
scalar_t* vector_{};
|
|
1701
1635
|
|
|
1702
1636
|
public:
|
|
1703
|
-
explicit node_t(byte_t* tape
|
|
1637
|
+
explicit node_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1704
1638
|
byte_t* tape() const noexcept { return tape_; }
|
|
1705
|
-
scalar_t* vector() const noexcept { return vector_; }
|
|
1706
1639
|
byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
|
|
1640
|
+
explicit operator bool() const noexcept { return tape_; }
|
|
1707
1641
|
|
|
1708
1642
|
node_t() = default;
|
|
1709
1643
|
node_t(node_t const&) = default;
|
|
1710
1644
|
node_t& operator=(node_t const&) = default;
|
|
1711
1645
|
|
|
1712
|
-
misaligned_ref_gt<
|
|
1713
|
-
misaligned_ref_gt<
|
|
1714
|
-
misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(
|
|
1646
|
+
misaligned_ref_gt<key_t const> ckey() const noexcept { return {tape_}; }
|
|
1647
|
+
misaligned_ref_gt<key_t> key() const noexcept { return {tape_}; }
|
|
1648
|
+
misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(key_t)}; }
|
|
1715
1649
|
|
|
1716
|
-
void
|
|
1717
|
-
void
|
|
1718
|
-
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
|
|
1719
|
-
|
|
1720
|
-
operator vector_view_t() const noexcept { return {vector(), dim()}; }
|
|
1721
|
-
vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
|
|
1722
|
-
explicit operator bool() const noexcept { return tape_; }
|
|
1650
|
+
void key(key_t v) noexcept { return misaligned_store<key_t>(tape_, v); }
|
|
1651
|
+
void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(key_t), v); }
|
|
1723
1652
|
};
|
|
1724
1653
|
|
|
1654
|
+
static_assert(std::is_trivially_copy_constructible<node_t>::value, "Nodes must be light!");
|
|
1655
|
+
static_assert(std::is_trivially_destructible<node_t>::value, "Nodes must be light!");
|
|
1656
|
+
|
|
1725
1657
|
/**
|
|
1726
1658
|
* @brief A slice of the node's tape, containing a the list of neighbors
|
|
1727
1659
|
* for a node in a single graph level. It's pre-allocated to fit
|
|
1728
|
-
* as many neighbors
|
|
1660
|
+
* as many neighbors "slots", as may be needed at the target level,
|
|
1729
1661
|
* and starts with a single integer `neighbors_count_t` counter.
|
|
1730
1662
|
*/
|
|
1731
1663
|
class neighbors_ref_t {
|
|
1732
1664
|
byte_t* tape_;
|
|
1733
1665
|
|
|
1734
|
-
static constexpr std::size_t shift(std::size_t i = 0) {
|
|
1666
|
+
static constexpr std::size_t shift(std::size_t i = 0) {
|
|
1667
|
+
return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
|
|
1668
|
+
}
|
|
1735
1669
|
|
|
1736
1670
|
public:
|
|
1737
1671
|
neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
|
|
1738
|
-
misaligned_ptr_gt<
|
|
1739
|
-
misaligned_ptr_gt<
|
|
1740
|
-
misaligned_ptr_gt<
|
|
1741
|
-
misaligned_ptr_gt<
|
|
1742
|
-
|
|
1672
|
+
misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
|
|
1673
|
+
misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
|
|
1674
|
+
misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
|
|
1675
|
+
misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
|
|
1676
|
+
compressed_slot_t operator[](std::size_t i) const noexcept {
|
|
1677
|
+
return misaligned_load<compressed_slot_t>(tape_ + shift(i));
|
|
1678
|
+
}
|
|
1743
1679
|
std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
|
|
1744
|
-
void clear() noexcept {
|
|
1745
|
-
|
|
1680
|
+
void clear() noexcept {
|
|
1681
|
+
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1682
|
+
std::memset(tape_, 0, shift(n));
|
|
1683
|
+
// misaligned_store<neighbors_count_t>(tape_, 0);
|
|
1684
|
+
}
|
|
1685
|
+
void push_back(compressed_slot_t slot) noexcept {
|
|
1746
1686
|
neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
|
|
1747
|
-
misaligned_store<
|
|
1687
|
+
misaligned_store<compressed_slot_t>(tape_ + shift(n), slot);
|
|
1748
1688
|
misaligned_store<neighbors_count_t>(tape_, n + 1);
|
|
1749
1689
|
}
|
|
1750
1690
|
};
|
|
@@ -1759,44 +1699,72 @@ class index_gt {
|
|
|
1759
1699
|
next_candidates_t next_candidates{};
|
|
1760
1700
|
visits_bitset_t visits{};
|
|
1761
1701
|
std::default_random_engine level_generator{};
|
|
1762
|
-
metric_t metric{};
|
|
1763
1702
|
std::size_t iteration_cycles{};
|
|
1764
|
-
std::size_t
|
|
1703
|
+
std::size_t computed_distances_count{};
|
|
1704
|
+
|
|
1705
|
+
template <typename value_at, typename metric_at, typename entry_at> //
|
|
1706
|
+
inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1707
|
+
static_assert( //
|
|
1708
|
+
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1709
|
+
"Unexpected type");
|
|
1710
|
+
|
|
1711
|
+
computed_distances_count++;
|
|
1712
|
+
return metric(first, second);
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
template <typename metric_at, typename entry_at> //
|
|
1716
|
+
inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
|
|
1717
|
+
static_assert( //
|
|
1718
|
+
std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
|
|
1719
|
+
"Unexpected type");
|
|
1765
1720
|
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
return metric(a, b);
|
|
1721
|
+
computed_distances_count++;
|
|
1722
|
+
return metric(first, second);
|
|
1769
1723
|
}
|
|
1770
1724
|
};
|
|
1771
1725
|
|
|
1772
1726
|
index_config_t config_{};
|
|
1773
1727
|
index_limits_t limits_{};
|
|
1774
|
-
|
|
1728
|
+
|
|
1775
1729
|
mutable dynamic_allocator_t dynamic_allocator_{};
|
|
1776
1730
|
tape_allocator_t tape_allocator_{};
|
|
1731
|
+
|
|
1777
1732
|
precomputed_constants_t pre_{};
|
|
1778
|
-
|
|
1733
|
+
memory_mapped_file_t viewed_file_{};
|
|
1779
1734
|
|
|
1780
|
-
|
|
1781
|
-
usearch_align_m mutable std::atomic<std::size_t>
|
|
1735
|
+
/// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
|
|
1736
|
+
usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
|
|
1782
1737
|
|
|
1783
|
-
/// @brief
|
|
1738
|
+
/// @brief Number of "slots" already storing non-null nodes.
|
|
1739
|
+
usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
|
|
1740
|
+
|
|
1741
|
+
/// @brief Controls access to `max_level_` and `entry_slot_`.
|
|
1784
1742
|
/// If any thread is updating those values, no other threads can `add()` or `search()`.
|
|
1785
1743
|
std::mutex global_mutex_{};
|
|
1744
|
+
|
|
1745
|
+
/// @brief The level of the top-most graph in the index. Grows as the logarithm of size, starts from zero.
|
|
1786
1746
|
level_t max_level_{};
|
|
1787
|
-
id_t entry_id_{};
|
|
1788
1747
|
|
|
1789
|
-
|
|
1790
|
-
|
|
1748
|
+
/// @brief The slot in which the only node of the top-level graph is stored.
|
|
1749
|
+
std::size_t entry_slot_{};
|
|
1750
|
+
|
|
1751
|
+
using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
|
|
1752
|
+
|
|
1753
|
+
/// @brief C-style array of `node_t` smart-pointers.
|
|
1754
|
+
buffer_gt<node_t, nodes_allocator_t> nodes_{};
|
|
1755
|
+
|
|
1756
|
+
/// @brief Mutex, that limits concurrent access to `nodes_`.
|
|
1791
1757
|
mutable visits_bitset_t nodes_mutexes_{};
|
|
1792
1758
|
|
|
1793
|
-
using contexts_allocator_t = typename
|
|
1794
|
-
|
|
1759
|
+
using contexts_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<context_t>;
|
|
1760
|
+
|
|
1761
|
+
/// @brief Array of thread-specific buffers for temporary data.
|
|
1762
|
+
mutable buffer_gt<context_t, contexts_allocator_t> contexts_{};
|
|
1795
1763
|
|
|
1796
1764
|
public:
|
|
1797
1765
|
std::size_t connectivity() const noexcept { return config_.connectivity; }
|
|
1798
|
-
std::size_t capacity() const noexcept { return
|
|
1799
|
-
std::size_t size() const noexcept { return
|
|
1766
|
+
std::size_t capacity() const noexcept { return nodes_capacity_; }
|
|
1767
|
+
std::size_t size() const noexcept { return nodes_count_; }
|
|
1800
1768
|
std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
|
|
1801
1769
|
index_config_t const& config() const noexcept { return config_; }
|
|
1802
1770
|
index_limits_t const& limits() const noexcept { return limits_; }
|
|
@@ -1806,16 +1774,17 @@ class index_gt {
|
|
|
1806
1774
|
* @section Exceptions
|
|
1807
1775
|
* Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
|
|
1808
1776
|
*/
|
|
1809
|
-
explicit index_gt(
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1777
|
+
explicit index_gt( //
|
|
1778
|
+
index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
|
|
1779
|
+
tape_allocator_t tape_allocator = {}) noexcept
|
|
1780
|
+
: config_(config), limits_(0, 0), dynamic_allocator_(std::move(dynamic_allocator)),
|
|
1781
|
+
tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), nodes_count_(0u), max_level_(-1),
|
|
1782
|
+
entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
|
|
1814
1783
|
|
|
1815
1784
|
/**
|
|
1816
1785
|
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
1817
1786
|
*/
|
|
1818
|
-
index_gt fork() noexcept { return index_gt{config_,
|
|
1787
|
+
index_gt fork() noexcept { return index_gt{config_, dynamic_allocator_, tape_allocator_}; }
|
|
1819
1788
|
|
|
1820
1789
|
~index_gt() noexcept { reset(); }
|
|
1821
1790
|
|
|
@@ -1837,21 +1806,21 @@ class index_gt {
|
|
|
1837
1806
|
}
|
|
1838
1807
|
};
|
|
1839
1808
|
|
|
1840
|
-
copy_result_t copy(
|
|
1809
|
+
copy_result_t copy(index_copy_config_t /*config*/ = {}) const noexcept {
|
|
1841
1810
|
copy_result_t result;
|
|
1842
1811
|
index_gt& other = result.index;
|
|
1843
|
-
other = index_gt(config_,
|
|
1812
|
+
other = index_gt(config_, dynamic_allocator_, tape_allocator_);
|
|
1844
1813
|
if (!other.reserve(limits_))
|
|
1845
1814
|
return result.failed("Failed to reserve the contexts");
|
|
1846
1815
|
|
|
1847
1816
|
// Now all is left - is to allocate new `node_t` instances and populate
|
|
1848
1817
|
// the `other.nodes_` array into it.
|
|
1849
|
-
for (std::size_t i = 0; i !=
|
|
1850
|
-
other.nodes_[i] = other.node_make_copy_(
|
|
1818
|
+
for (std::size_t i = 0; i != nodes_count_; ++i)
|
|
1819
|
+
other.nodes_[i] = other.node_make_copy_(node_bytes_(nodes_[i]));
|
|
1851
1820
|
|
|
1852
|
-
other.
|
|
1821
|
+
other.nodes_count_ = nodes_count_.load();
|
|
1853
1822
|
other.max_level_ = max_level_;
|
|
1854
|
-
other.
|
|
1823
|
+
other.entry_slot_ = entry_slot_;
|
|
1855
1824
|
return result;
|
|
1856
1825
|
}
|
|
1857
1826
|
|
|
@@ -1862,13 +1831,10 @@ class index_gt {
|
|
|
1862
1831
|
member_iterator_t begin() noexcept { return {this, 0}; }
|
|
1863
1832
|
member_iterator_t end() noexcept { return {this, size()}; }
|
|
1864
1833
|
|
|
1865
|
-
member_ref_t at(std::size_t
|
|
1866
|
-
|
|
1867
|
-
}
|
|
1868
|
-
|
|
1869
|
-
member_cref_t at(std::size_t i) const noexcept {
|
|
1870
|
-
return {nodes_[i].label(), nodes_[i].vector_view(), static_cast<id_t>(i)};
|
|
1871
|
-
}
|
|
1834
|
+
member_ref_t at(std::size_t slot) noexcept { return {nodes_[slot].key(), slot}; }
|
|
1835
|
+
member_cref_t at(std::size_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
|
|
1836
|
+
member_iterator_t iterator_at(std::size_t slot) noexcept { return {this, slot}; }
|
|
1837
|
+
member_citerator_t citerator_at(std::size_t slot) const noexcept { return {this, slot}; }
|
|
1872
1838
|
|
|
1873
1839
|
dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
|
|
1874
1840
|
tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
|
|
@@ -1876,40 +1842,40 @@ class index_gt {
|
|
|
1876
1842
|
#pragma region Adjusting Configuration
|
|
1877
1843
|
|
|
1878
1844
|
/**
|
|
1879
|
-
* @brief
|
|
1880
|
-
*
|
|
1881
|
-
*
|
|
1845
|
+
* @brief Erases all the vectors from the index.
|
|
1846
|
+
*
|
|
1847
|
+
* Will change `size()` to zero, but will keep the same `capacity()`.
|
|
1848
|
+
* Will keep the number of available threads/contexts the same as it was.
|
|
1882
1849
|
*/
|
|
1883
1850
|
void clear() noexcept {
|
|
1884
1851
|
if (!has_reset<tape_allocator_t>()) {
|
|
1885
|
-
std::size_t n =
|
|
1852
|
+
std::size_t n = nodes_count_;
|
|
1886
1853
|
for (std::size_t i = 0; i != n; ++i)
|
|
1887
1854
|
node_free_(i);
|
|
1888
1855
|
} else
|
|
1889
1856
|
tape_allocator_.deallocate(nullptr, 0);
|
|
1890
|
-
|
|
1857
|
+
nodes_count_ = 0;
|
|
1891
1858
|
max_level_ = -1;
|
|
1892
|
-
|
|
1859
|
+
entry_slot_ = 0u;
|
|
1893
1860
|
}
|
|
1894
1861
|
|
|
1895
1862
|
/**
|
|
1896
|
-
* @brief
|
|
1897
|
-
*
|
|
1898
|
-
*
|
|
1863
|
+
* @brief Erases all members from index, closing files, and returning RAM to OS.
|
|
1864
|
+
*
|
|
1865
|
+
* Will change both `size()` and `capacity()` to zero.
|
|
1866
|
+
* Will deallocate all threads/contexts.
|
|
1867
|
+
* If the index is memory-mapped - releases the mapping and the descriptor.
|
|
1899
1868
|
*/
|
|
1900
1869
|
void reset() noexcept {
|
|
1901
1870
|
clear();
|
|
1902
1871
|
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
for (std::size_t i = 0; i != limits_.threads(); ++i)
|
|
1907
|
-
contexts_[i].~context_t();
|
|
1908
|
-
contexts_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
|
|
1909
|
-
}
|
|
1872
|
+
nodes_ = {};
|
|
1873
|
+
contexts_ = {};
|
|
1874
|
+
nodes_mutexes_ = {};
|
|
1910
1875
|
limits_ = index_limits_t{0, 0};
|
|
1911
|
-
|
|
1912
|
-
|
|
1876
|
+
nodes_capacity_ = 0;
|
|
1877
|
+
viewed_file_ = memory_mapped_file_t{};
|
|
1878
|
+
tape_allocator_ = {};
|
|
1913
1879
|
}
|
|
1914
1880
|
|
|
1915
1881
|
/**
|
|
@@ -1918,24 +1884,23 @@ class index_gt {
|
|
|
1918
1884
|
void swap(index_gt& other) noexcept {
|
|
1919
1885
|
std::swap(config_, other.config_);
|
|
1920
1886
|
std::swap(limits_, other.limits_);
|
|
1921
|
-
std::swap(metric_, other.metric_);
|
|
1922
1887
|
std::swap(dynamic_allocator_, other.dynamic_allocator_);
|
|
1923
1888
|
std::swap(tape_allocator_, other.tape_allocator_);
|
|
1924
1889
|
std::swap(pre_, other.pre_);
|
|
1925
1890
|
std::swap(viewed_file_, other.viewed_file_);
|
|
1926
1891
|
std::swap(max_level_, other.max_level_);
|
|
1927
|
-
std::swap(
|
|
1892
|
+
std::swap(entry_slot_, other.entry_slot_);
|
|
1928
1893
|
std::swap(nodes_, other.nodes_);
|
|
1929
1894
|
std::swap(nodes_mutexes_, other.nodes_mutexes_);
|
|
1930
1895
|
std::swap(contexts_, other.contexts_);
|
|
1931
1896
|
|
|
1932
1897
|
// Non-atomic parts.
|
|
1933
|
-
std::size_t capacity_copy =
|
|
1934
|
-
std::size_t
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
other.
|
|
1938
|
-
other.
|
|
1898
|
+
std::size_t capacity_copy = nodes_capacity_;
|
|
1899
|
+
std::size_t count_copy = nodes_count_;
|
|
1900
|
+
nodes_capacity_ = other.nodes_capacity_.load();
|
|
1901
|
+
nodes_count_ = other.nodes_count_.load();
|
|
1902
|
+
other.nodes_capacity_ = capacity_copy;
|
|
1903
|
+
other.nodes_count_ = count_copy;
|
|
1939
1904
|
}
|
|
1940
1905
|
|
|
1941
1906
|
/**
|
|
@@ -1949,72 +1914,30 @@ class index_gt {
|
|
|
1949
1914
|
&& limits.members <= limits_.members)
|
|
1950
1915
|
return true;
|
|
1951
1916
|
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
node_t* new_nodes = node_allocator.allocate(limits.members);
|
|
1957
|
-
if (!new_nodes)
|
|
1917
|
+
visits_bitset_t new_mutexes(limits.members);
|
|
1918
|
+
buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
|
|
1919
|
+
buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
|
|
1920
|
+
if (!new_nodes || !new_contexts || !new_mutexes)
|
|
1958
1921
|
return false;
|
|
1959
1922
|
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
if (!new_contexts) {
|
|
1964
|
-
node_allocator.deallocate(new_nodes, limits.members);
|
|
1965
|
-
return false;
|
|
1966
|
-
}
|
|
1967
|
-
for (std::size_t i = 0; i != limits_threads; ++i) {
|
|
1968
|
-
context_t& context = new_contexts[i];
|
|
1969
|
-
new (&context) context_t();
|
|
1970
|
-
context.metric = metric_;
|
|
1971
|
-
if (!context.visits.resize(limits.members)) {
|
|
1972
|
-
for (std::size_t j = 0; j != i; ++j)
|
|
1973
|
-
context.visits.reset();
|
|
1974
|
-
node_allocator.deallocate(new_nodes, limits.members);
|
|
1975
|
-
context_allocator.deallocate(new_contexts, limits_threads);
|
|
1923
|
+
for (context_t& context : new_contexts) {
|
|
1924
|
+
context.visits = visits_bitset_t(limits.members);
|
|
1925
|
+
if (!context.visits)
|
|
1976
1926
|
return false;
|
|
1977
|
-
}
|
|
1978
|
-
}
|
|
1979
|
-
|
|
1980
|
-
// We have passed all the require memory allocations.
|
|
1981
|
-
// The remaining code can't fail. Let's just reuse some of our existing buffers.
|
|
1982
|
-
for (std::size_t i = 0; i != limits_.threads(); ++i) {
|
|
1983
|
-
context_t& old_context = contexts_[i];
|
|
1984
|
-
context_t& context = new_contexts[i];
|
|
1985
|
-
std::swap(old_context.top_candidates, context.top_candidates);
|
|
1986
|
-
std::swap(old_context.next_candidates, context.next_candidates);
|
|
1987
|
-
std::swap(old_context.iteration_cycles, context.iteration_cycles);
|
|
1988
|
-
std::swap(old_context.measurements_count, context.measurements_count);
|
|
1989
|
-
old_context.visits.reset();
|
|
1990
1927
|
}
|
|
1991
1928
|
|
|
1992
1929
|
// Move the nodes info, and deallocate previous buffers.
|
|
1993
1930
|
if (nodes_)
|
|
1994
|
-
std::memcpy(new_nodes, nodes_, sizeof(node_t) * size())
|
|
1995
|
-
if (contexts_)
|
|
1996
|
-
context_allocator.deallocate(contexts_, limits_.threads());
|
|
1931
|
+
std::memcpy(new_nodes.data(), nodes_.data(), sizeof(node_t) * size());
|
|
1997
1932
|
|
|
1998
1933
|
limits_ = limits;
|
|
1999
|
-
|
|
2000
|
-
nodes_ = new_nodes;
|
|
2001
|
-
contexts_ = new_contexts;
|
|
1934
|
+
nodes_capacity_ = limits.members;
|
|
1935
|
+
nodes_ = std::move(new_nodes);
|
|
1936
|
+
contexts_ = std::move(new_contexts);
|
|
1937
|
+
nodes_mutexes_ = std::move(new_mutexes);
|
|
2002
1938
|
return true;
|
|
2003
1939
|
}
|
|
2004
1940
|
|
|
2005
|
-
/**
|
|
2006
|
-
* @brief Optimizes configuration options to fit the maximum number
|
|
2007
|
-
* of neighbors in CPU-cache-aligned buffers.
|
|
2008
|
-
*/
|
|
2009
|
-
static index_config_t optimize(index_config_t config) noexcept {
|
|
2010
|
-
precomputed_constants_t pre = precompute_(config);
|
|
2011
|
-
std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
|
|
2012
|
-
std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
|
|
2013
|
-
std::size_t added_connections = (rounded_size - bytes_per_node_base) / sizeof(id_t);
|
|
2014
|
-
config.connectivity = config.connectivity + added_connections / base_level_multiple_();
|
|
2015
|
-
return config;
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
1941
|
#pragma endregion
|
|
2019
1942
|
|
|
2020
1943
|
#pragma region Construction and Search
|
|
@@ -2022,9 +1945,9 @@ class index_gt {
|
|
|
2022
1945
|
struct add_result_t {
|
|
2023
1946
|
error_t error{};
|
|
2024
1947
|
std::size_t new_size{};
|
|
2025
|
-
std::size_t
|
|
2026
|
-
std::size_t
|
|
2027
|
-
|
|
1948
|
+
std::size_t visited_members{};
|
|
1949
|
+
std::size_t computed_distances{};
|
|
1950
|
+
std::size_t slot{};
|
|
2028
1951
|
|
|
2029
1952
|
explicit operator bool() const noexcept { return !error; }
|
|
2030
1953
|
add_result_t failed(error_t message) noexcept {
|
|
@@ -2033,29 +1956,31 @@ class index_gt {
|
|
|
2033
1956
|
}
|
|
2034
1957
|
};
|
|
2035
1958
|
|
|
1959
|
+
/// @brief Describes a matched search result, augmenting `member_cref_t`
|
|
1960
|
+
/// contents with `distance` to the query object.
|
|
2036
1961
|
struct match_t {
|
|
2037
1962
|
member_cref_t member;
|
|
2038
1963
|
distance_t distance;
|
|
2039
1964
|
};
|
|
2040
1965
|
|
|
2041
1966
|
class search_result_t {
|
|
2042
|
-
|
|
2043
|
-
top_candidates_t
|
|
1967
|
+
node_t const* nodes_{};
|
|
1968
|
+
top_candidates_t const* top_{};
|
|
2044
1969
|
|
|
2045
1970
|
friend class index_gt;
|
|
2046
|
-
inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
|
|
1971
|
+
inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
|
|
1972
|
+
: nodes_(index.nodes_), top_(&top) {}
|
|
2047
1973
|
|
|
2048
1974
|
public:
|
|
1975
|
+
/** @brief Number of search results found. */
|
|
2049
1976
|
std::size_t count{};
|
|
2050
|
-
|
|
2051
|
-
std::size_t
|
|
1977
|
+
/** @brief Number of graph nodes traversed. */
|
|
1978
|
+
std::size_t visited_members{};
|
|
1979
|
+
/** @brief Number of times the distances were computed. */
|
|
1980
|
+
std::size_t computed_distances{};
|
|
2052
1981
|
error_t error{};
|
|
2053
1982
|
|
|
2054
|
-
|
|
2055
|
-
// given that search_result_t can represent a failed result (which implies no valid top_candidates_t &top_),
|
|
2056
|
-
// the top_ probably needs to be a pointer.
|
|
2057
|
-
inline search_result_t(index_gt const& index) noexcept
|
|
2058
|
-
: index_(index), top_(index.contexts_[0].top_candidates) {}
|
|
1983
|
+
inline search_result_t() noexcept {}
|
|
2059
1984
|
inline search_result_t(search_result_t&&) = default;
|
|
2060
1985
|
inline search_result_t& operator=(search_result_t&&) = default;
|
|
2061
1986
|
|
|
@@ -2071,46 +1996,88 @@ class index_gt {
|
|
|
2071
1996
|
inline match_t operator[](std::size_t i) const noexcept { return at(i); }
|
|
2072
1997
|
inline match_t front() const noexcept { return at(0); }
|
|
2073
1998
|
inline match_t back() const noexcept { return at(count - 1); }
|
|
2074
|
-
inline bool contains(
|
|
1999
|
+
inline bool contains(key_t key) const noexcept {
|
|
2075
2000
|
for (std::size_t i = 0; i != count; ++i)
|
|
2076
|
-
if (at(i).member.
|
|
2001
|
+
if (at(i).member.key == key)
|
|
2077
2002
|
return true;
|
|
2078
2003
|
return false;
|
|
2079
2004
|
}
|
|
2080
2005
|
inline match_t at(std::size_t i) const noexcept {
|
|
2081
|
-
candidate_t const* top_ordered = top_
|
|
2006
|
+
candidate_t const* top_ordered = top_->data();
|
|
2082
2007
|
candidate_t candidate = top_ordered[i];
|
|
2083
|
-
node_t node =
|
|
2084
|
-
return {member_cref_t{node.
|
|
2008
|
+
node_t node = nodes_[candidate.slot];
|
|
2009
|
+
return {member_cref_t{node.ckey(), candidate.slot}, candidate.distance};
|
|
2085
2010
|
}
|
|
2086
|
-
inline std::size_t
|
|
2011
|
+
inline std::size_t merge_into( //
|
|
2012
|
+
key_t* keys, distance_t* distances, //
|
|
2013
|
+
std::size_t old_count, std::size_t max_count) const noexcept {
|
|
2014
|
+
|
|
2015
|
+
std::size_t merged_count = old_count;
|
|
2087
2016
|
for (std::size_t i = 0; i != count; ++i) {
|
|
2088
2017
|
match_t result = operator[](i);
|
|
2089
|
-
|
|
2018
|
+
distance_t* merged_end = distances + merged_count;
|
|
2019
|
+
std::size_t offset = std::lower_bound(distances, merged_end, result.distance) - distances;
|
|
2020
|
+
if (offset == max_count)
|
|
2021
|
+
continue;
|
|
2022
|
+
|
|
2023
|
+
std::size_t count_worse = merged_count - offset - (max_count == merged_count);
|
|
2024
|
+
std::memmove(keys + offset + 1, keys + offset, count_worse * sizeof(key_t));
|
|
2025
|
+
std::memmove(distances + offset + 1, distances + offset, count_worse * sizeof(distance_t));
|
|
2026
|
+
keys[offset] = result.member.key;
|
|
2027
|
+
distances[offset] = result.distance;
|
|
2028
|
+
merged_count += merged_count != max_count;
|
|
2029
|
+
}
|
|
2030
|
+
return merged_count;
|
|
2031
|
+
}
|
|
2032
|
+
inline std::size_t dump_to(key_t* keys, distance_t* distances) const noexcept {
|
|
2033
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
2034
|
+
match_t result = operator[](i);
|
|
2035
|
+
keys[i] = result.member.key;
|
|
2090
2036
|
distances[i] = result.distance;
|
|
2091
2037
|
}
|
|
2092
2038
|
return count;
|
|
2093
2039
|
}
|
|
2094
|
-
inline std::size_t dump_to(
|
|
2040
|
+
inline std::size_t dump_to(key_t* keys) const noexcept {
|
|
2095
2041
|
for (std::size_t i = 0; i != count; ++i) {
|
|
2096
2042
|
match_t result = operator[](i);
|
|
2097
|
-
|
|
2043
|
+
keys[i] = result.member.key;
|
|
2098
2044
|
}
|
|
2099
2045
|
return count;
|
|
2100
2046
|
}
|
|
2101
2047
|
};
|
|
2102
2048
|
|
|
2103
2049
|
/**
|
|
2104
|
-
* @brief
|
|
2050
|
+
* @brief Inserts a new entry into the index. Thread-safe. Supports @b heterogeneous lookups.
|
|
2051
|
+
* Expects needed capacity to be reserved ahead of time: `size() < capacity()`.
|
|
2052
|
+
*
|
|
2053
|
+
* @tparam metric_at
|
|
2054
|
+
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2055
|
+
* It should be callable into distinctly different scenarios:
|
|
2056
|
+
* - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
|
|
2057
|
+
* - `distance_t operator() (entry_at, entry_at)` - between existing entries.
|
|
2058
|
+
* Where any possible `entry_at` has both two interfaces: `std::size_t slot()`, `key_t key()`.
|
|
2105
2059
|
*
|
|
2106
|
-
* @param[in]
|
|
2107
|
-
* @param[in]
|
|
2060
|
+
* @param[in] key External identifier/name/descriptor for the new entry.
|
|
2061
|
+
* @param[in] value Content that will be compared against other entries to index.
|
|
2062
|
+
* @param[in] metric Callable object measuring distance between ::value and present objects.
|
|
2108
2063
|
* @param[in] config Configuration options for this specific operation.
|
|
2064
|
+
* @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
|
|
2109
2065
|
*/
|
|
2110
|
-
|
|
2066
|
+
template < //
|
|
2067
|
+
typename value_at, //
|
|
2068
|
+
typename metric_at, //
|
|
2069
|
+
typename callback_at = dummy_callback_t, //
|
|
2070
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2071
|
+
>
|
|
2072
|
+
add_result_t add( //
|
|
2073
|
+
key_t key, value_at&& value, metric_at&& metric, //
|
|
2074
|
+
index_update_config_t config = {}, //
|
|
2075
|
+
callback_at&& callback = callback_at{}, //
|
|
2076
|
+
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2111
2077
|
|
|
2112
|
-
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
2113
2078
|
add_result_t result;
|
|
2079
|
+
if (is_immutable())
|
|
2080
|
+
return result.failed("Can't add to an immutable index");
|
|
2114
2081
|
|
|
2115
2082
|
// Make sure we have enough local memory to perform this request
|
|
2116
2083
|
context_t& context = contexts_[config.thread];
|
|
@@ -2121,7 +2088,8 @@ class index_gt {
|
|
|
2121
2088
|
|
|
2122
2089
|
// The top list needs one more slot than the connectivity of the base level
|
|
2123
2090
|
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
2124
|
-
std::size_t
|
|
2091
|
+
std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
|
|
2092
|
+
std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
|
|
2125
2093
|
if (!top.reserve(top_limit))
|
|
2126
2094
|
return result.failed("Out of memory!");
|
|
2127
2095
|
if (!next.reserve(config.expansion))
|
|
@@ -2129,60 +2097,98 @@ class index_gt {
|
|
|
2129
2097
|
|
|
2130
2098
|
// Determining how much memory to allocate for the node depends on the target level
|
|
2131
2099
|
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
2132
|
-
level_t max_level_copy = max_level_;
|
|
2133
|
-
|
|
2100
|
+
level_t max_level_copy = max_level_; // Copy under lock
|
|
2101
|
+
std::size_t entry_idx_copy = entry_slot_; // Copy under lock
|
|
2134
2102
|
level_t target_level = choose_random_level_(context.level_generator);
|
|
2135
|
-
|
|
2136
|
-
|
|
2103
|
+
|
|
2104
|
+
// Make sure we are not overflowing
|
|
2105
|
+
std::size_t capacity = nodes_capacity_.load();
|
|
2106
|
+
std::size_t new_slot = nodes_count_.fetch_add(1);
|
|
2107
|
+
if (new_slot >= capacity) {
|
|
2108
|
+
nodes_count_.fetch_sub(1);
|
|
2109
|
+
return result.failed("Reserve capacity ahead of insertions!");
|
|
2110
|
+
}
|
|
2137
2111
|
|
|
2138
2112
|
// Allocate the neighbors
|
|
2139
|
-
node_t node = node_make_(
|
|
2140
|
-
if (!node)
|
|
2113
|
+
node_t node = node_make_(key, target_level);
|
|
2114
|
+
if (!node) {
|
|
2115
|
+
nodes_count_.fetch_sub(1);
|
|
2141
2116
|
return result.failed("Out of memory!");
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2117
|
+
}
|
|
2118
|
+
if (target_level <= max_level_copy)
|
|
2119
|
+
new_level_lock.unlock();
|
|
2120
|
+
|
|
2121
|
+
nodes_[new_slot] = node;
|
|
2122
|
+
result.new_size = new_slot + 1;
|
|
2123
|
+
result.slot = new_slot;
|
|
2124
|
+
callback(at(new_slot));
|
|
2125
|
+
node_lock_t new_lock = node_lock_(new_slot);
|
|
2148
2126
|
|
|
2149
2127
|
// Do nothing for the first element
|
|
2150
|
-
if (!
|
|
2151
|
-
|
|
2128
|
+
if (!new_slot) {
|
|
2129
|
+
entry_slot_ = new_slot;
|
|
2152
2130
|
max_level_ = target_level;
|
|
2153
2131
|
return result;
|
|
2154
2132
|
}
|
|
2155
2133
|
|
|
2156
2134
|
// Pull stats
|
|
2157
|
-
result.
|
|
2158
|
-
result.
|
|
2135
|
+
result.computed_distances = context.computed_distances_count;
|
|
2136
|
+
result.visited_members = context.iteration_cycles;
|
|
2159
2137
|
|
|
2160
|
-
connect_node_across_levels_(
|
|
2138
|
+
connect_node_across_levels_( //
|
|
2139
|
+
value, metric, prefetch, //
|
|
2140
|
+
new_slot, entry_idx_copy, max_level_copy, target_level, //
|
|
2141
|
+
config, context);
|
|
2161
2142
|
|
|
2162
2143
|
// Normalize stats
|
|
2163
|
-
result.
|
|
2164
|
-
result.
|
|
2144
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2145
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2165
2146
|
|
|
2166
2147
|
// Updating the entry point if needed
|
|
2167
2148
|
if (target_level > max_level_copy) {
|
|
2168
|
-
|
|
2149
|
+
entry_slot_ = new_slot;
|
|
2169
2150
|
max_level_ = target_level;
|
|
2170
2151
|
}
|
|
2171
2152
|
return result;
|
|
2172
2153
|
}
|
|
2173
2154
|
|
|
2174
2155
|
/**
|
|
2175
|
-
* @brief
|
|
2156
|
+
* @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
|
|
2176
2157
|
*
|
|
2177
|
-
* @
|
|
2178
|
-
*
|
|
2179
|
-
*
|
|
2158
|
+
* @tparam metric_at
|
|
2159
|
+
* A function responsible for computing the distance @b (dis-similarity) between two objects.
|
|
2160
|
+
* It should be callable into distinctly different scenarios:
|
|
2161
|
+
* - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
|
|
2162
|
+
* - `distance_t operator() (entry_at, entry_at)` - between existing entries.
|
|
2163
|
+
* For any possible `entry_at` following interfaces will work:
|
|
2164
|
+
* - `std::size_t get_slot(entry_at const &)`
|
|
2165
|
+
* - `key_t get_key(entry_at const &)`
|
|
2166
|
+
*
|
|
2167
|
+
* @param[in] iterator Iterator pointing to an existing entry to be replaced.
|
|
2168
|
+
* @param[in] key External identifier/name/descriptor for the entry.
|
|
2169
|
+
* @param[in] value Content that will be compared against other entries in the index.
|
|
2170
|
+
* @param[in] metric Callable object measuring distance between ::value and present objects.
|
|
2180
2171
|
* @param[in] config Configuration options for this specific operation.
|
|
2172
|
+
* @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
|
|
2181
2173
|
*/
|
|
2182
|
-
|
|
2174
|
+
template < //
|
|
2175
|
+
typename value_at, //
|
|
2176
|
+
typename metric_at, //
|
|
2177
|
+
typename callback_at = dummy_callback_t, //
|
|
2178
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2179
|
+
>
|
|
2180
|
+
add_result_t update( //
|
|
2181
|
+
member_iterator_t iterator, //
|
|
2182
|
+
key_t key, //
|
|
2183
|
+
value_at&& value, //
|
|
2184
|
+
metric_at&& metric, //
|
|
2185
|
+
index_update_config_t config = {}, //
|
|
2186
|
+
callback_at&& callback = callback_at{}, //
|
|
2187
|
+
prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
|
|
2183
2188
|
|
|
2184
2189
|
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
2185
2190
|
add_result_t result;
|
|
2191
|
+
std::size_t old_slot = iterator.slot_;
|
|
2186
2192
|
|
|
2187
2193
|
// Make sure we have enough local memory to perform this request
|
|
2188
2194
|
context_t& context = contexts_[config.thread];
|
|
@@ -2193,57 +2199,77 @@ class index_gt {
|
|
|
2193
2199
|
|
|
2194
2200
|
// The top list needs one more slot than the connectivity of the base level
|
|
2195
2201
|
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
2196
|
-
std::size_t
|
|
2202
|
+
std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
|
|
2203
|
+
std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
|
|
2197
2204
|
if (!top.reserve(top_limit))
|
|
2198
2205
|
return result.failed("Out of memory!");
|
|
2199
2206
|
if (!next.reserve(config.expansion))
|
|
2200
2207
|
return result.failed("Out of memory!");
|
|
2201
2208
|
|
|
2202
|
-
node_lock_t new_lock = node_lock_(
|
|
2203
|
-
node_t node =
|
|
2209
|
+
node_lock_t new_lock = node_lock_(old_slot);
|
|
2210
|
+
node_t node = node_at_(old_slot);
|
|
2211
|
+
|
|
2212
|
+
level_t node_level = node.level();
|
|
2213
|
+
span_bytes_t node_bytes = node_bytes_(node);
|
|
2214
|
+
std::memset(node_bytes.data(), 0, node_bytes.size());
|
|
2215
|
+
node.level(node_level);
|
|
2204
2216
|
|
|
2205
2217
|
// Pull stats
|
|
2206
|
-
result.
|
|
2207
|
-
result.
|
|
2218
|
+
result.computed_distances = context.computed_distances_count;
|
|
2219
|
+
result.visited_members = context.iteration_cycles;
|
|
2208
2220
|
|
|
2209
|
-
connect_node_across_levels_(
|
|
2210
|
-
|
|
2221
|
+
connect_node_across_levels_( //
|
|
2222
|
+
value, metric, prefetch, //
|
|
2223
|
+
old_slot, entry_slot_, max_level_, node_level, //
|
|
2224
|
+
config, context);
|
|
2225
|
+
node.key(key);
|
|
2211
2226
|
|
|
2212
2227
|
// Normalize stats
|
|
2213
|
-
result.
|
|
2214
|
-
result.
|
|
2228
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2229
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2230
|
+
result.slot = old_slot;
|
|
2215
2231
|
|
|
2232
|
+
callback(at(old_slot));
|
|
2216
2233
|
return result;
|
|
2217
2234
|
}
|
|
2218
2235
|
|
|
2219
2236
|
/**
|
|
2220
2237
|
* @brief Searches for the closest elements to the given ::query. Thread-safe.
|
|
2221
2238
|
*
|
|
2222
|
-
* @param[in] query
|
|
2239
|
+
* @param[in] query Content that will be compared against other entries in the index.
|
|
2223
2240
|
* @param[in] wanted The upper bound for the number of results to return.
|
|
2224
2241
|
* @param[in] config Configuration options for this specific operation.
|
|
2225
2242
|
* @param[in] predicate Optional filtering predicate for `member_cref_t`.
|
|
2226
2243
|
* @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
|
|
2227
2244
|
*/
|
|
2228
|
-
template <
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
predicate_at
|
|
2245
|
+
template < //
|
|
2246
|
+
typename value_at, //
|
|
2247
|
+
typename metric_at, //
|
|
2248
|
+
typename predicate_at = dummy_predicate_t, //
|
|
2249
|
+
typename prefetch_at = dummy_prefetch_t //
|
|
2250
|
+
>
|
|
2251
|
+
search_result_t search( //
|
|
2252
|
+
value_at&& query, //
|
|
2253
|
+
std::size_t wanted, //
|
|
2254
|
+
metric_at&& metric, //
|
|
2255
|
+
index_search_config_t config = {}, //
|
|
2256
|
+
predicate_at&& predicate = predicate_at{}, //
|
|
2257
|
+
prefetch_at&& prefetch = prefetch_at{}) const noexcept {
|
|
2232
2258
|
|
|
2233
2259
|
context_t& context = contexts_[config.thread];
|
|
2234
2260
|
top_candidates_t& top = context.top_candidates;
|
|
2235
2261
|
search_result_t result{*this, top};
|
|
2236
|
-
if (!
|
|
2262
|
+
if (!nodes_count_)
|
|
2237
2263
|
return result;
|
|
2238
2264
|
|
|
2239
2265
|
// Go down the level, tracking only the closest match
|
|
2240
|
-
result.
|
|
2241
|
-
result.
|
|
2266
|
+
result.computed_distances = context.computed_distances_count;
|
|
2267
|
+
result.visited_members = context.iteration_cycles;
|
|
2242
2268
|
|
|
2243
2269
|
if (config.exact) {
|
|
2244
2270
|
if (!top.reserve(wanted))
|
|
2245
2271
|
return result.failed("Out of memory!");
|
|
2246
|
-
search_exact_(query, wanted, context
|
|
2272
|
+
search_exact_(query, metric, predicate, wanted, context);
|
|
2247
2273
|
} else {
|
|
2248
2274
|
next_candidates_t& next = context.next_candidates;
|
|
2249
2275
|
std::size_t expansion = (std::max)(config.expansion, wanted);
|
|
@@ -2252,9 +2278,10 @@ class index_gt {
|
|
|
2252
2278
|
if (!top.reserve(expansion))
|
|
2253
2279
|
return result.failed("Out of memory!");
|
|
2254
2280
|
|
|
2255
|
-
|
|
2281
|
+
std::size_t closest_slot = search_for_one_(query, metric, prefetch, entry_slot_, max_level_, 0, context);
|
|
2282
|
+
|
|
2256
2283
|
// For bottom layer we need a more optimized procedure
|
|
2257
|
-
if (!search_to_find_in_base_(
|
|
2284
|
+
if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
|
|
2258
2285
|
return result.failed("Out of memory!");
|
|
2259
2286
|
}
|
|
2260
2287
|
|
|
@@ -2262,8 +2289,8 @@ class index_gt {
|
|
|
2262
2289
|
top.shrink(wanted);
|
|
2263
2290
|
|
|
2264
2291
|
// Normalize stats
|
|
2265
|
-
result.
|
|
2266
|
-
result.
|
|
2292
|
+
result.computed_distances = context.computed_distances_count - result.computed_distances;
|
|
2293
|
+
result.visited_members = context.iteration_cycles - result.visited_members;
|
|
2267
2294
|
result.count = top.size();
|
|
2268
2295
|
return result;
|
|
2269
2296
|
}
|
|
@@ -2283,13 +2310,13 @@ class index_gt {
|
|
|
2283
2310
|
stats_t result{};
|
|
2284
2311
|
result.nodes = size();
|
|
2285
2312
|
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
2286
|
-
node_t node =
|
|
2287
|
-
std::size_t max_edges = node.level() * config_.connectivity +
|
|
2313
|
+
node_t node = node_at_(i);
|
|
2314
|
+
std::size_t max_edges = node.level() * config_.connectivity + config_.connectivity_base;
|
|
2288
2315
|
std::size_t edges = 0;
|
|
2289
2316
|
for (level_t level = 0; level <= node.level(); ++level)
|
|
2290
2317
|
edges += neighbors_(node, level).size();
|
|
2291
2318
|
|
|
2292
|
-
result.allocated_bytes += node_bytes_(node);
|
|
2319
|
+
result.allocated_bytes += node_bytes_(node).size();
|
|
2293
2320
|
result.edges += edges;
|
|
2294
2321
|
result.max_edges += max_edges;
|
|
2295
2322
|
}
|
|
@@ -2302,15 +2329,15 @@ class index_gt {
|
|
|
2302
2329
|
|
|
2303
2330
|
std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
|
|
2304
2331
|
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
2305
|
-
node_t node =
|
|
2332
|
+
node_t node = node_at_(i);
|
|
2306
2333
|
if (static_cast<std::size_t>(node.level()) < level)
|
|
2307
2334
|
continue;
|
|
2308
2335
|
|
|
2309
2336
|
result.edges += neighbors_(node, level).size();
|
|
2310
|
-
result.allocated_bytes += node_head_bytes_() +
|
|
2337
|
+
result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
|
|
2311
2338
|
}
|
|
2312
2339
|
|
|
2313
|
-
std::size_t max_edges_per_node = level ? config_.
|
|
2340
|
+
std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
|
|
2314
2341
|
result.max_edges = result.nodes * max_edges_per_node;
|
|
2315
2342
|
return result;
|
|
2316
2343
|
}
|
|
@@ -2318,6 +2345,8 @@ class index_gt {
|
|
|
2318
2345
|
/**
|
|
2319
2346
|
* @brief A relatively accurate lower bound on the amount of memory consumed by the system.
|
|
2320
2347
|
* In practice it's error will be below 10%.
|
|
2348
|
+
*
|
|
2349
|
+
* @see `stream_length` for the length of the binary serialized representation.
|
|
2321
2350
|
*/
|
|
2322
2351
|
std::size_t memory_usage(std::size_t allocator_entry_bytes = default_allocator_entry_bytes()) const noexcept {
|
|
2323
2352
|
std::size_t total = 0;
|
|
@@ -2335,367 +2364,297 @@ class index_gt {
|
|
|
2335
2364
|
return total;
|
|
2336
2365
|
}
|
|
2337
2366
|
|
|
2338
|
-
std::size_t memory_usage_per_node(
|
|
2339
|
-
|
|
2340
|
-
void change_metric(metric_t const& m) noexcept {
|
|
2341
|
-
metric_ = m;
|
|
2342
|
-
for (std::size_t i = 0; i != limits_.threads(); ++i)
|
|
2343
|
-
contexts_[i].metric = metric_;
|
|
2344
|
-
}
|
|
2367
|
+
std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
|
|
2345
2368
|
|
|
2346
2369
|
#pragma endregion
|
|
2347
2370
|
|
|
2348
2371
|
#pragma region Serialization
|
|
2349
2372
|
|
|
2350
|
-
struct serialization_result_t {
|
|
2351
|
-
error_t error;
|
|
2352
|
-
|
|
2353
|
-
explicit operator bool() const noexcept { return !error; }
|
|
2354
|
-
serialization_result_t failed(error_t message) noexcept {
|
|
2355
|
-
error = std::move(message);
|
|
2356
|
-
return std::move(*this);
|
|
2357
|
-
}
|
|
2358
|
-
};
|
|
2359
|
-
|
|
2360
2373
|
/**
|
|
2361
|
-
* @brief Saves serialized binary index representation to disk
|
|
2362
|
-
* co-locating vectors and neighbors lists.
|
|
2363
|
-
* Available on Linux, MacOS, Windows.
|
|
2374
|
+
* @brief Saves serialized binary index representation to a file, generally on disk.
|
|
2364
2375
|
*/
|
|
2365
2376
|
template <typename progress_at = dummy_progress_t>
|
|
2366
|
-
serialization_result_t save(
|
|
2377
|
+
serialization_result_t save(output_file_t file, progress_at&& progress = {}) const noexcept {
|
|
2378
|
+
|
|
2379
|
+
serialization_result_t result = file.open_if_not();
|
|
2380
|
+
if (result)
|
|
2381
|
+
stream(
|
|
2382
|
+
[&](void* buffer, std::size_t length) {
|
|
2383
|
+
result = file.write(buffer, length);
|
|
2384
|
+
return !!result;
|
|
2385
|
+
},
|
|
2386
|
+
std::forward<progress_at>(progress));
|
|
2387
|
+
return result;
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
/**
|
|
2391
|
+
* @brief Saves serialized binary index representation to a stream.
|
|
2392
|
+
*/
|
|
2393
|
+
template <typename output_callback_at, typename progress_at = dummy_progress_t>
|
|
2394
|
+
serialization_result_t stream(output_callback_at&& callback, progress_at&& progress = {}) const noexcept {
|
|
2367
2395
|
|
|
2368
|
-
// Make sure we have right to write to that file
|
|
2369
2396
|
serialization_result_t result;
|
|
2370
|
-
std::FILE* file = std::fopen(file_path, "wb");
|
|
2371
|
-
if (!file)
|
|
2372
|
-
return result.failed(std::strerror(errno));
|
|
2373
2397
|
|
|
2374
|
-
//
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
//
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
state.scalar_kind = metric_.scalar_kind();
|
|
2393
|
-
state.size = size_;
|
|
2394
|
-
state.entry_idx = entry_id_;
|
|
2395
|
-
|
|
2396
|
-
// Augment with metadata
|
|
2397
|
-
std::size_t graphs_bytes = 0;
|
|
2398
|
-
std::size_t vectors_bytes = 0;
|
|
2399
|
-
for (std::size_t i = 0; i != state.size; ++i) {
|
|
2400
|
-
node_t node = node_with_id_(i);
|
|
2401
|
-
std::size_t node_bytes = node_bytes_(node);
|
|
2402
|
-
std::size_t node_vector_bytes = node_vector_bytes_(node);
|
|
2403
|
-
graphs_bytes += node_bytes - node_vector_bytes;
|
|
2404
|
-
vectors_bytes += node_vector_bytes;
|
|
2398
|
+
// Export some basic metadata
|
|
2399
|
+
index_serialized_header_t header;
|
|
2400
|
+
header.size = nodes_count_;
|
|
2401
|
+
header.connectivity = config_.connectivity;
|
|
2402
|
+
header.connectivity_base = config_.connectivity_base;
|
|
2403
|
+
header.max_level = max_level_;
|
|
2404
|
+
header.entry_slot = entry_slot_;
|
|
2405
|
+
if (!callback(&header, sizeof(header)))
|
|
2406
|
+
return result.failed("Failed to serialize into stream");
|
|
2407
|
+
|
|
2408
|
+
// Export the number of levels per node
|
|
2409
|
+
// That is both enough to estimate the overall memory consumption,
|
|
2410
|
+
// and to be able to estimate the offsets of every entry in the file.
|
|
2411
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2412
|
+
node_t node = node_at_(i);
|
|
2413
|
+
level_t level = node.level();
|
|
2414
|
+
if (!callback(&level, sizeof(level)))
|
|
2415
|
+
return result.failed("Failed to serialize into stream");
|
|
2405
2416
|
}
|
|
2406
|
-
state.bytes_for_graphs = graphs_bytes;
|
|
2407
|
-
state.bytes_for_vectors = vectors_bytes;
|
|
2408
|
-
state.bytes_checksum = 0;
|
|
2409
|
-
|
|
2410
|
-
// Perform serialization
|
|
2411
|
-
auto write_chunk = [&](void* begin, std::size_t length) {
|
|
2412
|
-
std::size_t written = std::fwrite(begin, length, 1, file);
|
|
2413
|
-
if (!written) {
|
|
2414
|
-
std::fclose(file);
|
|
2415
|
-
result.failed(std::strerror(errno));
|
|
2416
|
-
}
|
|
2417
|
-
};
|
|
2418
2417
|
|
|
2419
|
-
//
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
for (std::size_t i = 0; i != state.size; ++i) {
|
|
2426
|
-
node_t node = node_with_id_(i);
|
|
2427
|
-
std::size_t node_bytes = node_bytes_(node);
|
|
2428
|
-
std::size_t node_vector_bytes = node_vector_bytes_(node);
|
|
2429
|
-
// Dump neighbors and vectors, as vectors may be in a disjoint location
|
|
2430
|
-
write_chunk(node.tape(), node_bytes - node_vector_bytes);
|
|
2431
|
-
if (result.error)
|
|
2432
|
-
return result;
|
|
2433
|
-
write_chunk(node.vector(), node_vector_bytes);
|
|
2434
|
-
if (result.error)
|
|
2435
|
-
return result;
|
|
2436
|
-
progress(i, state.size);
|
|
2418
|
+
// After that dump the nodes themselves
|
|
2419
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2420
|
+
span_bytes_t node_bytes = node_bytes_(node_at_(i));
|
|
2421
|
+
if (!callback(node_bytes.data(), node_bytes.size()))
|
|
2422
|
+
return result.failed("Failed to serialize into stream");
|
|
2423
|
+
progress(i, header.size);
|
|
2437
2424
|
}
|
|
2438
2425
|
|
|
2439
|
-
std::fclose(file);
|
|
2440
2426
|
return {};
|
|
2441
2427
|
}
|
|
2442
2428
|
|
|
2443
2429
|
/**
|
|
2444
|
-
* @brief
|
|
2445
|
-
|
|
2446
|
-
|
|
2430
|
+
* @brief Estimate the binary length (in bytes) of the serialized index.
|
|
2431
|
+
*/
|
|
2432
|
+
std::size_t stream_length() const noexcept {
|
|
2433
|
+
std::size_t neighbors_length = 0;
|
|
2434
|
+
for (std::size_t i = 0; i != size(); ++i)
|
|
2435
|
+
neighbors_length += node_bytes_(node_at_(i).level());
|
|
2436
|
+
return sizeof(index_serialized_header_t) + neighbors_length;
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
/**
|
|
2440
|
+
* @brief Loads the serialized binary index representation from disk to RAM.
|
|
2441
|
+
* Adjusts the configuration properties of the constructed index to
|
|
2442
|
+
* match the settings in the file.
|
|
2447
2443
|
*/
|
|
2448
2444
|
template <typename progress_at = dummy_progress_t>
|
|
2449
|
-
serialization_result_t load(
|
|
2445
|
+
serialization_result_t load(input_file_t file, progress_at&& progress = {}) noexcept {
|
|
2450
2446
|
|
|
2451
2447
|
// Remove previously stored objects
|
|
2452
2448
|
reset();
|
|
2453
2449
|
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
std::FILE* file = std::fopen(file_path, "rb");
|
|
2458
|
-
if (!file)
|
|
2459
|
-
return result.failed(std::strerror(errno));
|
|
2460
|
-
|
|
2461
|
-
auto read_chunk = [&](void* begin, std::size_t length) {
|
|
2462
|
-
std::size_t read = std::fread(begin, length, 1, file);
|
|
2463
|
-
if (!read) {
|
|
2464
|
-
std::fclose(file);
|
|
2465
|
-
result.failed(std::strerror(errno));
|
|
2466
|
-
}
|
|
2467
|
-
};
|
|
2468
|
-
|
|
2469
|
-
// Read the header
|
|
2470
|
-
{
|
|
2471
|
-
read_chunk(&state_buffer[0], sizeof(file_header_t));
|
|
2472
|
-
if (result.error)
|
|
2473
|
-
return result;
|
|
2450
|
+
serialization_result_t result = file.open_if_not();
|
|
2451
|
+
if (!result)
|
|
2452
|
+
return result;
|
|
2474
2453
|
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
if (state.bytes_per_id != sizeof(id_t)) {
|
|
2481
|
-
std::fclose(file);
|
|
2482
|
-
return result.failed("Incompatible ID type!");
|
|
2483
|
-
}
|
|
2454
|
+
// Pull basic metadata
|
|
2455
|
+
index_serialized_header_t header;
|
|
2456
|
+
result = file.read(&header, sizeof(header));
|
|
2457
|
+
if (!result)
|
|
2458
|
+
return result;
|
|
2484
2459
|
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2460
|
+
// Allocate some dynamic memory to read all the levels
|
|
2461
|
+
using levels_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<level_t>;
|
|
2462
|
+
buffer_gt<level_t, levels_allocator_t> levels(header.size);
|
|
2463
|
+
if (!levels)
|
|
2464
|
+
return result.failed("Out of memory");
|
|
2465
|
+
result = file.read(levels, header.size * sizeof(level_t));
|
|
2466
|
+
if (!result)
|
|
2467
|
+
return result;
|
|
2488
2468
|
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2469
|
+
// Submit metadata
|
|
2470
|
+
config_.connectivity = header.connectivity;
|
|
2471
|
+
config_.connectivity_base = header.connectivity_base;
|
|
2472
|
+
pre_ = precompute_(config_);
|
|
2473
|
+
index_limits_t limits;
|
|
2474
|
+
limits.members = header.size;
|
|
2475
|
+
if (!reserve(limits)) {
|
|
2476
|
+
reset();
|
|
2477
|
+
return result.failed("Out of memory");
|
|
2498
2478
|
}
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
if (result
|
|
2508
|
-
|
|
2509
|
-
read_chunk(&dim, sizeof(dim));
|
|
2510
|
-
if (result.error)
|
|
2479
|
+
nodes_count_ = header.size;
|
|
2480
|
+
max_level_ = static_cast<level_t>(header.max_level);
|
|
2481
|
+
entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
|
|
2482
|
+
|
|
2483
|
+
// Load the nodes
|
|
2484
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2485
|
+
span_bytes_t node_bytes = node_malloc_(levels[i]);
|
|
2486
|
+
result = file.read(node_bytes.data(), node_bytes.size());
|
|
2487
|
+
if (!result) {
|
|
2488
|
+
reset();
|
|
2511
2489
|
return result;
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
std::size_t node_bytes = node_bytes_(dim, level);
|
|
2517
|
-
node_t node = node_malloc_(dim, level);
|
|
2518
|
-
node.label(label);
|
|
2519
|
-
node.dim(dim);
|
|
2520
|
-
node.level(level);
|
|
2521
|
-
read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
|
|
2522
|
-
if (result.error)
|
|
2523
|
-
return result;
|
|
2524
|
-
nodes_[i] = node;
|
|
2525
|
-
progress(i, size);
|
|
2490
|
+
}
|
|
2491
|
+
nodes_[i] = node_t{node_bytes.data()};
|
|
2492
|
+
progress(i, header.size);
|
|
2526
2493
|
}
|
|
2527
|
-
|
|
2528
|
-
std::fclose(file);
|
|
2529
|
-
reset_view_();
|
|
2530
2494
|
return {};
|
|
2531
2495
|
}
|
|
2532
2496
|
|
|
2533
2497
|
/**
|
|
2534
2498
|
* @brief Memory-maps the serialized binary index representation from disk,
|
|
2535
|
-
* @b without copying
|
|
2536
|
-
* Available on Linux, MacOS, but @b not on Windows.
|
|
2499
|
+
* @b without copying data into RAM, and fetching it on-demand.
|
|
2537
2500
|
*/
|
|
2538
2501
|
template <typename progress_at = dummy_progress_t>
|
|
2539
|
-
serialization_result_t view(
|
|
2502
|
+
serialization_result_t view(memory_mapped_file_t file, std::size_t offset = 0,
|
|
2503
|
+
progress_at&& progress = {}) noexcept {
|
|
2540
2504
|
|
|
2541
2505
|
// Remove previously stored objects
|
|
2542
2506
|
reset();
|
|
2543
2507
|
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
2548
|
-
|
|
2549
|
-
HANDLE file_handle =
|
|
2550
|
-
CreateFile(file_path, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
|
|
2551
|
-
if (file_handle == INVALID_HANDLE_VALUE)
|
|
2552
|
-
return result.failed("Opening file failed!");
|
|
2508
|
+
serialization_result_t result = file.open_if_not();
|
|
2509
|
+
if (!result)
|
|
2510
|
+
return result;
|
|
2553
2511
|
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
if (
|
|
2557
|
-
|
|
2558
|
-
|
|
2512
|
+
// Pull basic metadata
|
|
2513
|
+
index_serialized_header_t header;
|
|
2514
|
+
if (file.size() - offset < sizeof(header))
|
|
2515
|
+
return result.failed("File is corrupted and lacks a header");
|
|
2516
|
+
std::memcpy(&header, file.data() + offset, sizeof(header));
|
|
2517
|
+
|
|
2518
|
+
// Precompute offsets of every node, but before that we need to update the configs
|
|
2519
|
+
// This could have been done with `std::exclusive_scan`, but it's only available from C++17.
|
|
2520
|
+
using offsets_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
|
|
2521
|
+
buffer_gt<std::size_t, offsets_allocator_t> offsets(header.size);
|
|
2522
|
+
if (!offsets)
|
|
2523
|
+
return result.failed("Out of memory");
|
|
2524
|
+
|
|
2525
|
+
config_.connectivity = header.connectivity;
|
|
2526
|
+
config_.connectivity_base = header.connectivity_base;
|
|
2527
|
+
pre_ = precompute_(config_);
|
|
2528
|
+
misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
|
|
2529
|
+
offsets[0] = offset + sizeof(header) + sizeof(level_t) * header.size;
|
|
2530
|
+
for (std::size_t i = 1; i < header.size; ++i)
|
|
2531
|
+
offsets[i] = offsets[i - 1] + node_bytes_(levels[i - 1]);
|
|
2532
|
+
|
|
2533
|
+
std::size_t total_bytes = offsets[header.size - 1] + node_bytes_(levels[header.size - 1]);
|
|
2534
|
+
if (file.size() < total_bytes) {
|
|
2535
|
+
reset();
|
|
2536
|
+
return result.failed("File is corrupted and can't fit all the nodes");
|
|
2559
2537
|
}
|
|
2560
2538
|
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2539
|
+
// Submit metadata and reserve memory
|
|
2540
|
+
index_limits_t limits;
|
|
2541
|
+
limits.members = header.size;
|
|
2542
|
+
if (!reserve(limits)) {
|
|
2543
|
+
reset();
|
|
2544
|
+
return result.failed("Out of memory");
|
|
2566
2545
|
}
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
#else
|
|
2576
|
-
int descriptor = open(file_path, O_RDONLY);
|
|
2577
|
-
#endif
|
|
2578
|
-
|
|
2579
|
-
// Estimate the file size
|
|
2580
|
-
struct stat file_stat;
|
|
2581
|
-
int fstat_status = fstat(descriptor, &file_stat);
|
|
2582
|
-
if (fstat_status < 0) {
|
|
2583
|
-
close(descriptor);
|
|
2584
|
-
return result.failed(std::strerror(errno));
|
|
2546
|
+
nodes_count_ = header.size;
|
|
2547
|
+
max_level_ = static_cast<level_t>(header.max_level);
|
|
2548
|
+
entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
|
|
2549
|
+
|
|
2550
|
+
// Rapidly address all the nodes
|
|
2551
|
+
for (std::size_t i = 0; i != header.size; ++i) {
|
|
2552
|
+
nodes_[i] = node_t{(byte_t*)file.data() + offsets[i]};
|
|
2553
|
+
progress(i, header.size);
|
|
2585
2554
|
}
|
|
2555
|
+
viewed_file_ = std::move(file);
|
|
2556
|
+
return {};
|
|
2557
|
+
}
|
|
2586
2558
|
|
|
2587
|
-
|
|
2588
|
-
byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_PRIVATE, descriptor, 0);
|
|
2589
|
-
if (file == MAP_FAILED) {
|
|
2590
|
-
close(descriptor);
|
|
2591
|
-
return result.failed(std::strerror(errno));
|
|
2592
|
-
}
|
|
2593
|
-
viewed_file_.file_descriptor = descriptor;
|
|
2594
|
-
viewed_file_.ptr = file;
|
|
2595
|
-
viewed_file_.length = file_stat.st_size;
|
|
2596
|
-
#endif // Platform specific code
|
|
2559
|
+
#pragma endregion
|
|
2597
2560
|
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2561
|
+
/**
|
|
2562
|
+
* @brief Performs compaction on the whole HNSW index, purging some entries
|
|
2563
|
+
* and links to them, while also generating a more efficient mapping,
|
|
2564
|
+
* putting the more frequently used entries closer together.
|
|
2565
|
+
*
|
|
2566
|
+
*
|
|
2567
|
+
* Scans the whole collection, removing the links leading towards
|
|
2568
|
+
* banned entries. This essentially isolates some nodes from the rest
|
|
2569
|
+
* of the graph, while keeping their outgoing links, in case the node
|
|
2570
|
+
* is structurally relevant and has a crucial role in the index.
|
|
2571
|
+
* It won't reclaim the memory.
|
|
2572
|
+
*
|
|
2573
|
+
* @param[in] allow_member Predicate to mark nodes for isolation.
|
|
2574
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
2575
|
+
* @param[in] progress Callback to report the execution progress.
|
|
2576
|
+
*/
|
|
2577
|
+
template <typename values_at, typename metric_at, //
|
|
2578
|
+
typename slot_transition_at = dummy_label_to_label_mapping_t, //
|
|
2579
|
+
typename executor_at = dummy_executor_t, //
|
|
2580
|
+
typename progress_at = dummy_progress_t, //
|
|
2581
|
+
typename prefetch_at = dummy_prefetch_t>
|
|
2582
|
+
void compact( //
|
|
2583
|
+
values_at&& values, //
|
|
2584
|
+
metric_at&& metric, //
|
|
2585
|
+
slot_transition_at&& slot_transition, //
|
|
2609
2586
|
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2587
|
+
executor_at&& executor = executor_at{}, //
|
|
2588
|
+
progress_at&& progress = progress_at{}, //
|
|
2589
|
+
prefetch_at&& prefetch = prefetch_at{}) noexcept {
|
|
2590
|
+
|
|
2591
|
+
// Export all the keys, slots, and levels.
|
|
2592
|
+
// Partition them with the predicate.
|
|
2593
|
+
// Sort the allowed entries in descending order of their level.
|
|
2594
|
+
// Create a new array mapping old slots to the new ones (INT_MAX for deleted items).
|
|
2595
|
+
struct slot_level_t {
|
|
2596
|
+
compressed_slot_t old_slot;
|
|
2597
|
+
compressed_slot_t cluster;
|
|
2598
|
+
level_t level;
|
|
2599
|
+
};
|
|
2600
|
+
using slot_level_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<slot_level_t>;
|
|
2601
|
+
buffer_gt<slot_level_t, slot_level_allocator_t> slots_and_levels(size());
|
|
2602
|
+
|
|
2603
|
+
// For every bottom level node, determine its parent cluster
|
|
2604
|
+
executor.execute_bulk(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
|
|
2605
|
+
context_t& context = contexts_[thread_idx];
|
|
2606
|
+
std::size_t cluster = search_for_one_( //
|
|
2607
|
+
values[citerator_at(old_slot)], //
|
|
2608
|
+
metric, prefetch, //
|
|
2609
|
+
entry_slot_, max_level_, 0, context);
|
|
2610
|
+
slots_and_levels[old_slot] = { //
|
|
2611
|
+
static_cast<compressed_slot_t>(old_slot), //
|
|
2612
|
+
static_cast<compressed_slot_t>(cluster), //
|
|
2613
|
+
node_at_(old_slot).level()};
|
|
2614
|
+
});
|
|
2613
2615
|
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
return result.failed("Out of memory!");
|
|
2616
|
+
// Where the actual permutation happens:
|
|
2617
|
+
std::sort(slots_and_levels.begin(), slots_and_levels.end(), [](slot_level_t const& a, slot_level_t const& b) {
|
|
2618
|
+
return a.level == b.level ? a.cluster < b.cluster : a.level > b.level;
|
|
2619
|
+
});
|
|
2619
2620
|
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2621
|
+
using size_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
|
|
2622
|
+
buffer_gt<std::size_t, size_allocator_t> old_slot_to_new(slots_and_levels.size());
|
|
2623
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot)
|
|
2624
|
+
old_slot_to_new[slots_and_levels[new_slot].old_slot] = new_slot;
|
|
2624
2625
|
|
|
2625
|
-
//
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
for (std::size_t i = 0; i != size; ++i) {
|
|
2629
|
-
byte_t* tape = (byte_t*)(file + progress_bytes);
|
|
2630
|
-
dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
|
|
2631
|
-
level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
|
|
2632
|
-
|
|
2633
|
-
std::size_t node_bytes = node_bytes_(dim, level);
|
|
2634
|
-
std::size_t node_vector_bytes = dim * sizeof(scalar_t);
|
|
2635
|
-
nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
|
|
2636
|
-
progress_bytes += node_bytes;
|
|
2637
|
-
progress(i, size);
|
|
2638
|
-
}
|
|
2626
|
+
// Erase all the incoming links
|
|
2627
|
+
buffer_gt<node_t, nodes_allocator_t> reordered_nodes(slots_and_levels.size());
|
|
2628
|
+
tape_allocator_t reordered_tape;
|
|
2639
2629
|
|
|
2640
|
-
|
|
2641
|
-
|
|
2630
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
|
|
2631
|
+
std::size_t old_slot = slots_and_levels[new_slot].old_slot;
|
|
2632
|
+
node_t old_node = node_at_(old_slot);
|
|
2642
2633
|
|
|
2643
|
-
|
|
2634
|
+
std::size_t node_bytes = node_bytes_(old_node.level());
|
|
2635
|
+
byte_t* new_data = (byte_t*)reordered_tape.allocate(node_bytes);
|
|
2636
|
+
node_t new_node{new_data};
|
|
2637
|
+
std::memcpy(new_data, old_node.tape(), node_bytes);
|
|
2644
2638
|
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
std::size_t engagements{};
|
|
2649
|
-
std::size_t cycles{};
|
|
2650
|
-
std::size_t measurements{};
|
|
2639
|
+
for (level_t level = 0; level <= old_node.level(); ++level)
|
|
2640
|
+
for (misaligned_ref_gt<compressed_slot_t> neighbor : neighbors_(new_node, level))
|
|
2641
|
+
neighbor = static_cast<compressed_slot_t>(old_slot_to_new[compressed_slot_t(neighbor)]);
|
|
2651
2642
|
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
return std::move(*this);
|
|
2643
|
+
reordered_nodes[new_slot] = new_node;
|
|
2644
|
+
|
|
2645
|
+
progress(new_slot, slots_and_levels.size());
|
|
2656
2646
|
}
|
|
2657
|
-
};
|
|
2658
2647
|
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
* @param[inout] second_to_first Container to map ::second labels to ::first.
|
|
2666
|
-
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
2667
|
-
* @param[in] progress Callback to report the execution progress.
|
|
2668
|
-
*/
|
|
2669
|
-
template < //
|
|
2670
|
-
typename first_to_second_at = dummy_label_to_label_mapping_t, //
|
|
2671
|
-
typename second_to_first_at = dummy_label_to_label_mapping_t, //
|
|
2672
|
-
typename executor_at = dummy_executor_t, //
|
|
2673
|
-
typename progress_at = dummy_progress_t //
|
|
2674
|
-
>
|
|
2675
|
-
static join_result_t join( //
|
|
2676
|
-
index_gt const& first, index_gt const& second, //
|
|
2677
|
-
join_config_t config = {}, //
|
|
2678
|
-
first_to_second_at&& first_to_second = first_to_second_at{}, //
|
|
2679
|
-
second_to_first_at&& second_to_first = second_to_first_at{}, //
|
|
2680
|
-
executor_at&& executor = executor_at{}, //
|
|
2681
|
-
progress_at&& progress = progress_at{}) noexcept {
|
|
2648
|
+
for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
|
|
2649
|
+
std::size_t old_slot = slots_and_levels[new_slot].old_slot;
|
|
2650
|
+
slot_transition(node_at_(old_slot).ckey(), //
|
|
2651
|
+
static_cast<compressed_slot_t>(old_slot), //
|
|
2652
|
+
static_cast<compressed_slot_t>(new_slot));
|
|
2653
|
+
}
|
|
2682
2654
|
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
config, //
|
|
2687
|
-
std::forward<second_to_first_at>(second_to_first), //
|
|
2688
|
-
std::forward<first_to_second_at>(first_to_second), //
|
|
2689
|
-
std::forward<executor_at>(executor), //
|
|
2690
|
-
std::forward<progress_at>(progress));
|
|
2691
|
-
else
|
|
2692
|
-
return join_small_and_big_( //
|
|
2693
|
-
first, second, //
|
|
2694
|
-
config, //
|
|
2695
|
-
std::forward<first_to_second_at>(first_to_second), //
|
|
2696
|
-
std::forward<second_to_first_at>(second_to_first), //
|
|
2697
|
-
std::forward<executor_at>(executor), //
|
|
2698
|
-
std::forward<progress_at>(progress));
|
|
2655
|
+
nodes_ = std::move(reordered_nodes);
|
|
2656
|
+
tape_allocator_ = std::move(reordered_tape);
|
|
2657
|
+
entry_slot_ = old_slot_to_new[entry_slot_];
|
|
2699
2658
|
}
|
|
2700
2659
|
|
|
2701
2660
|
/**
|
|
@@ -2720,293 +2679,80 @@ class index_gt {
|
|
|
2720
2679
|
progress_at&& progress = progress_at{}) noexcept {
|
|
2721
2680
|
|
|
2722
2681
|
// Erase all the incoming links
|
|
2723
|
-
|
|
2724
|
-
|
|
2682
|
+
std::size_t nodes_count = size();
|
|
2683
|
+
executor.execute_bulk(nodes_count, [&](std::size_t, std::size_t node_idx) {
|
|
2684
|
+
node_t node = node_at_(node_idx);
|
|
2725
2685
|
for (level_t level = 0; level <= node.level(); ++level) {
|
|
2726
2686
|
neighbors_ref_t neighbors = neighbors_(node, level);
|
|
2727
2687
|
std::size_t old_size = neighbors.size();
|
|
2728
2688
|
neighbors.clear();
|
|
2729
2689
|
for (std::size_t i = 0; i != old_size; ++i) {
|
|
2730
|
-
|
|
2731
|
-
node_t neighbor =
|
|
2732
|
-
if (allow_member(member_cref_t{neighbor.
|
|
2733
|
-
neighbors.push_back(
|
|
2690
|
+
compressed_slot_t neighbor_slot = neighbors[i];
|
|
2691
|
+
node_t neighbor = node_at_(neighbor_slot);
|
|
2692
|
+
if (allow_member(member_cref_t{neighbor.ckey(), neighbor_slot}))
|
|
2693
|
+
neighbors.push_back(neighbor_slot);
|
|
2734
2694
|
}
|
|
2735
2695
|
}
|
|
2696
|
+
progress(node_idx, nodes_count);
|
|
2736
2697
|
});
|
|
2737
2698
|
}
|
|
2738
2699
|
|
|
2739
2700
|
private:
|
|
2740
|
-
template <typename first_to_second_at, typename second_to_first_at, typename executor_at, typename progress_at>
|
|
2741
|
-
static join_result_t join_small_and_big_( //
|
|
2742
|
-
index_gt const& men, index_gt const& women, //
|
|
2743
|
-
join_config_t config, //
|
|
2744
|
-
first_to_second_at&& man_to_woman, //
|
|
2745
|
-
second_to_first_at&& woman_to_man, //
|
|
2746
|
-
executor_at&& executor, //
|
|
2747
|
-
progress_at&& progress) noexcept {
|
|
2748
|
-
|
|
2749
|
-
join_result_t result;
|
|
2750
|
-
|
|
2751
|
-
// Sanity checks and argument validation:
|
|
2752
|
-
if (&men == &women)
|
|
2753
|
-
return result.failed("Can't join with itself, consider copying");
|
|
2754
|
-
if (config.max_proposals == 0)
|
|
2755
|
-
config.max_proposals = std::log(men.size()) + executor.size();
|
|
2756
|
-
config.max_proposals = (std::min)(men.size(), config.max_proposals);
|
|
2757
|
-
|
|
2758
|
-
using proposals_count_t = std::uint16_t;
|
|
2759
|
-
using ids_allocator_t = typename allocator_traits_t::template rebind_alloc<id_t>;
|
|
2760
|
-
dynamic_allocator_t& alloc = men.dynamic_allocator_;
|
|
2761
|
-
|
|
2762
|
-
// Create an atomic queue, as a ring structure, from/to which
|
|
2763
|
-
// free men will be added/pulled.
|
|
2764
|
-
std::mutex free_men_mutex{};
|
|
2765
|
-
ring_gt<id_t, ids_allocator_t> free_men;
|
|
2766
|
-
free_men.reserve(men.size());
|
|
2767
|
-
for (std::size_t i = 0; i != men.size(); ++i)
|
|
2768
|
-
free_men.push(static_cast<id_t>(i));
|
|
2769
|
-
|
|
2770
|
-
// We are gonna need some temporary memory.
|
|
2771
|
-
proposals_count_t* proposal_counts = (proposals_count_t*)alloc.allocate(sizeof(proposals_count_t) * men.size());
|
|
2772
|
-
id_t* man_to_woman_ids = (id_t*)alloc.allocate(sizeof(id_t) * men.size());
|
|
2773
|
-
id_t* woman_to_man_ids = (id_t*)alloc.allocate(sizeof(id_t) * women.size());
|
|
2774
|
-
if (!proposal_counts || !man_to_woman_ids || !woman_to_man_ids)
|
|
2775
|
-
return result.failed("Can't temporary mappings");
|
|
2776
|
-
|
|
2777
|
-
id_t missing_id;
|
|
2778
|
-
std::memset((void*)&missing_id, 0xFF, sizeof(id_t));
|
|
2779
|
-
std::memset((void*)man_to_woman_ids, 0xFF, sizeof(id_t) * men.size());
|
|
2780
|
-
std::memset((void*)woman_to_man_ids, 0xFF, sizeof(id_t) * women.size());
|
|
2781
|
-
std::memset(proposal_counts, 0, sizeof(proposals_count_t) * men.size());
|
|
2782
|
-
|
|
2783
|
-
// Define locks, to limit concurrent accesses to `man_to_woman_ids` and `woman_to_man_ids`.
|
|
2784
|
-
visits_bitset_t men_locks, women_locks;
|
|
2785
|
-
if (!men_locks.resize(men.size()) || !women_locks.resize(women.size()))
|
|
2786
|
-
return result.failed("Can't allocate locks");
|
|
2787
|
-
|
|
2788
|
-
// Accumulate statistics from all the contexts,
|
|
2789
|
-
// to have a baseline to compare with, by the time the `join` is finished.
|
|
2790
|
-
std::size_t old_measurements{};
|
|
2791
|
-
std::size_t old_cycles{};
|
|
2792
|
-
for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
|
|
2793
|
-
old_measurements += women.contexts_[thread_idx].measurements_count;
|
|
2794
|
-
old_cycles += women.contexts_[thread_idx].iteration_cycles;
|
|
2795
|
-
}
|
|
2796
|
-
std::atomic<std::size_t> rounds{0};
|
|
2797
|
-
std::atomic<std::size_t> engagements{0};
|
|
2798
|
-
|
|
2799
|
-
// Concurrently process all the men
|
|
2800
|
-
executor.execute_bulk([&](std::size_t thread_idx) {
|
|
2801
|
-
context_t& context = women.contexts_[thread_idx];
|
|
2802
|
-
search_config_t search_config;
|
|
2803
|
-
search_config.expansion = config.expansion;
|
|
2804
|
-
search_config.exact = config.exact;
|
|
2805
|
-
search_config.thread = thread_idx;
|
|
2806
|
-
id_t free_man_id;
|
|
2807
|
-
|
|
2808
|
-
// While there exist a free man who still has a woman to propose to.
|
|
2809
|
-
while (true) {
|
|
2810
|
-
std::size_t passed_rounds = 0;
|
|
2811
|
-
std::size_t total_rounds = 0;
|
|
2812
|
-
{
|
|
2813
|
-
std::unique_lock<std::mutex> pop_lock(free_men_mutex);
|
|
2814
|
-
if (!free_men.try_pop(free_man_id))
|
|
2815
|
-
// Primary exit path, we have exhausted the list of candidates
|
|
2816
|
-
break;
|
|
2817
|
-
passed_rounds = ++rounds;
|
|
2818
|
-
total_rounds = passed_rounds + free_men.size();
|
|
2819
|
-
}
|
|
2820
|
-
progress(passed_rounds, total_rounds);
|
|
2821
|
-
while (men_locks.atomic_set(free_man_id))
|
|
2822
|
-
;
|
|
2823
|
-
|
|
2824
|
-
node_t free_man = men.node_with_id_(free_man_id);
|
|
2825
|
-
proposals_count_t& free_man_proposals = proposal_counts[free_man_id];
|
|
2826
|
-
if (free_man_proposals >= config.max_proposals)
|
|
2827
|
-
continue;
|
|
2828
|
-
|
|
2829
|
-
// Find the closest woman, to whom this man hasn't proposed yet.
|
|
2830
|
-
free_man_proposals++;
|
|
2831
|
-
search_result_t candidates = women.search(free_man.vector_view(), free_man_proposals, search_config);
|
|
2832
|
-
if (!candidates) {
|
|
2833
|
-
// TODO:
|
|
2834
|
-
}
|
|
2835
|
-
|
|
2836
|
-
match_t match = candidates.back();
|
|
2837
|
-
member_cref_t woman = match.member;
|
|
2838
|
-
while (women_locks.atomic_set(woman.id))
|
|
2839
|
-
;
|
|
2840
|
-
|
|
2841
|
-
id_t husband_id = woman_to_man_ids[woman.id];
|
|
2842
|
-
bool woman_is_free = husband_id == missing_id;
|
|
2843
|
-
if (woman_is_free) {
|
|
2844
|
-
// Engagement
|
|
2845
|
-
man_to_woman_ids[free_man_id] = woman.id;
|
|
2846
|
-
woman_to_man_ids[woman.id] = free_man_id;
|
|
2847
|
-
engagements++;
|
|
2848
|
-
} else {
|
|
2849
|
-
distance_t distance_from_husband = context.measure(woman.vector, men.node_with_id_(husband_id));
|
|
2850
|
-
distance_t distance_from_candidate = match.distance;
|
|
2851
|
-
if (distance_from_husband > distance_from_candidate) {
|
|
2852
|
-
// Break-up
|
|
2853
|
-
while (men_locks.atomic_set(husband_id))
|
|
2854
|
-
;
|
|
2855
|
-
man_to_woman_ids[husband_id] = missing_id;
|
|
2856
|
-
men_locks.atomic_reset(husband_id);
|
|
2857
|
-
|
|
2858
|
-
// New Engagement
|
|
2859
|
-
man_to_woman_ids[free_man_id] = woman.id;
|
|
2860
|
-
woman_to_man_ids[woman.id] = free_man_id;
|
|
2861
|
-
engagements++;
|
|
2862
|
-
|
|
2863
|
-
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
2864
|
-
free_men.push(husband_id);
|
|
2865
|
-
} else {
|
|
2866
|
-
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
2867
|
-
free_men.push(free_man_id);
|
|
2868
|
-
}
|
|
2869
|
-
}
|
|
2870
|
-
|
|
2871
|
-
men_locks.atomic_reset(free_man_id);
|
|
2872
|
-
women_locks.atomic_reset(woman.id);
|
|
2873
|
-
}
|
|
2874
|
-
});
|
|
2875
|
-
|
|
2876
|
-
// Export the IDs into labels:
|
|
2877
|
-
std::size_t intersection_size = 0;
|
|
2878
|
-
for (std::size_t i = 0; i != men.size(); ++i) {
|
|
2879
|
-
id_t woman_id = man_to_woman_ids[i];
|
|
2880
|
-
if (woman_id != missing_id) {
|
|
2881
|
-
label_t man = men.node_with_id_(i).label();
|
|
2882
|
-
label_t woman = women.node_with_id_(woman_id).label();
|
|
2883
|
-
man_to_woman[man] = woman;
|
|
2884
|
-
woman_to_man[woman] = man;
|
|
2885
|
-
intersection_size++;
|
|
2886
|
-
}
|
|
2887
|
-
}
|
|
2888
|
-
|
|
2889
|
-
// Deallocate memory
|
|
2890
|
-
alloc.deallocate((byte_t*)proposal_counts, sizeof(proposals_count_t) * men.size());
|
|
2891
|
-
alloc.deallocate((byte_t*)man_to_woman_ids, sizeof(id_t) * men.size());
|
|
2892
|
-
alloc.deallocate((byte_t*)woman_to_man_ids, sizeof(id_t) * women.size());
|
|
2893
|
-
|
|
2894
|
-
// Export stats
|
|
2895
|
-
result.engagements = engagements;
|
|
2896
|
-
result.intersection_size = intersection_size;
|
|
2897
|
-
for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
|
|
2898
|
-
result.measurements += women.contexts_[thread_idx].measurements_count;
|
|
2899
|
-
result.cycles += women.contexts_[thread_idx].iteration_cycles;
|
|
2900
|
-
}
|
|
2901
|
-
result.measurements -= old_measurements;
|
|
2902
|
-
result.cycles -= old_cycles;
|
|
2903
|
-
return result;
|
|
2904
|
-
}
|
|
2905
|
-
|
|
2906
|
-
void reset_view_() noexcept {
|
|
2907
|
-
if (!viewed_file_)
|
|
2908
|
-
return;
|
|
2909
|
-
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
2910
|
-
UnmapViewOfFile(viewed_file_.ptr);
|
|
2911
|
-
CloseHandle(viewed_file_.mapping_handle);
|
|
2912
|
-
CloseHandle(viewed_file_.file_handle);
|
|
2913
|
-
|
|
2914
|
-
#else
|
|
2915
|
-
munmap(viewed_file_.ptr, viewed_file_.length);
|
|
2916
|
-
close(viewed_file_.file_descriptor);
|
|
2917
|
-
#endif
|
|
2918
|
-
viewed_file_ = {};
|
|
2919
|
-
}
|
|
2920
|
-
|
|
2921
2701
|
inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
|
|
2922
2702
|
precomputed_constants_t pre;
|
|
2923
|
-
pre.connectivity_max_base = config.connectivity * base_level_multiple_();
|
|
2924
2703
|
pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
|
|
2925
|
-
pre.neighbors_bytes = config.connectivity * sizeof(
|
|
2926
|
-
pre.neighbors_base_bytes =
|
|
2704
|
+
pre.neighbors_bytes = config.connectivity * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
|
|
2705
|
+
pre.neighbors_base_bytes = config.connectivity_base * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
|
|
2927
2706
|
return pre;
|
|
2928
2707
|
}
|
|
2929
2708
|
|
|
2930
|
-
inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
|
|
2931
|
-
inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
|
|
2932
|
-
return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
|
|
2933
|
-
}
|
|
2934
|
-
|
|
2935
2709
|
using span_bytes_t = span_gt<byte_t>;
|
|
2936
|
-
struct node_bytes_split_t {
|
|
2937
|
-
span_bytes_t tape{};
|
|
2938
|
-
span_bytes_t vector{};
|
|
2939
|
-
|
|
2940
|
-
node_bytes_split_t() {}
|
|
2941
|
-
node_bytes_split_t(span_bytes_t tape, span_bytes_t vector) noexcept : tape(tape), vector(vector) {}
|
|
2942
2710
|
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
}
|
|
2947
|
-
|
|
2948
|
-
inline
|
|
2949
|
-
|
|
2950
|
-
std::size_t bytes_in_tape = node_head_bytes_() + levels_bytes;
|
|
2951
|
-
return {{node.tape(), bytes_in_tape}, {(byte_t*)node.vector(), node_vector_bytes_(node)}};
|
|
2711
|
+
inline span_bytes_t node_bytes_(node_t node) const noexcept { return {node.tape(), node_bytes_(node.level())}; }
|
|
2712
|
+
inline std::size_t node_bytes_(level_t level) const noexcept {
|
|
2713
|
+
return node_head_bytes_() + node_neighbors_bytes_(level);
|
|
2714
|
+
}
|
|
2715
|
+
inline std::size_t node_neighbors_bytes_(node_t node) const noexcept { return node_neighbors_bytes_(node.level()); }
|
|
2716
|
+
inline std::size_t node_neighbors_bytes_(level_t level) const noexcept {
|
|
2717
|
+
return pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
|
|
2952
2718
|
}
|
|
2953
2719
|
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
node_bytes_split_t node_malloc_(dim_t dims_to_store, level_t level) noexcept {
|
|
2958
|
-
|
|
2959
|
-
std::size_t vector_bytes = node_vector_bytes_(dims_to_store);
|
|
2960
|
-
std::size_t node_bytes = node_bytes_(dims_to_store, level);
|
|
2961
|
-
std::size_t non_vector_bytes = node_bytes - vector_bytes;
|
|
2962
|
-
|
|
2720
|
+
span_bytes_t node_malloc_(level_t level) noexcept {
|
|
2721
|
+
std::size_t node_bytes = node_bytes_(level);
|
|
2963
2722
|
byte_t* data = (byte_t*)tape_allocator_.allocate(node_bytes);
|
|
2964
|
-
|
|
2965
|
-
return node_bytes_split_t{};
|
|
2966
|
-
return {{data, non_vector_bytes}, {data + non_vector_bytes, vector_bytes}};
|
|
2723
|
+
return data ? span_bytes_t{data, node_bytes} : span_bytes_t{};
|
|
2967
2724
|
}
|
|
2968
2725
|
|
|
2969
|
-
node_t node_make_(
|
|
2970
|
-
|
|
2971
|
-
if (
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
node_t node = node_bytes;
|
|
2978
|
-
node.label(label);
|
|
2979
|
-
node.dim(static_cast<dim_t>(vector.size()));
|
|
2726
|
+
node_t node_make_(key_t key, level_t level) noexcept {
|
|
2727
|
+
span_bytes_t node_bytes = node_malloc_(level);
|
|
2728
|
+
if (!node_bytes)
|
|
2729
|
+
return {};
|
|
2730
|
+
|
|
2731
|
+
std::memset(node_bytes.data(), 0, node_bytes.size());
|
|
2732
|
+
node_t node{(byte_t*)node_bytes.data()};
|
|
2733
|
+
node.key(key);
|
|
2980
2734
|
node.level(level);
|
|
2981
2735
|
return node;
|
|
2982
2736
|
}
|
|
2983
2737
|
|
|
2984
|
-
node_t node_make_copy_(
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
node_t old_node = old_bytes;
|
|
2991
|
-
node_bytes_split_t node_bytes = node_malloc_(old_node.vector_view().size(), old_node.level());
|
|
2992
|
-
std::memcpy(node_bytes.tape.data(), old_bytes.tape.data(), old_bytes.tape.size());
|
|
2993
|
-
std::memcpy(node_bytes.vector.data(), old_bytes.vector.data(), old_bytes.vector.size());
|
|
2994
|
-
return node_bytes;
|
|
2995
|
-
}
|
|
2738
|
+
node_t node_make_copy_(span_bytes_t old_bytes) noexcept {
|
|
2739
|
+
byte_t* data = (byte_t*)tape_allocator_.allocate(old_bytes.size());
|
|
2740
|
+
if (!data)
|
|
2741
|
+
return {};
|
|
2742
|
+
std::memcpy(data, old_bytes.data(), old_bytes.size());
|
|
2743
|
+
return node_t{data};
|
|
2996
2744
|
}
|
|
2997
2745
|
|
|
2998
|
-
void node_free_(std::size_t
|
|
2999
|
-
|
|
2746
|
+
void node_free_(std::size_t idx) noexcept {
|
|
3000
2747
|
if (viewed_file_)
|
|
3001
2748
|
return;
|
|
3002
2749
|
|
|
3003
|
-
node_t& node = nodes_[
|
|
3004
|
-
|
|
3005
|
-
tape_allocator_.deallocate(node.tape(), node_bytes);
|
|
2750
|
+
node_t& node = nodes_[idx];
|
|
2751
|
+
tape_allocator_.deallocate(node.tape(), node_bytes_(node).size());
|
|
3006
2752
|
node = node_t{};
|
|
3007
2753
|
}
|
|
3008
2754
|
|
|
3009
|
-
inline node_t
|
|
2755
|
+
inline node_t node_at_(std::size_t idx) const noexcept { return nodes_[idx]; }
|
|
3010
2756
|
inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
|
|
3011
2757
|
|
|
3012
2758
|
inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
|
|
@@ -3019,91 +2765,102 @@ class index_gt {
|
|
|
3019
2765
|
|
|
3020
2766
|
struct node_lock_t {
|
|
3021
2767
|
visits_bitset_t& bitset;
|
|
3022
|
-
std::size_t
|
|
3023
|
-
|
|
3024
|
-
inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
|
|
2768
|
+
std::size_t slot;
|
|
2769
|
+
inline ~node_lock_t() noexcept { bitset.atomic_reset(slot); }
|
|
3025
2770
|
};
|
|
3026
2771
|
|
|
3027
|
-
inline node_lock_t node_lock_(std::size_t
|
|
3028
|
-
while (nodes_mutexes_.atomic_set(
|
|
2772
|
+
inline node_lock_t node_lock_(std::size_t slot) const noexcept {
|
|
2773
|
+
while (nodes_mutexes_.atomic_set(slot))
|
|
3029
2774
|
;
|
|
3030
|
-
return {nodes_mutexes_,
|
|
2775
|
+
return {nodes_mutexes_, slot};
|
|
3031
2776
|
}
|
|
3032
2777
|
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
2778
|
+
template <typename value_at, typename metric_at, typename prefetch_at>
|
|
2779
|
+
void connect_node_across_levels_( //
|
|
2780
|
+
value_at&& value, metric_at&& metric, prefetch_at&& prefetch, //
|
|
2781
|
+
std::size_t node_slot, std::size_t entry_slot, level_t max_level, level_t target_level, //
|
|
2782
|
+
index_update_config_t const& config, context_t& context) usearch_noexcept_m {
|
|
3037
2783
|
|
|
3038
2784
|
// Go down the level, tracking only the closest match
|
|
3039
|
-
|
|
2785
|
+
std::size_t closest_slot = search_for_one_( //
|
|
2786
|
+
value, metric, prefetch, //
|
|
2787
|
+
entry_slot, max_level, target_level, context);
|
|
3040
2788
|
|
|
3041
2789
|
// From `target_level` down perform proper extensive search
|
|
3042
2790
|
for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
|
|
3043
2791
|
// TODO: Handle out of memory conditions
|
|
3044
|
-
search_to_insert_(
|
|
3045
|
-
|
|
3046
|
-
reconnect_neighbor_nodes_(
|
|
2792
|
+
search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
|
|
2793
|
+
closest_slot = connect_new_node_(metric, node_slot, level, context);
|
|
2794
|
+
reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
|
|
3047
2795
|
}
|
|
3048
2796
|
}
|
|
3049
2797
|
|
|
3050
|
-
|
|
2798
|
+
template <typename metric_at>
|
|
2799
|
+
std::size_t connect_new_node_( //
|
|
2800
|
+
metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
|
|
3051
2801
|
|
|
3052
|
-
node_t new_node =
|
|
2802
|
+
node_t new_node = node_at_(new_slot);
|
|
3053
2803
|
top_candidates_t& top = context.top_candidates;
|
|
3054
2804
|
|
|
3055
|
-
// Outgoing links from `
|
|
2805
|
+
// Outgoing links from `new_slot`:
|
|
3056
2806
|
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3057
2807
|
{
|
|
3058
2808
|
usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
|
|
3059
|
-
candidates_view_t top_view = refine_(
|
|
2809
|
+
candidates_view_t top_view = refine_(metric, config_.connectivity, top, context);
|
|
3060
2810
|
|
|
3061
2811
|
for (std::size_t idx = 0; idx != top_view.size(); idx++) {
|
|
3062
2812
|
usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
3063
|
-
usearch_assert_m(level <=
|
|
3064
|
-
new_neighbors.push_back(top_view[idx].
|
|
2813
|
+
usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
|
|
2814
|
+
new_neighbors.push_back(top_view[idx].slot);
|
|
3065
2815
|
}
|
|
3066
2816
|
}
|
|
3067
2817
|
|
|
3068
2818
|
return new_neighbors[0];
|
|
3069
2819
|
}
|
|
3070
2820
|
|
|
3071
|
-
|
|
2821
|
+
template <typename value_at, typename metric_at>
|
|
2822
|
+
void reconnect_neighbor_nodes_( //
|
|
2823
|
+
metric_at&& metric, std::size_t new_slot, value_at&& value, level_t level,
|
|
2824
|
+
context_t& context) usearch_noexcept_m {
|
|
3072
2825
|
|
|
3073
|
-
node_t new_node =
|
|
2826
|
+
node_t new_node = node_at_(new_slot);
|
|
3074
2827
|
top_candidates_t& top = context.top_candidates;
|
|
3075
2828
|
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
3076
2829
|
|
|
3077
2830
|
// Reverse links from the neighbors:
|
|
3078
|
-
std::size_t const connectivity_max = level ? config_.connectivity :
|
|
3079
|
-
for (
|
|
3080
|
-
|
|
3081
|
-
|
|
2831
|
+
std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
|
|
2832
|
+
for (compressed_slot_t close_slot : new_neighbors) {
|
|
2833
|
+
if (close_slot == new_slot)
|
|
2834
|
+
continue;
|
|
2835
|
+
node_lock_t close_lock = node_lock_(close_slot);
|
|
2836
|
+
node_t close_node = node_at_(close_slot);
|
|
3082
2837
|
|
|
3083
2838
|
neighbors_ref_t close_header = neighbors_(close_node, level);
|
|
3084
2839
|
usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
|
|
3085
|
-
usearch_assert_m(
|
|
2840
|
+
usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
|
|
3086
2841
|
usearch_assert_m(level <= close_node.level(), "Linking to missing level");
|
|
3087
2842
|
|
|
3088
|
-
// If `
|
|
2843
|
+
// If `new_slot` is already present in the neighboring connections of `close_slot`
|
|
3089
2844
|
// then no need to modify any connections or run the heuristics.
|
|
3090
2845
|
if (close_header.size() < connectivity_max) {
|
|
3091
|
-
close_header.push_back(
|
|
2846
|
+
close_header.push_back(static_cast<compressed_slot_t>(new_slot));
|
|
3092
2847
|
continue;
|
|
3093
2848
|
}
|
|
3094
2849
|
|
|
3095
2850
|
// To fit a new connection we need to drop an existing one.
|
|
3096
2851
|
top.clear();
|
|
3097
2852
|
usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
|
|
3098
|
-
top.insert_reserved(
|
|
3099
|
-
|
|
3100
|
-
|
|
2853
|
+
top.insert_reserved(
|
|
2854
|
+
{context.measure(value, citerator_at(close_slot), metric), static_cast<compressed_slot_t>(new_slot)});
|
|
2855
|
+
for (compressed_slot_t successor_slot : close_header)
|
|
2856
|
+
top.insert_reserved(
|
|
2857
|
+
{context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
|
|
3101
2858
|
|
|
3102
2859
|
// Export the results:
|
|
3103
2860
|
close_header.clear();
|
|
3104
|
-
candidates_view_t top_view = refine_(
|
|
2861
|
+
candidates_view_t top_view = refine_(metric, connectivity_max, top, context);
|
|
3105
2862
|
for (std::size_t idx = 0; idx != top_view.size(); idx++)
|
|
3106
|
-
close_header.push_back(top_view[idx].
|
|
2863
|
+
close_header.push_back(top_view[idx].slot);
|
|
3107
2864
|
}
|
|
3108
2865
|
}
|
|
3109
2866
|
|
|
@@ -3113,31 +2870,105 @@ class index_gt {
|
|
|
3113
2870
|
return (level_t)r;
|
|
3114
2871
|
}
|
|
3115
2872
|
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
2873
|
+
struct candidates_range_t;
|
|
2874
|
+
class candidates_iterator_t {
|
|
2875
|
+
friend struct candidates_range_t;
|
|
2876
|
+
|
|
2877
|
+
index_gt const& index_;
|
|
2878
|
+
neighbors_ref_t neighbors_;
|
|
2879
|
+
visits_bitset_t& visits_;
|
|
2880
|
+
std::size_t current_;
|
|
2881
|
+
|
|
2882
|
+
candidates_iterator_t& skip_missing() noexcept {
|
|
2883
|
+
while (current_ != neighbors_.size()) {
|
|
2884
|
+
compressed_slot_t neighbor_slot = neighbors_[current_];
|
|
2885
|
+
if (visits_.test(neighbor_slot))
|
|
2886
|
+
current_++;
|
|
2887
|
+
else
|
|
2888
|
+
break;
|
|
2889
|
+
}
|
|
2890
|
+
return *this;
|
|
2891
|
+
}
|
|
2892
|
+
|
|
2893
|
+
public:
|
|
2894
|
+
using element_t = compressed_slot_t;
|
|
2895
|
+
using iterator_category = std::forward_iterator_tag;
|
|
2896
|
+
using value_type = element_t;
|
|
2897
|
+
using difference_type = std::ptrdiff_t;
|
|
2898
|
+
using pointer = misaligned_ptr_gt<element_t>;
|
|
2899
|
+
using reference = misaligned_ref_gt<element_t>;
|
|
2900
|
+
|
|
2901
|
+
reference operator*() const noexcept { return slot(); }
|
|
2902
|
+
candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_bitset_t& visits,
|
|
2903
|
+
std::size_t progress) noexcept
|
|
2904
|
+
: index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
|
|
2905
|
+
candidates_iterator_t operator++(int) noexcept {
|
|
2906
|
+
return candidates_iterator_t(index_, visits_, neighbors_, current_ + 1).skip_missing();
|
|
2907
|
+
}
|
|
2908
|
+
candidates_iterator_t& operator++() noexcept {
|
|
2909
|
+
++current_;
|
|
2910
|
+
skip_missing();
|
|
2911
|
+
return *this;
|
|
2912
|
+
}
|
|
2913
|
+
bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
|
|
2914
|
+
bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
|
|
2915
|
+
|
|
2916
|
+
key_t key() const noexcept { return index_->node_at_(slot()).key(); }
|
|
2917
|
+
compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
|
|
2918
|
+
friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
|
|
2919
|
+
friend inline key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
|
|
2920
|
+
};
|
|
2921
|
+
|
|
2922
|
+
struct candidates_range_t {
|
|
2923
|
+
index_gt const& index;
|
|
2924
|
+
neighbors_ref_t neighbors;
|
|
2925
|
+
visits_bitset_t& visits;
|
|
2926
|
+
|
|
2927
|
+
candidates_iterator_t begin() const noexcept {
|
|
2928
|
+
return candidates_iterator_t{index, neighbors, visits, 0}.skip_missing();
|
|
2929
|
+
}
|
|
2930
|
+
candidates_iterator_t end() const noexcept { return {index, neighbors, visits, neighbors.size()}; }
|
|
2931
|
+
};
|
|
2932
|
+
|
|
2933
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
2934
|
+
std::size_t search_for_one_( //
|
|
2935
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
2936
|
+
std::size_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
|
|
2937
|
+
|
|
2938
|
+
visits_bitset_t& visits = context.visits;
|
|
2939
|
+
visits.clear();
|
|
3120
2940
|
|
|
3121
|
-
|
|
2941
|
+
// Optional prefetching
|
|
2942
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
2943
|
+
prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
|
|
2944
|
+
|
|
2945
|
+
distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
|
|
3122
2946
|
for (level_t level = begin_level; level > end_level; --level) {
|
|
3123
2947
|
bool changed;
|
|
3124
2948
|
do {
|
|
3125
2949
|
changed = false;
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
2950
|
+
node_lock_t closest_lock = node_lock_(closest_slot);
|
|
2951
|
+
neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
|
|
2952
|
+
|
|
2953
|
+
// Optional prefetching
|
|
2954
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
2955
|
+
candidates_range_t missing_candidates{*this, closest_neighbors, visits};
|
|
2956
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
2957
|
+
}
|
|
2958
|
+
|
|
2959
|
+
// Actual traversal
|
|
2960
|
+
for (compressed_slot_t candidate_slot : closest_neighbors) {
|
|
2961
|
+
distance_t candidate_dist = context.measure(query, citerator_at(candidate_slot), metric);
|
|
3131
2962
|
if (candidate_dist < closest_dist) {
|
|
3132
2963
|
closest_dist = candidate_dist;
|
|
3133
|
-
|
|
2964
|
+
closest_slot = candidate_slot;
|
|
3134
2965
|
changed = true;
|
|
3135
2966
|
}
|
|
3136
2967
|
}
|
|
3137
2968
|
context.iteration_cycles++;
|
|
3138
2969
|
} while (changed);
|
|
3139
2970
|
}
|
|
3140
|
-
return
|
|
2971
|
+
return closest_slot;
|
|
3141
2972
|
}
|
|
3142
2973
|
|
|
3143
2974
|
/**
|
|
@@ -3145,8 +2976,11 @@ class index_gt {
|
|
|
3145
2976
|
* Locks the nodes in the process, assuming other threads are updating neighbors lists.
|
|
3146
2977
|
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3147
2978
|
*/
|
|
3148
|
-
|
|
3149
|
-
|
|
2979
|
+
template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
|
|
2980
|
+
bool search_to_insert_( //
|
|
2981
|
+
value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
|
|
2982
|
+
std::size_t start_slot, std::size_t new_slot, level_t level, std::size_t top_limit,
|
|
2983
|
+
context_t& context) noexcept {
|
|
3150
2984
|
|
|
3151
2985
|
visits_bitset_t& visits = context.visits;
|
|
3152
2986
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
@@ -3156,10 +2990,14 @@ class index_gt {
|
|
|
3156
2990
|
next.clear();
|
|
3157
2991
|
top.clear();
|
|
3158
2992
|
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
2993
|
+
// Optional prefetching
|
|
2994
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
2995
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
2996
|
+
|
|
2997
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
2998
|
+
next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
|
|
2999
|
+
top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3000
|
+
visits.set(start_slot);
|
|
3163
3001
|
|
|
3164
3002
|
while (!next.empty()) {
|
|
3165
3003
|
|
|
@@ -3170,24 +3008,30 @@ class index_gt {
|
|
|
3170
3008
|
next.pop();
|
|
3171
3009
|
context.iteration_cycles++;
|
|
3172
3010
|
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3011
|
+
compressed_slot_t candidate_slot = candidacy.slot;
|
|
3012
|
+
if (new_slot == candidate_slot)
|
|
3013
|
+
continue;
|
|
3014
|
+
node_t candidate_ref = node_at_(candidate_slot);
|
|
3015
|
+
node_lock_t candidate_lock = node_lock_(candidate_slot);
|
|
3176
3016
|
neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
|
|
3177
3017
|
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3018
|
+
// Optional prefetching
|
|
3019
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3020
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3021
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3022
|
+
}
|
|
3182
3023
|
|
|
3183
|
-
|
|
3184
|
-
|
|
3024
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
3025
|
+
if (visits.set(successor_slot))
|
|
3026
|
+
continue;
|
|
3185
3027
|
|
|
3028
|
+
// node_lock_t successor_lock = node_lock_(successor_slot);
|
|
3029
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3186
3030
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
3187
3031
|
// This can substantially grow our priority queue:
|
|
3188
|
-
next.insert({-successor_dist,
|
|
3032
|
+
next.insert({-successor_dist, successor_slot});
|
|
3189
3033
|
// This will automatically evict poor matches:
|
|
3190
|
-
top.insert({successor_dist,
|
|
3034
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
3191
3035
|
radius = top.top().distance;
|
|
3192
3036
|
}
|
|
3193
3037
|
}
|
|
@@ -3200,10 +3044,10 @@ class index_gt {
|
|
|
3200
3044
|
* Doesn't lock any nodes, assuming read-only simultaneous access.
|
|
3201
3045
|
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
3202
3046
|
*/
|
|
3203
|
-
template <typename predicate_at>
|
|
3204
|
-
bool search_to_find_in_base_(
|
|
3205
|
-
|
|
3206
|
-
|
|
3047
|
+
template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
|
|
3048
|
+
bool search_to_find_in_base_( //
|
|
3049
|
+
value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
|
|
3050
|
+
std::size_t start_slot, std::size_t expansion, context_t& context) const noexcept {
|
|
3207
3051
|
|
|
3208
3052
|
visits_bitset_t& visits = context.visits;
|
|
3209
3053
|
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
@@ -3214,10 +3058,14 @@ class index_gt {
|
|
|
3214
3058
|
next.clear();
|
|
3215
3059
|
top.clear();
|
|
3216
3060
|
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3061
|
+
// Optional prefetching
|
|
3062
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
|
|
3063
|
+
prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
|
|
3064
|
+
|
|
3065
|
+
distance_t radius = context.measure(query, citerator_at(start_slot), metric);
|
|
3066
|
+
next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3067
|
+
top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
|
|
3068
|
+
visits.set(start_slot);
|
|
3221
3069
|
|
|
3222
3070
|
while (!next.empty()) {
|
|
3223
3071
|
|
|
@@ -3228,28 +3076,29 @@ class index_gt {
|
|
|
3228
3076
|
next.pop();
|
|
3229
3077
|
context.iteration_cycles++;
|
|
3230
3078
|
|
|
3231
|
-
|
|
3232
|
-
neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
|
|
3079
|
+
neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
|
|
3233
3080
|
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3081
|
+
// Optional prefetching
|
|
3082
|
+
if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
|
|
3083
|
+
candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
|
|
3084
|
+
prefetch(missing_candidates.begin(), missing_candidates.end());
|
|
3085
|
+
}
|
|
3238
3086
|
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3087
|
+
for (compressed_slot_t successor_slot : candidate_neighbors) {
|
|
3088
|
+
if (visits.set(successor_slot))
|
|
3089
|
+
continue;
|
|
3242
3090
|
|
|
3091
|
+
distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
|
|
3243
3092
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
3244
3093
|
// This can substantially grow our priority queue:
|
|
3245
|
-
next.insert({-successor_dist,
|
|
3246
|
-
if (
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3094
|
+
next.insert({-successor_dist, successor_slot});
|
|
3095
|
+
if (!is_dummy<predicate_at>())
|
|
3096
|
+
if (!predicate(member_cref_t{node_at_(successor_slot).ckey(), successor_slot}))
|
|
3097
|
+
continue;
|
|
3098
|
+
|
|
3099
|
+
// This will automatically evict poor matches:
|
|
3100
|
+
top.insert({successor_dist, successor_slot}, top_limit);
|
|
3101
|
+
radius = top.top().distance;
|
|
3253
3102
|
}
|
|
3254
3103
|
}
|
|
3255
3104
|
}
|
|
@@ -3258,33 +3107,35 @@ class index_gt {
|
|
|
3258
3107
|
}
|
|
3259
3108
|
|
|
3260
3109
|
/**
|
|
3261
|
-
* @brief Iterates through all
|
|
3110
|
+
* @brief Iterates through all members, without actually touching the index.
|
|
3262
3111
|
*/
|
|
3263
|
-
template <typename predicate_at>
|
|
3264
|
-
void search_exact_(
|
|
3265
|
-
|
|
3266
|
-
|
|
3112
|
+
template <typename value_at, typename metric_at, typename predicate_at>
|
|
3113
|
+
void search_exact_( //
|
|
3114
|
+
value_at&& query, metric_at&& metric, predicate_at&& predicate, //
|
|
3115
|
+
std::size_t count, context_t& context) const noexcept {
|
|
3267
3116
|
|
|
3268
3117
|
top_candidates_t& top = context.top_candidates;
|
|
3269
3118
|
top.clear();
|
|
3270
3119
|
top.reserve(count);
|
|
3271
3120
|
for (std::size_t i = 0; i != size(); ++i) {
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3121
|
+
if (!is_dummy<predicate_at>())
|
|
3122
|
+
if (!predicate(at(i)))
|
|
3123
|
+
continue;
|
|
3124
|
+
|
|
3125
|
+
distance_t distance = context.measure(query, citerator_at(i), metric);
|
|
3126
|
+
top.insert(candidate_t{distance, static_cast<compressed_slot_t>(i)}, count);
|
|
3277
3127
|
}
|
|
3278
3128
|
}
|
|
3279
3129
|
|
|
3280
|
-
void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
|
|
3281
|
-
|
|
3282
3130
|
/**
|
|
3283
3131
|
* @brief This algorithm from the original paper implements a heuristic,
|
|
3284
3132
|
* that massively reduces the number of connections a point has,
|
|
3285
3133
|
* to keep only the neighbors, that are from each other.
|
|
3286
3134
|
*/
|
|
3287
|
-
|
|
3135
|
+
template <typename metric_at>
|
|
3136
|
+
candidates_view_t refine_( //
|
|
3137
|
+
metric_at&& metric, //
|
|
3138
|
+
std::size_t needed, top_candidates_t& top, context_t& context) const noexcept {
|
|
3288
3139
|
|
|
3289
3140
|
top.sort_ascending();
|
|
3290
3141
|
candidate_t* top_data = top.data();
|
|
@@ -3296,14 +3147,14 @@ class index_gt {
|
|
|
3296
3147
|
std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
|
|
3297
3148
|
while (submitted_count < needed && consumed_count < top_count) {
|
|
3298
3149
|
candidate_t candidate = top_data[consumed_count];
|
|
3299
|
-
node_t candidate_ref = node_with_id_(candidate.id);
|
|
3300
|
-
distance_t candidate_dist = candidate.distance;
|
|
3301
3150
|
bool good = true;
|
|
3302
3151
|
for (std::size_t idx = 0; idx < submitted_count; idx++) {
|
|
3303
3152
|
candidate_t submitted = top_data[idx];
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3153
|
+
distance_t inter_result_dist = context.measure( //
|
|
3154
|
+
citerator_at(candidate.slot), //
|
|
3155
|
+
citerator_at(submitted.slot), //
|
|
3156
|
+
metric);
|
|
3157
|
+
if (inter_result_dist < candidate.distance) {
|
|
3307
3158
|
good = false;
|
|
3308
3159
|
break;
|
|
3309
3160
|
}
|
|
@@ -3321,44 +3172,218 @@ class index_gt {
|
|
|
3321
3172
|
}
|
|
3322
3173
|
};
|
|
3323
3174
|
|
|
3175
|
+
struct join_result_t {
|
|
3176
|
+
error_t error{};
|
|
3177
|
+
std::size_t intersection_size{};
|
|
3178
|
+
std::size_t engagements{};
|
|
3179
|
+
std::size_t visited_members{};
|
|
3180
|
+
std::size_t computed_distances{};
|
|
3181
|
+
|
|
3182
|
+
explicit operator bool() const noexcept { return !error; }
|
|
3183
|
+
join_result_t failed(error_t message) noexcept {
|
|
3184
|
+
error = std::move(message);
|
|
3185
|
+
return std::move(*this);
|
|
3186
|
+
}
|
|
3187
|
+
};
|
|
3188
|
+
|
|
3324
3189
|
/**
|
|
3325
|
-
* @brief
|
|
3326
|
-
*
|
|
3190
|
+
* @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
|
|
3191
|
+
* to perform fast one-to-one matching between two large collections
|
|
3192
|
+
* of vectors, using approximate nearest neighbors search.
|
|
3193
|
+
*
|
|
3194
|
+
* @param[inout] man_to_woman Container to map ::first keys to ::second.
|
|
3195
|
+
* @param[inout] woman_to_man Container to map ::second keys to ::first.
|
|
3196
|
+
* @param[in] executor Thread-pool to execute the job in parallel.
|
|
3197
|
+
* @param[in] progress Callback to report the execution progress.
|
|
3327
3198
|
*/
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3199
|
+
template < //
|
|
3200
|
+
|
|
3201
|
+
typename men_at, //
|
|
3202
|
+
typename women_at, //
|
|
3203
|
+
typename men_values_at, //
|
|
3204
|
+
typename women_values_at, //
|
|
3205
|
+
typename men_metric_at, //
|
|
3206
|
+
typename women_metric_at, //
|
|
3207
|
+
|
|
3208
|
+
typename man_to_woman_at = dummy_label_to_label_mapping_t, //
|
|
3209
|
+
typename woman_to_man_at = dummy_label_to_label_mapping_t, //
|
|
3210
|
+
typename executor_at = dummy_executor_t, //
|
|
3211
|
+
typename progress_at = dummy_progress_t //
|
|
3212
|
+
>
|
|
3213
|
+
static join_result_t join( //
|
|
3214
|
+
men_at const& men, //
|
|
3215
|
+
women_at const& women, //
|
|
3216
|
+
men_values_at const& men_values, //
|
|
3217
|
+
women_values_at const& women_values, //
|
|
3218
|
+
men_metric_at&& men_metric, //
|
|
3219
|
+
women_metric_at&& women_metric, //
|
|
3220
|
+
|
|
3221
|
+
index_join_config_t config = {}, //
|
|
3222
|
+
man_to_woman_at&& man_to_woman = man_to_woman_at{}, //
|
|
3223
|
+
woman_to_man_at&& woman_to_man = woman_to_man_at{}, //
|
|
3224
|
+
executor_at&& executor = executor_at{}, //
|
|
3225
|
+
progress_at&& progress = progress_at{}) noexcept {
|
|
3226
|
+
|
|
3227
|
+
if (women.size() < men.size())
|
|
3228
|
+
return unum::usearch::join( //
|
|
3229
|
+
women, men, //
|
|
3230
|
+
women_values, men_values, //
|
|
3231
|
+
std::forward<women_metric_at>(women_metric), std::forward<men_metric_at>(men_metric), //
|
|
3232
|
+
|
|
3233
|
+
config, //
|
|
3234
|
+
std::forward<woman_to_man_at>(woman_to_man), //
|
|
3235
|
+
std::forward<man_to_woman_at>(man_to_woman), //
|
|
3236
|
+
std::forward<executor_at>(executor), //
|
|
3237
|
+
std::forward<progress_at>(progress));
|
|
3238
|
+
|
|
3239
|
+
join_result_t result;
|
|
3240
|
+
|
|
3241
|
+
// Sanity checks and argument validation:
|
|
3242
|
+
if (&men == &women)
|
|
3243
|
+
return result.failed("Can't join with itself, consider copying");
|
|
3244
|
+
|
|
3245
|
+
if (config.max_proposals == 0)
|
|
3246
|
+
config.max_proposals = std::log(men.size()) + executor.size();
|
|
3247
|
+
|
|
3248
|
+
using proposals_count_t = std::uint16_t;
|
|
3249
|
+
config.max_proposals = (std::min)(men.size(), config.max_proposals);
|
|
3250
|
+
|
|
3251
|
+
using distance_t = typename men_at::distance_t;
|
|
3252
|
+
using dynamic_allocator_traits_t = typename men_at::dynamic_allocator_traits_t;
|
|
3253
|
+
using man_key_t = typename men_at::key_t;
|
|
3254
|
+
using woman_key_t = typename women_at::key_t;
|
|
3255
|
+
|
|
3256
|
+
// Use the `compressed_slot_t` type of the larger collection
|
|
3257
|
+
using compressed_slot_t = typename women_at::compressed_slot_t;
|
|
3258
|
+
using compressed_slot_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<compressed_slot_t>;
|
|
3259
|
+
using proposals_count_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<proposals_count_t>;
|
|
3260
|
+
|
|
3261
|
+
// Create an atomic queue, as a ring structure, from/to which
|
|
3262
|
+
// free men will be added/pulled.
|
|
3263
|
+
std::mutex free_men_mutex{};
|
|
3264
|
+
ring_gt<compressed_slot_t, compressed_slot_allocator_t> free_men;
|
|
3265
|
+
free_men.reserve(men.size());
|
|
3266
|
+
for (std::size_t i = 0; i != men.size(); ++i)
|
|
3267
|
+
free_men.push(static_cast<compressed_slot_t>(i));
|
|
3268
|
+
|
|
3269
|
+
// We are gonna need some temporary memory.
|
|
3270
|
+
buffer_gt<proposals_count_t, proposals_count_allocator_t> proposal_counts(men.size());
|
|
3271
|
+
buffer_gt<compressed_slot_t, compressed_slot_allocator_t> man_to_woman_slots(men.size());
|
|
3272
|
+
buffer_gt<compressed_slot_t, compressed_slot_allocator_t> woman_to_man_slots(women.size());
|
|
3273
|
+
if (!proposal_counts || !man_to_woman_slots || !woman_to_man_slots)
|
|
3274
|
+
return result.failed("Can't temporary mappings");
|
|
3275
|
+
|
|
3276
|
+
compressed_slot_t missing_slot;
|
|
3277
|
+
std::memset((void*)&missing_slot, 0xFF, sizeof(compressed_slot_t));
|
|
3278
|
+
std::memset((void*)man_to_woman_slots.data(), 0xFF, sizeof(compressed_slot_t) * men.size());
|
|
3279
|
+
std::memset((void*)woman_to_man_slots.data(), 0xFF, sizeof(compressed_slot_t) * women.size());
|
|
3280
|
+
std::memset(proposal_counts.data(), 0, sizeof(proposals_count_t) * men.size());
|
|
3281
|
+
|
|
3282
|
+
// Define locks, to limit concurrent accesses to `man_to_woman_slots` and `woman_to_man_slots`.
|
|
3283
|
+
visits_bitset_t men_locks(men.size()), women_locks(women.size());
|
|
3284
|
+
if (!men_locks || !women_locks)
|
|
3285
|
+
return result.failed("Can't allocate locks");
|
|
3286
|
+
|
|
3287
|
+
std::atomic<std::size_t> rounds{0};
|
|
3288
|
+
std::atomic<std::size_t> engagements{0};
|
|
3289
|
+
std::atomic<std::size_t> computed_distances{0};
|
|
3290
|
+
std::atomic<std::size_t> visited_members{0};
|
|
3291
|
+
|
|
3292
|
+
// Concurrently process all the men
|
|
3293
|
+
executor.execute_bulk([&](std::size_t thread_idx) {
|
|
3294
|
+
index_search_config_t search_config;
|
|
3295
|
+
search_config.expansion = config.expansion;
|
|
3296
|
+
search_config.exact = config.exact;
|
|
3297
|
+
search_config.thread = thread_idx;
|
|
3298
|
+
compressed_slot_t free_man_slot;
|
|
3299
|
+
|
|
3300
|
+
// While there exist a free man who still has a woman to propose to.
|
|
3301
|
+
while (true) {
|
|
3302
|
+
std::size_t passed_rounds = 0;
|
|
3303
|
+
std::size_t total_rounds = 0;
|
|
3304
|
+
{
|
|
3305
|
+
std::unique_lock<std::mutex> pop_lock(free_men_mutex);
|
|
3306
|
+
if (!free_men.try_pop(free_man_slot))
|
|
3307
|
+
// Primary exit path, we have exhausted the list of candidates
|
|
3308
|
+
break;
|
|
3309
|
+
passed_rounds = ++rounds;
|
|
3310
|
+
total_rounds = passed_rounds + free_men.size();
|
|
3311
|
+
}
|
|
3312
|
+
progress(passed_rounds, total_rounds);
|
|
3313
|
+
while (men_locks.atomic_set(free_man_slot))
|
|
3314
|
+
;
|
|
3315
|
+
|
|
3316
|
+
proposals_count_t& free_man_proposals = proposal_counts[free_man_slot];
|
|
3317
|
+
if (free_man_proposals >= config.max_proposals)
|
|
3318
|
+
continue;
|
|
3319
|
+
|
|
3320
|
+
// Find the closest woman, to whom this man hasn't proposed yet.
|
|
3321
|
+
++free_man_proposals;
|
|
3322
|
+
auto candidates = women.search(men_values[free_man_slot], free_man_proposals, women_metric, search_config);
|
|
3323
|
+
visited_members += candidates.visited_members;
|
|
3324
|
+
computed_distances += candidates.computed_distances;
|
|
3325
|
+
if (!candidates) {
|
|
3326
|
+
// TODO:
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
auto match = candidates.back();
|
|
3330
|
+
auto woman = match.member;
|
|
3331
|
+
while (women_locks.atomic_set(woman.slot))
|
|
3332
|
+
;
|
|
3333
|
+
|
|
3334
|
+
compressed_slot_t husband_slot = woman_to_man_slots[woman.slot];
|
|
3335
|
+
bool woman_is_free = husband_slot == missing_slot;
|
|
3336
|
+
if (woman_is_free) {
|
|
3337
|
+
// Engagement
|
|
3338
|
+
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
3339
|
+
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3340
|
+
engagements++;
|
|
3341
|
+
} else {
|
|
3342
|
+
distance_t distance_from_husband = women_metric(women_values[woman.slot], men_values[husband_slot]);
|
|
3343
|
+
distance_t distance_from_candidate = match.distance;
|
|
3344
|
+
if (distance_from_husband > distance_from_candidate) {
|
|
3345
|
+
// Break-up
|
|
3346
|
+
while (men_locks.atomic_set(husband_slot))
|
|
3347
|
+
;
|
|
3348
|
+
man_to_woman_slots[husband_slot] = missing_slot;
|
|
3349
|
+
men_locks.atomic_reset(husband_slot);
|
|
3350
|
+
|
|
3351
|
+
// New Engagement
|
|
3352
|
+
man_to_woman_slots[free_man_slot] = woman.slot;
|
|
3353
|
+
woman_to_man_slots[woman.slot] = free_man_slot;
|
|
3354
|
+
engagements++;
|
|
3355
|
+
|
|
3356
|
+
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
3357
|
+
free_men.push(husband_slot);
|
|
3358
|
+
} else {
|
|
3359
|
+
std::unique_lock<std::mutex> push_lock(free_men_mutex);
|
|
3360
|
+
free_men.push(free_man_slot);
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
|
|
3364
|
+
men_locks.atomic_reset(free_man_slot);
|
|
3365
|
+
women_locks.atomic_reset(woman.slot);
|
|
3366
|
+
}
|
|
3367
|
+
});
|
|
3368
|
+
|
|
3369
|
+
// Export the "slots" into keys:
|
|
3370
|
+
std::size_t intersection_size = 0;
|
|
3371
|
+
for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
|
|
3372
|
+
compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
|
|
3373
|
+
if (woman_slot != missing_slot) {
|
|
3374
|
+
man_key_t man = men.at(man_slot).key;
|
|
3375
|
+
woman_key_t woman = women.at(woman_slot).key;
|
|
3376
|
+
man_to_woman[man] = woman;
|
|
3377
|
+
woman_to_man[woman] = man;
|
|
3378
|
+
intersection_size++;
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
|
|
3382
|
+
// Export stats
|
|
3383
|
+
result.engagements = engagements;
|
|
3384
|
+
result.intersection_size = intersection_size;
|
|
3385
|
+
result.computed_distances = computed_distances;
|
|
3386
|
+
result.visited_members = visited_members;
|
|
3362
3387
|
return result;
|
|
3363
3388
|
}
|
|
3364
3389
|
|