usearch 0.22.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/.github/workflows/prerelease.yml +3 -3
  2. package/.releaserc +9 -0
  3. package/.vscode/launch.json +15 -11
  4. package/.vscode/settings.json +12 -2
  5. package/.vscode/tasks.json +4 -4
  6. package/CMakeLists.txt +6 -1
  7. package/Cargo.toml +2 -2
  8. package/README.md +39 -46
  9. package/build.rs +2 -2
  10. package/c/CMakeLists.txt +48 -0
  11. package/c/README.md +34 -0
  12. package/c/lib.cpp +83 -72
  13. package/c/test.c +20 -20
  14. package/c/usearch.h +132 -32
  15. package/conanfile.py +1 -1
  16. package/cpp/CMakeLists.txt +3 -1
  17. package/cpp/README.md +16 -16
  18. package/cpp/bench.cpp +50 -126
  19. package/cpp/test.cpp +164 -77
  20. package/docs/benchmarks.md +26 -5
  21. package/docs/c/reference.rst +3 -3
  22. package/docs/compilation.md +49 -45
  23. package/docs/cpp/index.rst +2 -2
  24. package/docs/cpp/reference.rst +13 -3
  25. package/docs/index.rst +8 -8
  26. package/docs/java/reference.rst +1 -1
  27. package/golang/lib.go +275 -141
  28. package/golang/lib_test.go +105 -0
  29. package/include/usearch/index.hpp +1733 -1659
  30. package/include/usearch/index_dense.hpp +1400 -0
  31. package/include/usearch/index_plugins.hpp +1263 -0
  32. package/java/README.md +1 -1
  33. package/java/cloud/unum/usearch/Index.java +10 -10
  34. package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
  35. package/java/test/IndexTest.java +1 -1
  36. package/javascript/README.md +7 -7
  37. package/javascript/lib.cpp +94 -55
  38. package/javascript/test.js +25 -11
  39. package/javascript/usearch.d.ts +87 -0
  40. package/javascript/usearch.js +2 -118
  41. package/objc/USearchObjective.mm +57 -31
  42. package/objc/include/USearchObjective.h +32 -22
  43. package/package.json +5 -3
  44. package/pyproject.toml +3 -3
  45. package/python/README.md +23 -14
  46. package/python/lib.cpp +472 -444
  47. package/python/scripts/bench.ipynb +3 -2
  48. package/python/scripts/bench.py +4 -4
  49. package/python/scripts/index_faiss.py +5 -5
  50. package/python/scripts/join.py +4 -4
  51. package/python/scripts/test.py +152 -61
  52. package/python/scripts/test_jit.py +10 -12
  53. package/python/scripts/test_sparse.py +32 -0
  54. package/python/usearch/client.py +26 -28
  55. package/python/usearch/eval.py +73 -23
  56. package/python/usearch/index.py +520 -228
  57. package/python/usearch/io.py +57 -25
  58. package/python/usearch/numba.py +4 -4
  59. package/python/usearch/server.py +10 -10
  60. package/rust/lib.cpp +22 -27
  61. package/rust/lib.hpp +11 -12
  62. package/rust/lib.rs +3 -3
  63. package/setup.py +5 -2
  64. package/simsimd/.github/workflows/prerelease.yml +73 -0
  65. package/simsimd/.github/workflows/release.yml +110 -0
  66. package/simsimd/.github/workflows/update_version.sh +3 -0
  67. package/simsimd/.releaserc +77 -0
  68. package/simsimd/.vscode/launch.json +27 -1
  69. package/simsimd/.vscode/settings.json +22 -1
  70. package/simsimd/.vscode/tasks.json +6 -0
  71. package/simsimd/CMakeLists.txt +3 -2
  72. package/simsimd/README.md +82 -46
  73. package/simsimd/bench.cxx +63 -24
  74. package/simsimd/include/simsimd/simsimd.h +88 -66
  75. package/simsimd/include/simsimd/simsimd_chem.h +126 -0
  76. package/simsimd/package.json +8 -0
  77. package/simsimd/pyproject.toml +42 -0
  78. package/simsimd/python/python.c +81 -0
  79. package/simsimd/python/run.sh +1 -0
  80. package/simsimd/python/test.c +109 -0
  81. package/simsimd/python/test.py +23 -0
  82. package/simsimd/setup.py +44 -0
  83. package/swift/Index+Sugar.swift +75 -23
  84. package/swift/README.md +26 -2
  85. package/swift/Test.swift +19 -7
  86. package/swift/USearch.swift +2 -2
  87. package/tsconfig.json +8 -0
  88. package/wasm/CMakeLists.txt +1 -0
  89. package/wasmer.toml +5 -1
  90. package/wolfram/CMakeLists.txt +1 -0
  91. package/wolfram/lib.cpp +11 -13
  92. package/golang/usearch.h +0 -9
  93. package/golang/usearch_par_test.go +0 -40
  94. package/golang/usearch_test.go +0 -82
  95. package/include/usearch/index_punned_dense.hpp +0 -871
  96. package/include/usearch/index_punned_helpers.hpp +0 -566
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @file index.hpp
3
- * @author Ashot Vardanian
3
+ * @author Ash Vardanian
4
4
  * @brief Single-header Vector Search.
5
5
  * @date 2023-04-26
6
6
  *
@@ -14,6 +14,7 @@
14
14
  #define USEARCH_VERSION_PATCH 0
15
15
 
16
16
  // Inferring C++ version
17
+ // https://stackoverflow.com/a/61552074
17
18
  #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
18
19
  #define USEARCH_DEFINED_CPP17
19
20
  #endif
@@ -42,10 +43,13 @@
42
43
  #endif
43
44
 
44
45
  // Inferring hardware bitness: 32 vs 64
45
- #if __WORDSIZE == 64
46
+ // https://stackoverflow.com/a/5273354
47
+ #if INTPTR_MAX == INT64_MAX
46
48
  #define USEARCH_64BIT_ENV
47
- #else
49
+ #elif INTPTR_MAX == INT32_MAX
48
50
  #define USEARCH_32BIT_ENV
51
+ #else
52
+ #error Unknown pointer size or missing size macros!
49
53
  #endif
50
54
 
51
55
  #if !defined(USEARCH_USE_OPENMP)
@@ -119,51 +123,7 @@
119
123
  namespace unum {
120
124
  namespace usearch {
121
125
 
122
- using f64_t = double;
123
- using f32_t = float;
124
126
  using byte_t = char;
125
- enum b1x8_t : unsigned char {};
126
-
127
- enum class metric_kind_t : std::uint8_t {
128
- unknown_k = 0,
129
- // Classics:
130
- ip_k = 'i',
131
- cos_k = 'c',
132
- l2sq_k = 'e',
133
-
134
- // Custom:
135
- pearson_k = 'p',
136
- haversine_k = 'h',
137
-
138
- // Sets:
139
- jaccard_k = 'j',
140
- hamming_k = 'b',
141
- tanimoto_k = 't',
142
- sorensen_k = 's',
143
- };
144
-
145
- enum class scalar_kind_t : std::uint8_t {
146
- unknown_k = 0,
147
- f64_k,
148
- f32_k,
149
- f16_k,
150
- f8_k,
151
- b1x8_k,
152
- };
153
-
154
- template <typename scalar_at> scalar_kind_t common_scalar_kind() noexcept {
155
- if (std::is_same<scalar_at, f32_t>())
156
- return scalar_kind_t::f32_k;
157
- if (std::is_same<scalar_at, f64_t>())
158
- return scalar_kind_t::f64_k;
159
- if (std::is_same<scalar_at, b1x8_t>())
160
- return scalar_kind_t::b1x8_k;
161
- return scalar_kind_t::unknown_k;
162
- }
163
-
164
- template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
165
-
166
- template <typename at> at square(at value) noexcept { return value * value; }
167
127
 
168
128
  template <std::size_t multiple_ak> std::size_t divide_round_up(std::size_t num) noexcept {
169
129
  return (num + multiple_ak - 1) / multiple_ak;
@@ -187,6 +147,7 @@ inline std::size_t ceil2(std::size_t v) noexcept {
187
147
  return v;
188
148
  }
189
149
 
150
+ /// @brief Simply dereferencing misaligned pointers can be dangerous.
190
151
  template <typename at> void misaligned_store(void* ptr, at v) noexcept {
191
152
  static_assert(!std::is_reference<at>::value, "Can't store a reference");
192
153
  std::memcpy(ptr, &v, sizeof(at));
@@ -207,54 +168,80 @@ template <typename at, typename other_at = at> at exchange(at& obj, other_at&& n
207
168
  return old_value;
208
169
  }
209
170
 
171
+ /// @brief The `std::destroy_at` alternative for C++11.
172
+ template <typename at, typename sfinae_at = at>
173
+ typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
174
+ template <typename at, typename sfinae_at = at>
175
+ typename std::enable_if<!std::is_pod<sfinae_at>::value>::type destroy_at(at* obj) {
176
+ obj->~sfinae_at();
177
+ }
178
+
179
+ /// @brief The `std::construct_at` alternative for C++11.
180
+ template <typename at, typename sfinae_at = at>
181
+ typename std::enable_if<std::is_pod<sfinae_at>::value>::type construct_at(at*) {}
182
+ template <typename at, typename sfinae_at = at>
183
+ typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* obj) {
184
+ new (obj) at();
185
+ }
186
+
187
+ /**
188
+ * @brief A reference to a misaligned memory location with a specific type.
189
+ * It is needed to avoid Undefined Behavior when dereferencing addresses
190
+ * indivisible by `sizeof(at)`.
191
+ */
210
192
  template <typename at> class misaligned_ref_gt {
193
+ using element_t = at;
194
+ using mutable_t = typename std::remove_const<element_t>::type;
211
195
  byte_t* ptr_;
212
196
 
213
197
  public:
214
198
  misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
215
- operator at() const noexcept { return misaligned_load<at>(ptr_); }
216
- misaligned_ref_gt& operator=(at const& v) noexcept {
217
- misaligned_store<at>(ptr_, v);
199
+ operator mutable_t() const noexcept { return misaligned_load<mutable_t>(ptr_); }
200
+ misaligned_ref_gt& operator=(mutable_t const& v) noexcept {
201
+ misaligned_store<mutable_t>(ptr_, v);
218
202
  return *this;
219
203
  }
220
204
 
221
205
  void reset(byte_t* ptr) noexcept { ptr_ = ptr; }
206
+ byte_t* ptr() const noexcept { return ptr_; }
222
207
  };
223
208
 
209
+ /**
210
+ * @brief A pointer to a misaligned memory location with a specific type.
211
+ * It is needed to avoid Undefined Behavior when dereferencing addresses
212
+ * indivisible by `sizeof(at)`.
213
+ */
224
214
  template <typename at> class misaligned_ptr_gt {
215
+ using element_t = at;
216
+ using mutable_t = typename std::remove_const<element_t>::type;
225
217
  byte_t* ptr_;
226
218
 
227
219
  public:
228
220
  using iterator_category = std::random_access_iterator_tag;
229
- using value_type = at;
221
+ using value_type = element_t;
230
222
  using difference_type = std::ptrdiff_t;
231
- using pointer = misaligned_ptr_gt<at>;
232
- using reference = misaligned_ref_gt<at>;
223
+ using pointer = misaligned_ptr_gt<element_t>;
224
+ using reference = misaligned_ref_gt<element_t>;
233
225
 
234
226
  reference operator*() const noexcept { return {ptr_}; }
227
+ reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
228
+ value_type operator[](std::size_t i) const noexcept {
229
+ return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
230
+ }
235
231
 
236
232
  misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
237
- misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(at)); }
238
- misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(at)); }
239
- misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(at)); }
240
- misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(at)); }
233
+ misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(element_t)); }
234
+ misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(element_t)); }
235
+ misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(element_t)); }
236
+ misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(element_t)); }
237
+
238
+ // clang-format off
239
+ misaligned_ptr_gt& operator++() noexcept { ptr_ += sizeof(element_t); return *this; }
240
+ misaligned_ptr_gt& operator--() noexcept { ptr_ -= sizeof(element_t); return *this; }
241
+ misaligned_ptr_gt& operator+=(difference_type d) noexcept { ptr_ += d * sizeof(element_t); return *this; }
242
+ misaligned_ptr_gt& operator-=(difference_type d) noexcept { ptr_ -= d * sizeof(element_t); return *this; }
243
+ // clang-format on
241
244
 
242
- misaligned_ptr_gt& operator++() noexcept {
243
- ptr_ += sizeof(at);
244
- return *this;
245
- }
246
- misaligned_ptr_gt& operator--() noexcept {
247
- ptr_ -= sizeof(at);
248
- return *this;
249
- }
250
- misaligned_ptr_gt& operator+=(difference_type d) noexcept {
251
- ptr_ += d * sizeof(at);
252
- return *this;
253
- }
254
- misaligned_ptr_gt& operator-=(difference_type d) noexcept {
255
- ptr_ -= d * sizeof(at);
256
- return *this;
257
- }
258
245
  bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
259
246
  bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
260
247
  };
@@ -278,307 +265,56 @@ template <typename scalar_at> class span_gt {
278
265
  };
279
266
 
280
267
  /**
281
- * @brief Inner (Dot) Product distance.
282
- */
283
- template <typename scalar_at = float, typename result_at = scalar_at> struct ip_gt {
284
- using scalar_t = scalar_at;
285
- using view_t = span_gt<scalar_t const>;
286
- using result_t = result_at;
287
- using result_type = result_t;
288
-
289
- inline metric_kind_t kind() const noexcept { return metric_kind_t::ip_k; }
290
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
291
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
292
-
293
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
294
- result_type ab{};
295
- #if USEARCH_USE_OPENMP
296
- #pragma omp simd reduction(+ : ab)
297
- #elif defined(USEARCH_DEFINED_CLANG)
298
- #pragma clang loop vectorize(enable)
299
- #elif defined(USEARCH_DEFINED_GCC)
300
- #pragma GCC ivdep
301
- #endif
302
- for (std::size_t i = 0; i != dim; ++i)
303
- ab += result_t(a[i]) * result_t(b[i]);
304
- return 1 - ab;
305
- }
306
- };
307
-
308
- /**
309
- * @brief Cosine (Angular) distance.
310
- * Identical to the Inner Product of normalized vectors.
311
- * Unless you are running on an tiny embedded platform, this metric
312
- * is recommended over `::ip_gt` for low-precision scalars.
313
- */
314
- template <typename scalar_at = float, typename result_at = scalar_at> struct cos_gt {
315
- using scalar_t = scalar_at;
316
- using view_t = span_gt<scalar_t const>;
317
- using result_t = result_at;
318
- using result_type = result_t;
319
-
320
- inline metric_kind_t kind() const noexcept { return metric_kind_t::cos_k; }
321
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
322
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
323
-
324
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
325
- result_t ab{}, a2{}, b2{};
326
- #if USEARCH_USE_OPENMP
327
- #pragma omp simd reduction(+ : ab, a2, b2)
328
- #elif defined(USEARCH_DEFINED_CLANG)
329
- #pragma clang loop vectorize(enable)
330
- #elif defined(USEARCH_DEFINED_GCC)
331
- #pragma GCC ivdep
332
- #endif
333
- for (std::size_t i = 0; i != dim; ++i)
334
- ab += result_t(a[i]) * result_t(b[i]), //
335
- a2 += square<result_t>(a[i]), //
336
- b2 += square<result_t>(b[i]);
337
- return (ab != 0) ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 1;
338
- }
339
- };
340
-
341
- /**
342
- * @brief Squared Euclidean (L2) distance.
343
- * Square root is avoided at the end, as it won't affect the ordering.
268
+ * @brief Similar to `std::vector`, but doesn't support dynamic resizing.
269
+ * On the bright side, this can't throw exceptions.
344
270
  */
345
- template <typename scalar_at = float, typename result_at = scalar_at> struct l2sq_gt {
346
- using scalar_t = scalar_at;
347
- using view_t = span_gt<scalar_t const>;
348
- using result_t = result_at;
349
- using result_type = result_t;
350
-
351
- inline metric_kind_t kind() const noexcept { return metric_kind_t::l2sq_k; }
352
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
353
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
354
-
355
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
356
- result_t ab_deltas_sq{};
357
- #if USEARCH_USE_OPENMP
358
- #pragma omp simd reduction(+ : ab_deltas_sq)
359
- #elif defined(USEARCH_DEFINED_CLANG)
360
- #pragma clang loop vectorize(enable)
361
- #elif defined(USEARCH_DEFINED_GCC)
362
- #pragma GCC ivdep
363
- #endif
364
- for (std::size_t i = 0; i != dim; ++i)
365
- ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
366
- return ab_deltas_sq;
367
- }
368
- };
369
-
370
- /**
371
- * @brief Hamming distance computes the number of differing bits in
372
- * two arrays of integers. An example would be a textual document,
373
- * tokenized and hashed into a fixed-capacity bitset.
374
- */
375
- template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct hamming_gt {
376
- using scalar_t = scalar_at;
377
- using view_t = span_gt<scalar_t const>;
378
- using result_t = result_at;
379
- using result_type = result_t;
380
- static_assert( //
381
- std::is_unsigned<scalar_t>::value ||
382
- (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
383
- "Hamming distance requires unsigned integral words");
384
-
385
- inline metric_kind_t kind() const noexcept { return metric_kind_t::hamming_k; }
386
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
387
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
388
-
389
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
390
- constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
391
- result_t matches{};
392
- #if USEARCH_USE_OPENMP
393
- #pragma omp simd reduction(+ : matches)
394
- #elif defined(USEARCH_DEFINED_CLANG)
395
- #pragma clang loop vectorize(enable)
396
- #elif defined(USEARCH_DEFINED_GCC)
397
- #pragma GCC ivdep
398
- #endif
399
- for (std::size_t i = 0; i != words; ++i)
400
- matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
401
- return matches;
402
- }
403
- };
271
+ template <typename scalar_at, typename allocator_at> class buffer_gt {
272
+ scalar_at* data_;
273
+ std::size_t size_;
404
274
 
405
- /**
406
- * @brief Tanimoto distance is the intersection over bitwise union.
407
- * Often used in chemistry and biology to compare molecular fingerprints.
408
- */
409
- template <typename scalar_at = std::uint64_t, typename result_at = float> struct tanimoto_gt {
410
- using scalar_t = scalar_at;
411
- using view_t = span_gt<scalar_t const>;
412
- using result_t = result_at;
413
- using result_type = result_t;
414
- static_assert( //
415
- std::is_unsigned<scalar_t>::value ||
416
- (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
417
- "Tanimoto distance requires unsigned integral words");
418
- static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
419
-
420
- inline metric_kind_t kind() const noexcept { return metric_kind_t::tanimoto_k; }
421
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
422
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
423
-
424
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
425
- constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
426
- result_t and_count{};
427
- result_t or_count{};
428
- #if USEARCH_USE_OPENMP
429
- #pragma omp simd reduction(+ : and_count, or_count)
430
- #elif defined(USEARCH_DEFINED_CLANG)
431
- #pragma clang loop vectorize(enable)
432
- #elif defined(USEARCH_DEFINED_GCC)
433
- #pragma GCC ivdep
434
- #endif
435
- for (std::size_t i = 0; i != words; ++i)
436
- and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
437
- or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
438
- return 1 - result_t(and_count) / or_count;
275
+ public:
276
+ buffer_gt() noexcept : data_(nullptr), size_(0u) {}
277
+ buffer_gt(std::size_t size) noexcept : data_(allocator_at{}.allocate(size)), size_(data_ ? size : 0u) {
278
+ if (!std::is_trivially_default_constructible<scalar_at>::value)
279
+ for (std::size_t i = 0; i != size_; ++i)
280
+ construct_at(data_ + i);
281
+ }
282
+ ~buffer_gt() noexcept {
283
+ if (!std::is_trivially_destructible<scalar_at>::value)
284
+ for (std::size_t i = 0; i != size_; ++i)
285
+ destroy_at(data_ + i);
286
+ allocator_at{}.deallocate(data_, size_);
287
+ data_ = nullptr;
288
+ size_ = 0;
439
289
  }
440
- };
441
-
442
- /**
443
- * @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
444
- * Often used in chemistry and biology to compare molecular fingerprints.
445
- */
446
- template <typename scalar_at = std::uint64_t, typename result_at = float> struct sorensen_gt {
447
- using scalar_t = scalar_at;
448
- using view_t = span_gt<scalar_t const>;
449
- using result_t = result_at;
450
- using result_type = result_t;
451
- static_assert( //
452
- std::is_unsigned<scalar_t>::value ||
453
- (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
454
- "Sorensen-Dice distance requires unsigned integral words");
455
- static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
456
-
457
- inline metric_kind_t kind() const noexcept { return metric_kind_t::sorensen_k; }
458
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
459
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
460
-
461
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
462
- constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
463
- result_t and_count{};
464
- result_t any_count{};
465
- #if USEARCH_USE_OPENMP
466
- #pragma omp simd reduction(+ : and_count, any_count)
467
- #elif defined(USEARCH_DEFINED_CLANG)
468
- #pragma clang loop vectorize(enable)
469
- #elif defined(USEARCH_DEFINED_GCC)
470
- #pragma GCC ivdep
471
- #endif
472
- for (std::size_t i = 0; i != words; ++i)
473
- and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
474
- any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
475
- return 1 - 2 * result_t(and_count) / any_count;
290
+ scalar_at* data() const noexcept { return data_; }
291
+ std::size_t size() const noexcept { return size_; }
292
+ scalar_at* begin() const noexcept { return data_; }
293
+ scalar_at* end() const noexcept { return data_ + size_; }
294
+ operator scalar_at*() const noexcept { return data(); }
295
+ scalar_at& operator[](std::size_t i) noexcept { return data_[i]; }
296
+ scalar_at const& operator[](std::size_t i) const noexcept { return data_[i]; }
297
+ explicit operator bool() const noexcept { return data_; }
298
+ scalar_at* release() noexcept {
299
+ size_ = 0;
300
+ return exchange(data_, nullptr);
476
301
  }
477
- };
478
302
 
479
- /**
480
- * @brief Counts the number of matching elements in two unique sorted sets.
481
- * Can be used to compute the similarity between two textual documents
482
- * using the IDs of tokens present in them.
483
- * Similar to `tanimoto_gt` for dense representations.
484
- */
485
- template <typename scalar_at = std::int32_t, typename result_at = float> struct jaccard_gt {
486
- using scalar_t = scalar_at;
487
- using view_t = span_gt<scalar_t const>;
488
- using result_t = result_at;
489
- using result_type = result_t;
490
- static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
491
-
492
- inline metric_kind_t kind() const noexcept { return metric_kind_t::jaccard_k; }
493
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
494
- inline result_t operator()(view_t a, view_t b) const noexcept {
495
- return operator()(a.data(), b.data(), a.size(), b.size());
496
- }
497
-
498
- inline result_t operator()( //
499
- scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
500
- result_t intersection{};
501
- std::size_t i{};
502
- std::size_t j{};
503
- while (i != a_length && j != b_length) {
504
- intersection += a[i] == b[j];
505
- i += a[i] < b[j];
506
- j += a[i] >= b[j];
507
- }
508
- return 1 - intersection / (a_length + b_length - intersection);
509
- }
510
- };
303
+ buffer_gt(buffer_gt const&) = delete;
304
+ buffer_gt& operator=(buffer_gt const&) = delete;
511
305
 
512
- /**
513
- * @brief Counts the number of matching elements in two unique sorted sets.
514
- * Can be used to compute the similarity between two textual documents
515
- * using the IDs of tokens present in them.
516
- */
517
- template <typename scalar_at = float, typename result_at = float> struct pearson_correlation_gt {
518
- using scalar_t = scalar_at;
519
- using view_t = span_gt<scalar_t const>;
520
- using result_t = result_at;
521
- using result_type = result_t;
522
-
523
- inline metric_kind_t kind() const noexcept { return metric_kind_t::pearson_k; }
524
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
525
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
526
-
527
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
528
- result_t a_sum{}, b_sum{}, ab_sum{};
529
- result_t a_sq_sum{}, b_sq_sum{};
530
- #if USEARCH_USE_OPENMP
531
- #pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
532
- #elif defined(USEARCH_DEFINED_CLANG)
533
- #pragma clang loop vectorize(enable)
534
- #elif defined(USEARCH_DEFINED_GCC)
535
- #pragma GCC ivdep
536
- #endif
537
- for (std::size_t i = 0; i != dim; ++i) {
538
- a_sum += result_t(a[i]);
539
- b_sum += result_t(b[i]);
540
- ab_sum += result_t(a[i]) * result_t(b[i]);
541
- a_sq_sum += result_t(a[i]) * result_t(a[i]);
542
- b_sq_sum += result_t(b[i]) * result_t(b[i]);
543
- }
544
- result_t denom = std::sqrt((dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum));
545
- result_t corr = (dim * ab_sum - a_sum * b_sum) / denom;
546
- return -corr;
306
+ buffer_gt(buffer_gt&& other) noexcept : data_(exchange(other.data_, nullptr)), size_(exchange(other.size_, 0)) {}
307
+ buffer_gt& operator=(buffer_gt&& other) noexcept {
308
+ std::swap(data_, other.data_);
309
+ std::swap(size_, other.size_);
310
+ return *this;
547
311
  }
548
312
  };
549
313
 
550
314
  /**
551
- * @brief Haversine distance for the shortest distance between two nodes on
552
- * the surface of a 3D sphere, defined with latitude and longitude.
315
+ * @brief A lightweight error class for handling error messages,
316
+ * which are expected to be allocated in static memory.
553
317
  */
554
- template <typename scalar_at = float, typename result_at = scalar_at> struct haversine_gt {
555
- using scalar_t = scalar_at;
556
- using view_t = span_gt<scalar_t const>;
557
- using result_t = result_at;
558
- using result_type = result_t;
559
- static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
560
-
561
- inline metric_kind_t kind() const noexcept { return metric_kind_t::haversine_k; }
562
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
563
-
564
- inline result_t operator()(scalar_t const* a, scalar_t const* b) const noexcept {
565
- result_t lat_a = a[0], lon_a = a[1];
566
- result_t lat_b = b[0], lon_b = b[1];
567
-
568
- result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a);
569
- result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a);
570
-
571
- result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
572
- result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
573
-
574
- result_t x = //
575
- square(std::sin(lat_delta / 2.f)) +
576
- std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta / 2.f));
577
-
578
- return std::atan2(std::sqrt(x), std::sqrt(1.f - x));
579
- }
580
- };
581
-
582
318
  class error_t {
583
319
  char const* message_{};
584
320
 
@@ -596,11 +332,14 @@ class error_t {
596
332
  explicit operator bool() const noexcept { return message_ != nullptr; }
597
333
  char const* what() const noexcept { return message_; }
598
334
 
599
- #if defined(__cpp_exceptions) && (1 == __cpp_exceptions)
335
+ #if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
600
336
  ~error_t() noexcept(false) {
601
- if (message_)
602
- if (!std::uncaught_exception())
603
- raise();
337
+ #if defined(USEARCH_DEFINED_CPP17)
338
+ if (message_ && std::uncaught_exceptions() == 0)
339
+ #else
340
+ if (message_ && std::uncaught_exception() == 0)
341
+ #endif
342
+ raise();
604
343
  }
605
344
  void raise() noexcept(false) {
606
345
  if (message_)
@@ -615,6 +354,13 @@ class error_t {
615
354
  #endif
616
355
  };
617
356
 
357
+ /**
358
+ * @brief Similar to `std::expected` in C++23, wraps a statement evaluation result,
359
+ * or an error. It's used to avoid raising exception, and gracefully propagate
360
+ * the error.
361
+ *
362
+ * @tparam result_at The type of the expected result.
363
+ */
618
364
  template <typename result_at> struct expected_gt {
619
365
  result_at result;
620
366
  error_t error;
@@ -637,49 +383,41 @@ template <typename result_at> struct expected_gt {
637
383
 
638
384
  /**
639
385
  * @brief Light-weight bitset implementation to track visited nodes during graph traversal.
640
- * Extends basic functionality with atomic operations.
386
+ * Extends basic functionality with @b atomic operations.
641
387
  */
642
- template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
388
+ template <typename allocator_at = std::allocator<byte_t>> class visits_bitset_gt {
643
389
  using allocator_t = allocator_at;
644
390
  using byte_t = typename allocator_t::value_type;
645
391
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
646
392
 
647
- using slot_t = unsigned long;
393
+ using compressed_slot_t = unsigned long;
648
394
 
649
- static constexpr std::size_t bits_per_slot() { return sizeof(slot_t) * CHAR_BIT; }
650
- static constexpr slot_t bits_mask() { return sizeof(slot_t) * CHAR_BIT - 1; }
395
+ static constexpr std::size_t bits_per_slot() { return sizeof(compressed_slot_t) * CHAR_BIT; }
396
+ static constexpr compressed_slot_t bits_mask() { return sizeof(compressed_slot_t) * CHAR_BIT - 1; }
397
+ static constexpr std::size_t slots(std::size_t bits) { return divide_round_up<bits_per_slot()>(bits); }
651
398
 
652
- slot_t* slots_{};
399
+ compressed_slot_t* slots_{};
653
400
  /// @brief Number of slots.
654
401
  std::size_t count_{};
655
402
 
656
403
  public:
657
404
  visits_bitset_gt() noexcept {}
658
405
  ~visits_bitset_gt() noexcept { reset(); }
659
- void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(slot_t)); }
406
+
407
+ explicit operator bool() const noexcept { return slots_; }
408
+ void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(compressed_slot_t)); }
660
409
 
661
410
  void reset() noexcept {
662
411
  if (slots_)
663
- allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(slot_t));
412
+ allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(compressed_slot_t));
664
413
  slots_ = nullptr;
665
414
  count_ = 0;
666
415
  }
667
416
 
668
- bool resize(std::size_t capacity) noexcept {
669
-
670
- std::size_t count = divide_round_up<bits_per_slot()>(capacity);
671
- if (count <= count_)
672
- return true;
673
-
674
- slot_t* slots = (slot_t*)allocator_t{}.allocate(count * sizeof(slot_t));
675
- if (!slots)
676
- return false;
677
-
678
- reset();
679
- count_ = count;
680
- slots_ = slots;
417
+ visits_bitset_gt(std::size_t capacity) noexcept
418
+ : slots_((compressed_slot_t*)allocator_t{}.allocate(slots(capacity) * sizeof(compressed_slot_t))),
419
+ count_(slots_ ? slots(capacity) : 0u) {
681
420
  clear();
682
- return true;
683
421
  }
684
422
 
685
423
  visits_bitset_gt(visits_bitset_gt&& other) noexcept {
@@ -697,33 +435,54 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
697
435
  visits_bitset_gt& operator=(visits_bitset_gt const&) = delete;
698
436
 
699
437
  inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
700
- inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
438
+ inline bool set(std::size_t i) noexcept {
439
+ compressed_slot_t& slot = slots_[i / bits_per_slot()];
440
+ compressed_slot_t mask{1ul << (i & bits_mask())};
441
+ bool value = slot & mask;
442
+ slot |= mask;
443
+ return value;
444
+ }
701
445
 
702
446
  #if defined(USEARCH_DEFINED_WINDOWS)
703
447
 
704
448
  inline bool atomic_set(std::size_t i) noexcept {
705
- slot_t mask{1ul << (i & bits_mask())};
449
+ compressed_slot_t mask{1ul << (i & bits_mask())};
706
450
  return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
707
451
  }
708
452
 
709
453
  inline void atomic_reset(std::size_t i) noexcept {
710
- slot_t mask{1ul << (i & bits_mask())};
454
+ compressed_slot_t mask{1ul << (i & bits_mask())};
711
455
  InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
712
456
  }
713
457
 
714
458
  #else
715
459
 
716
460
  inline bool atomic_set(std::size_t i) noexcept {
717
- slot_t mask{1ul << (i & bits_mask())};
461
+ compressed_slot_t mask{1ul << (i & bits_mask())};
718
462
  return __atomic_fetch_or(&slots_[i / bits_per_slot()], mask, __ATOMIC_ACQUIRE) & mask;
719
463
  }
720
464
 
721
465
  inline void atomic_reset(std::size_t i) noexcept {
722
- slot_t mask{1ul << (i & bits_mask())};
466
+ compressed_slot_t mask{1ul << (i & bits_mask())};
723
467
  __atomic_fetch_and(&slots_[i / bits_per_slot()], ~mask, __ATOMIC_RELEASE);
724
468
  }
725
469
 
726
470
  #endif
471
+
472
+ class lock_t {
473
+ visits_bitset_gt& bitset_;
474
+ std::size_t bit_offset_;
475
+
476
+ public:
477
+ inline ~lock_t() noexcept { bitset_.atomic_reset(bit_offset_); }
478
+ inline lock_t(visits_bitset_gt& bitset, std::size_t bit_offset) noexcept
479
+ : bitset_(bitset), bit_offset_(bit_offset) {
480
+ while (bitset_.atomic_set(bit_offset_))
481
+ ;
482
+ }
483
+ };
484
+
485
+ inline lock_t lock(std::size_t i) noexcept { return {*this, i}; }
727
486
  };
728
487
 
729
488
  using visits_bitset_t = visits_bitset_gt<>;
@@ -1006,10 +765,10 @@ class usearch_pack_m uint40_t {
1006
765
 
1007
766
  public:
1008
767
  inline uint40_t() noexcept { broadcast(0); }
1009
- inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets, (char*)&n, 4); }
1010
- inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n, 5); }
1011
- #if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
1012
- inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n, 5); }
768
+ inline uint40_t(std::uint32_t n) noexcept { std::memcpy(&octets[1], &n, 4); }
769
+
770
+ #ifdef USEARCH_64BIT_ENV
771
+ inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
1013
772
  #endif
1014
773
 
1015
774
  uint40_t(uint40_t&&) = default;
@@ -1017,47 +776,26 @@ class usearch_pack_m uint40_t {
1017
776
  uint40_t& operator=(uint40_t&&) = default;
1018
777
  uint40_t& operator=(uint40_t const&) = default;
1019
778
 
1020
- inline uint40_t& operator+=(std::uint32_t n) noexcept {
1021
- std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
1022
- octets[4] += static_cast<unsigned char>((tail + n) < tail);
1023
- tail += n;
1024
- return *this;
1025
- }
1026
-
1027
- inline uint40_t& operator+=(std::size_t n) noexcept {
1028
- unsigned char* n_octets = reinterpret_cast<unsigned char*>(&n);
1029
- std::uint32_t& n_tail = *reinterpret_cast<std::uint32_t*>(n_octets);
1030
- std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
1031
- octets[4] += static_cast<unsigned char>((tail + n_tail) < tail);
1032
- tail += n_tail;
1033
- octets[4] += n_octets[4];
1034
- return *this;
1035
- }
1036
-
1037
- inline uint40_t operator+(std::size_t n) noexcept {
1038
- uint40_t other(*this);
1039
- other += n;
1040
- return other;
779
+ #if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
780
+ inline uint40_t(std::size_t n) noexcept {
781
+ #ifdef USEARCH_64BIT_ENV
782
+ std::memcpy(octets, &n, 5);
783
+ #else
784
+ std::memcpy(octets, &n, 4);
785
+ #endif
1041
786
  }
787
+ #endif
1042
788
 
1043
789
  inline operator std::size_t() const noexcept {
1044
790
  std::size_t result = 0;
1045
791
  #ifdef USEARCH_64BIT_ENV
1046
- std::memcpy((char*)&result + 3, octets, 5);
792
+ std::memcpy(&result, octets, 5);
1047
793
  #else
1048
- std::memcpy((char*)&result, octets + 1, 4);
794
+ std::memcpy(&result, octets + 1, 4);
1049
795
  #endif
1050
796
  return result;
1051
797
  }
1052
798
 
1053
- inline uint40_t& operator++() noexcept { return *this += 1u; }
1054
-
1055
- inline uint40_t operator++(int) noexcept {
1056
- uint40_t old = *this;
1057
- *this += 1u;
1058
- return old;
1059
- }
1060
-
1061
799
  inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
1062
800
  inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
1063
801
  };
@@ -1068,6 +806,9 @@ class usearch_pack_m uint40_t {
1068
806
 
1069
807
  static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
1070
808
 
809
+ /**
810
+ * @brief Basic single-threaded @b ring class, used for all kinds of task queues.
811
+ */
1071
812
  template <typename element_at, typename allocator_at = std::allocator<element_at>> //
1072
813
  class ring_gt {
1073
814
  public:
@@ -1202,12 +943,6 @@ constexpr std::size_t default_expansion_search() { return 64; }
1202
943
 
1203
944
  constexpr std::size_t default_allocator_entry_bytes() { return 64; }
1204
945
 
1205
- /**
1206
- * @brief The "magic" sequence helps infer the type of the file.
1207
- * USearch indexes start with the "usearch" string.
1208
- */
1209
- constexpr char const* default_magic() { return "usearch"; }
1210
-
1211
946
  /**
1212
947
  * @brief Configuration settings for the index construction.
1213
948
  * Includes the main `::connectivity` parameter (`M` in the paper)
@@ -1219,14 +954,16 @@ struct index_config_t {
1219
954
  /// > It is called `M` in the paper.
1220
955
  std::size_t connectivity = default_connectivity();
1221
956
 
1222
- /// @brief Parameter controlling the physical layout of vectors
1223
- /// in memory. When using multi-byte scalar types, like `float`,
1224
- /// dereferencing at mis-aligned locations may cause UB. So you
1225
- /// should set it to at least the size of expected floats.
1226
- /// Moreover, when using SIMD-accelerated metrics, you may prefer
1227
- /// to align to SIMD register size for higher performance with
1228
- /// less split-loads.
1229
- std::size_t vector_alignment = 1;
957
+ /// @brief Number of neighbors per graph node in base level graph.
958
+ /// Defaults to double of the other levels, so 64 in FAISS and 32 in hnswlib.
959
+ /// > It is called `M0` in the paper.
960
+ std::size_t connectivity_base = default_connectivity() * 2;
961
+
962
+ inline index_config_t() = default;
963
+ inline index_config_t(std::size_t c) noexcept
964
+ : connectivity(c ? c : default_connectivity()), connectivity_base(c ? c * 2 : default_connectivity() * 2) {}
965
+ inline index_config_t(std::size_t c, std::size_t cb) noexcept
966
+ : connectivity(c), connectivity_base((std::max)(c, cb)) {}
1230
967
  };
1231
968
 
1232
969
  struct index_limits_t {
@@ -1234,13 +971,13 @@ struct index_limits_t {
1234
971
  std::size_t threads_add = std::thread::hardware_concurrency();
1235
972
  std::size_t threads_search = std::thread::hardware_concurrency();
1236
973
 
1237
- index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
1238
- index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
1239
- std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
1240
- std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
974
+ inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
975
+ inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
976
+ inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
977
+ inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
1241
978
  };
1242
979
 
1243
- struct add_config_t {
980
+ struct index_update_config_t {
1244
981
  /// @brief Hyper-parameter controlling the quality of indexing.
1245
982
  /// Defaults to 40 in FAISS and 200 in hnswlib.
1246
983
  /// > It is called `efConstruction` in the paper.
@@ -1248,12 +985,9 @@ struct add_config_t {
1248
985
 
1249
986
  /// @brief Optional thread identifier for multi-threaded construction.
1250
987
  std::size_t thread = 0;
1251
-
1252
- /// @brief Don't copy the ::vector, if it's persisted elsewhere.
1253
- bool store_vector = true;
1254
988
  };
1255
989
 
1256
- struct search_config_t {
990
+ struct index_search_config_t {
1257
991
  /// @brief Hyper-parameter controlling the quality of search.
1258
992
  /// Defaults to 16 in FAISS and 10 in hnswlib.
1259
993
  /// > It is called `ef` in the paper.
@@ -1266,133 +1000,21 @@ struct search_config_t {
1266
1000
  bool exact = false;
1267
1001
  };
1268
1002
 
1269
- struct copy_config_t {
1270
- bool copy_vectors = true;
1271
- };
1003
+ struct index_copy_config_t {};
1272
1004
 
1273
- struct join_config_t {
1005
+ struct index_join_config_t {
1274
1006
  /// @brief Controls maximum number of proposals per man during stable marriage.
1275
1007
  std::size_t max_proposals = 0;
1008
+
1276
1009
  /// @brief Hyper-parameter controlling the quality of search.
1277
1010
  /// Defaults to 16 in FAISS and 10 in hnswlib.
1278
1011
  /// > It is called `ef` in the paper.
1279
1012
  std::size_t expansion = default_expansion_search();
1013
+
1280
1014
  /// @brief Brute-forces exhaustive search over all entries in the index.
1281
1015
  bool exact = false;
1282
1016
  };
1283
1017
 
1284
- using file_header_t = byte_t[64];
1285
-
1286
- /**
1287
- * @brief Serialized binary representations of the USearch index start with metadata.
1288
- * Metadata is parsed into a `file_head_t`, containing the USearch package version,
1289
- * and the properties of the index.
1290
- *
1291
- * It uses: 13 bytes for file versioning, 22 bytes for structural information = 35 bytes.
1292
- * The following 24 bytes contain binary size of the graph, of the vectors, and the checksum,
1293
- * leaving 5 bytes at the end vacant.
1294
- */
1295
- struct file_head_t {
1296
-
1297
- // Versioning: 7 + 2 * 3 = 13 bytes
1298
- using magic_t = char[7];
1299
- using version_major_t = std::uint16_t;
1300
- using version_minor_t = std::uint16_t;
1301
- using version_patch_t = std::uint16_t;
1302
-
1303
- // Structural: 1 * 6 + 8 * 2 = 22 bytes
1304
- using connectivity_t = std::uint8_t;
1305
- using max_level_t = std::uint8_t;
1306
- using vector_alignment_t = std::uint8_t;
1307
- using bytes_per_label_t = std::uint8_t;
1308
- using bytes_per_id_t = std::uint8_t;
1309
- using size_t = std::uint64_t;
1310
- using entry_idx_t = std::uint64_t;
1311
-
1312
- // Versioning:
1313
- char const* magic;
1314
- misaligned_ref_gt<version_major_t> version_major;
1315
- misaligned_ref_gt<version_minor_t> version_minor;
1316
- misaligned_ref_gt<version_patch_t> version_patch;
1317
-
1318
- // Structural:
1319
- misaligned_ref_gt<metric_kind_t> metric;
1320
- misaligned_ref_gt<connectivity_t> connectivity;
1321
- misaligned_ref_gt<max_level_t> max_level;
1322
- misaligned_ref_gt<vector_alignment_t> vector_alignment;
1323
- misaligned_ref_gt<bytes_per_label_t> bytes_per_label;
1324
- misaligned_ref_gt<bytes_per_id_t> bytes_per_id;
1325
- misaligned_ref_gt<scalar_kind_t> scalar_kind;
1326
- misaligned_ref_gt<size_t> size;
1327
- misaligned_ref_gt<entry_idx_t> entry_idx;
1328
-
1329
- // Additional:
1330
- misaligned_ref_gt<size_t> bytes_for_graphs;
1331
- misaligned_ref_gt<size_t> bytes_for_vectors;
1332
- misaligned_ref_gt<size_t> bytes_checksum;
1333
-
1334
- file_head_t(byte_t* ptr) noexcept
1335
- : magic((char const*)exchange(ptr, ptr + sizeof(magic_t))),
1336
- version_major(exchange(ptr, ptr + sizeof(version_major_t))),
1337
- version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
1338
- version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
1339
- metric(exchange(ptr, ptr + sizeof(metric_kind_t))), connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
1340
- max_level(exchange(ptr, ptr + sizeof(max_level_t))),
1341
- vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
1342
- bytes_per_label(exchange(ptr, ptr + sizeof(bytes_per_label_t))),
1343
- bytes_per_id(exchange(ptr, ptr + sizeof(bytes_per_id_t))),
1344
- scalar_kind(exchange(ptr, ptr + sizeof(scalar_kind_t))), size(exchange(ptr, ptr + sizeof(size_t))),
1345
- entry_idx(exchange(ptr, ptr + sizeof(entry_idx_t))), bytes_for_graphs(exchange(ptr, ptr + sizeof(size_t))),
1346
- bytes_for_vectors(exchange(ptr, ptr + sizeof(size_t))), bytes_checksum(exchange(ptr, ptr + sizeof(size_t))) {}
1347
- };
1348
-
1349
- struct file_head_result_t {
1350
-
1351
- using magic_t = file_head_t::magic_t;
1352
- using version_major_t = file_head_t::version_major_t;
1353
- using version_minor_t = file_head_t::version_minor_t;
1354
- using version_patch_t = file_head_t::version_patch_t;
1355
-
1356
- using connectivity_t = file_head_t::connectivity_t;
1357
- using max_level_t = file_head_t::max_level_t;
1358
- using vector_alignment_t = file_head_t::vector_alignment_t;
1359
- using bytes_per_label_t = file_head_t::bytes_per_label_t;
1360
- using bytes_per_id_t = file_head_t::bytes_per_id_t;
1361
- using size_t = file_head_t::size_t;
1362
- using entry_idx_t = file_head_t::entry_idx_t;
1363
-
1364
- // Versioning:
1365
- version_major_t version_major;
1366
- version_minor_t version_minor;
1367
- version_patch_t version_patch;
1368
-
1369
- // Structural:
1370
- metric_kind_t metric;
1371
- connectivity_t connectivity;
1372
- max_level_t max_level;
1373
- vector_alignment_t vector_alignment;
1374
- bytes_per_label_t bytes_per_label;
1375
- bytes_per_id_t bytes_per_id;
1376
- scalar_kind_t scalar_kind;
1377
- size_t size;
1378
- entry_idx_t entry_idx;
1379
-
1380
- // Additional:
1381
- size_t bytes_for_graphs;
1382
- size_t bytes_for_vectors;
1383
- size_t bytes_checksum;
1384
-
1385
- error_t error;
1386
-
1387
- explicit operator bool() const noexcept { return !error; }
1388
- file_head_result_t failed(error_t message) noexcept {
1389
- error = std::move(message);
1390
- return std::move(*this);
1391
- }
1392
- };
1393
-
1394
- static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 bytes");
1395
-
1396
1018
  /// @brief C++17 and newer version deprecate the `std::result_of`
1397
1019
  template <typename metric_at, typename... args_at>
1398
1020
  using return_type_gt =
@@ -1402,32 +1024,65 @@ using return_type_gt =
1402
1024
  typename std::result_of<metric_at(args_at...)>::type;
1403
1025
  #endif
1404
1026
 
1405
- /// @brief OS-specific to wrap open file-descriptors/handles.
1406
- #if defined(USEARCH_DEFINED_WINDOWS)
1407
- struct viewed_file_t {
1408
- HANDLE file_handle{};
1409
- HANDLE mapping_handle{};
1410
- void* ptr{};
1411
- size_t length{};
1412
- explicit operator bool() const noexcept { return mapping_handle != nullptr; }
1413
- };
1414
- #else
1415
- struct viewed_file_t {
1416
- int file_descriptor{};
1417
- void* ptr{};
1418
- size_t length{};
1419
- explicit operator bool() const noexcept { return file_descriptor != 0; }
1027
+ /**
1028
+ * @brief An example of what a USearch-compatible ad-hoc filter would look like.
1029
+ *
1030
+ * A similar function object can be passed to search queries to further filter entries
1031
+ * on their auxiliary properties, such as some categorical labels stored in an external DBMS.
1032
+ */
1033
+ struct dummy_predicate_t {
1034
+ template <typename member_at> constexpr bool operator()(member_at&&) const noexcept { return true; }
1420
1035
  };
1421
- #endif
1422
1036
 
1423
- struct dummy_predicate_t {
1424
- template <typename match_at> constexpr bool operator()(match_at&&) const noexcept { return true; }
1037
+ /**
1038
+ * @brief An example of what a USearch-compatible ad-hoc operation on in-flight entries.
1039
+ *
1040
+ * This kind of callbacks is used when the engine is being updated and you want to patch
1041
+ * the entries, while their are still under locks - limiting concurrent access and providing
1042
+ * consistency.
1043
+ */
1044
+ struct dummy_callback_t {
1045
+ template <typename member_at> void operator()(member_at&&) const noexcept {}
1425
1046
  };
1426
1047
 
1048
+ /**
1049
+ * @brief An example of what a USearch-compatible progress-bar should look like.
1050
+ *
1051
+ * This is particularly helpful when handling long-running tasks, like serialization,
1052
+ * saving, and loading from disk, or index-level joins.
1053
+ */
1427
1054
  struct dummy_progress_t {
1428
1055
  inline void operator()(std::size_t /*progress*/, std::size_t /*total*/) const noexcept {}
1429
1056
  };
1430
1057
 
1058
+ /**
1059
+ * @brief An example of what a USearch-compatible values prefetching mechanism should look like.
1060
+ *
1061
+ * USearch is designed to handle very large datasets, that may not fir into RAM. Fetching from
1062
+ * external memory is very expensive, so we've added a pre-fetching mechanism, that accepts
1063
+ * multiple objects at once, to cache in RAM ahead of the computation.
1064
+ * The received iterators support both `get_slot` and `get_key` operations.
1065
+ * An example usage may look like this:
1066
+ *
1067
+ * template <typename member_citerator_like_at>
1068
+ * inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {
1069
+ * for (; begin != end; ++begin)
1070
+ * io_uring_prefetch(offset_in_file(get_key(begin)));
1071
+ * }
1072
+ */
1073
+ struct dummy_prefetch_t {
1074
+ template <typename member_citerator_like_at>
1075
+ inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {}
1076
+ };
1077
+
1078
+ /**
1079
+ * @brief An example of what a USearch-compatible executor (thread-pool) should look like.
1080
+ *
1081
+ * It's expected to have `execute_bulk(callback)` API to schedule one task per thread;
1082
+ * an identical `execute_bulk(count, callback)` overload that also accepts the number
1083
+ * of tasks, and somehow schedules them between threads; as well as `size()` to
1084
+ * determine the number of available threads.
1085
+ */
1431
1086
  struct dummy_executor_t {
1432
1087
  dummy_executor_t() noexcept {}
1433
1088
  std::size_t size() const noexcept { return 1; }
@@ -1444,164 +1099,474 @@ struct dummy_executor_t {
1444
1099
  }
1445
1100
  };
1446
1101
 
1102
+ /**
1103
+ * @brief An example of what a USearch-compatible label-to-label mapping should look like.
1104
+ *
1105
+ * This is particularly helpful for "Semantic Joins", where we map entries of one collection
1106
+ * to entries of another. In assymetric setups, where A -> B is needed, but B -> A is not,
1107
+ * this can be passed to minimize memory usage.
1108
+ */
1447
1109
  struct dummy_label_to_label_mapping_t {
1448
1110
  struct member_ref_t {
1449
- template <typename label_at> member_ref_t& operator=(label_at&&) noexcept { return *this; }
1111
+ template <typename key_at> member_ref_t& operator=(key_at&&) noexcept { return *this; }
1450
1112
  };
1451
- template <typename label_at> member_ref_t operator[](label_at&&) const noexcept { return {}; }
1113
+ template <typename key_at> member_ref_t operator[](key_at&&) const noexcept { return {}; }
1452
1114
  };
1453
1115
 
1454
1116
  /**
1455
- * @brief Approximate Nearest Neighbors Search index using the
1456
- * Hierarchical Navigable Small World graph algorithm.
1457
- *
1458
- * @section Features
1459
- * - Thread-safe for concurrent construction.
1460
- * - Doesn't allocate new threads, and reuses the ones its called from.
1461
- * - Search for vectors of different dimensionality, if ::`metric_at` supports that.
1462
- *
1463
- * @tparam metric_at
1464
- * A function responsible for computing the distance (dis-similarity) between two vectors.
1465
- * Can have following signatures:
1466
- * - `distance_t (*) (span_gt<scalar_t const>, span_gt<scalar_t const>)`
1467
- * - `distance_t (*) (scalar_t const *, scalar_t const *)`
1468
- *
1469
- * @tparam label_at
1470
- * The type of unique labels to assign to vectors.
1471
- *
1472
- * @tparam id_at
1473
- * The smallest unsigned integer type to address indexed elements.
1474
- * Can be a built-in `uint32_t`, `uint64_t`, or our custom `uint40_t`.
1475
- *
1476
- * @tparam allocator_at
1477
- * Dynamic memory allocator.
1478
- *
1479
- * @section Usage
1480
- *
1481
- * @subsection Exceptions
1482
- *
1483
- * None of the methods throw exceptions in the "Release" compilation mode.
1484
- * It may only `throw` if your memory ::allocator_at or ::metric_at isn't
1485
- * safe to copy.
1486
- *
1487
- * @section Implementation Details
1488
- *
1489
- * Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
1490
- * Every added vector forms a node in one or more levels of the graph.
1491
- * Every node is present in the base level. Every following level contains a smaller
1492
- * fraction of nodes. During search, the operation starts with the smaller levels
1493
- * and zooms-in on every following iteration of larger graph traversals.
1494
- *
1495
- * Just one memory allocation is performed regardless of the number of levels.
1496
- * The adjacency lists across all levels are concatenated into that single buffer.
1497
- * That buffer starts with a "head", that stores the metadata, such as the
1498
- * tallest "level" of the graph that it belongs to, the external "label", and the
1499
- * number of "dimensions" in the vector.
1500
- *
1501
- * @subsection Colocated Nodes
1502
- *
1503
- * Every node contains some graph-related data, as well as the original vector.
1504
- * Assuming the vectors can get large, we allow separating the index from the vectors.
1505
- * They can either be stored in a single continuous block of memory, or in two separate
1506
- * locations. Vector storage can also be detached from `index_gt` and managed by the user.
1507
- * To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
1508
- * overloads.
1509
- *
1510
- * @subsection Missing Functionality
1511
- *
1512
- * To simplify the implementation, the `index_gt` lacks endpoints to remove existing
1513
- * vectors. That, however, is solved by `index_punned_dense_gt`, which also adds automatic casting.
1514
- *
1117
+ * @brief Checks if the provided object has a dummy type, emulating an interface,
1118
+ * but performaing no real computation.
1515
1119
  */
1516
- template <typename metric_at = ip_gt<float>, //
1517
- typename label_at = std::int64_t, //
1518
- typename id_at = std::uint32_t, //
1519
- typename allocator_at = std::allocator<char>, //
1520
- typename point_allocator_at = allocator_at> //
1521
- class index_gt {
1522
- public:
1523
- using metric_t = metric_at;
1524
- using label_t = label_at;
1525
- using id_t = id_at;
1526
- using allocator_t = allocator_at;
1527
- using point_allocator_t = point_allocator_at;
1528
- static_assert(sizeof(label_t) >= sizeof(id_t), "Having tiny labels doesn't make sense.");
1529
-
1530
- using scalar_t = typename metric_t::scalar_t;
1531
- using vector_view_t = span_gt<scalar_t const>;
1532
- using distance_t = return_type_gt<metric_t, vector_view_t, vector_view_t>;
1120
+ template <typename object_at> static constexpr bool is_dummy() {
1121
+ using object_t = typename std::remove_all_extents<object_at>::type;
1122
+ return std::is_same<object_t, dummy_predicate_t>::value || //
1123
+ std::is_same<object_t, dummy_callback_t>::value || //
1124
+ std::is_same<object_t, dummy_progress_t>::value || //
1125
+ std::is_same<object_t, dummy_prefetch_t>::value || //
1126
+ std::is_same<object_t, dummy_executor_t>::value || //
1127
+ std::is_same<object_t, dummy_label_to_label_mapping_t>::value;
1128
+ }
1533
1129
 
1534
- struct member_cref_t {
1535
- label_t label;
1536
- vector_view_t vector;
1537
- id_t id;
1538
- };
1130
+ template <typename, typename at> struct has_reset_gt {
1131
+ static_assert(std::integral_constant<at, false>::value, "Second template parameter needs to be of function type.");
1132
+ };
1539
1133
 
1540
- struct member_ref_t {
1541
- misaligned_ref_gt<label_t> label;
1542
- vector_view_t vector;
1543
- id_t id;
1134
+ template <typename check_at, typename return_at, typename... args_at>
1135
+ struct has_reset_gt<check_at, return_at(args_at...)> {
1136
+ private:
1137
+ template <typename at>
1138
+ static constexpr auto check(at*) ->
1139
+ typename std::is_same<decltype(std::declval<at>().reset(std::declval<args_at>()...)), return_at>::type;
1140
+ template <typename> static constexpr std::false_type check(...);
1544
1141
 
1545
- inline operator member_cref_t() const noexcept { return {label, vector, id}; }
1546
- };
1142
+ typedef decltype(check<check_at>(0)) type;
1547
1143
 
1548
- template <typename ref_at, typename index_at> class member_iterator_gt {
1549
- using ref_t = ref_at;
1550
- using index_t = index_at;
1551
- index_t* index_{};
1552
- std::size_t offset_{};
1144
+ public:
1145
+ static constexpr bool value = type::value;
1146
+ };
1553
1147
 
1554
- friend class index_gt;
1555
- member_iterator_gt() noexcept {}
1556
- member_iterator_gt(index_t* index, std::size_t offset) noexcept : index_(index), offset_(offset) {}
1148
+ /**
1149
+ * @brief Checks if a certain class has a member function called `reset`.
1150
+ */
1151
+ template <typename at> constexpr bool has_reset() { return has_reset_gt<at, void()>::value; }
1557
1152
 
1558
- public:
1559
- using iterator_category = std::random_access_iterator_tag;
1560
- using value_type = ref_t;
1561
- using difference_type = std::ptrdiff_t;
1562
- using pointer = void;
1563
- using reference = ref_t;
1153
+ struct serialization_result_t {
1154
+ error_t error;
1564
1155
 
1565
- reference operator*() const noexcept {
1566
- node_t node = index_->node_with_id_(offset_);
1567
- return {node.label(), node.vector_view(), static_cast<id_t>(offset_)};
1568
- }
1156
+ explicit operator bool() const noexcept { return !error; }
1157
+ serialization_result_t failed(error_t message) noexcept {
1158
+ error = std::move(message);
1159
+ return std::move(*this);
1160
+ }
1161
+ };
1569
1162
 
1570
- member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, offset_ + 1); }
1571
- member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, offset_ - 1); }
1572
- member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, offset_ + d); }
1573
- member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, offset_ - d); }
1163
+ /**
1164
+ * @brief Smart-pointer wrapping the LibC @b `FILE` for binary file @b outputs.
1165
+ *
1166
+ * This class raises no exceptions and corresponds errors through `serialization_result_t`.
1167
+ * The class automatically closes the file when the object is destroyed.
1168
+ */
1169
+ class output_file_t {
1170
+ char const* path_ = nullptr;
1171
+ std::FILE* file_ = nullptr;
1574
1172
 
1575
- member_iterator_gt& operator++() noexcept {
1576
- offset_ += 1;
1577
- return *this;
1578
- }
1579
- member_iterator_gt& operator--() noexcept {
1580
- offset_ -= 1;
1581
- return *this;
1582
- }
1583
- member_iterator_gt& operator+=(difference_type d) noexcept {
1584
- offset_ += d;
1585
- return *this;
1586
- }
1587
- member_iterator_gt& operator-=(difference_type d) noexcept {
1588
- offset_ -= d;
1589
- return *this;
1173
+ public:
1174
+ output_file_t(char const* path) noexcept : path_(path) {}
1175
+ ~output_file_t() noexcept { close(); }
1176
+ output_file_t(output_file_t&& other) noexcept
1177
+ : path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
1178
+ output_file_t& operator=(output_file_t&& other) noexcept {
1179
+ std::swap(path_, other.path_);
1180
+ std::swap(file_, other.file_);
1181
+ return *this;
1182
+ }
1183
+ serialization_result_t open_if_not() noexcept {
1184
+ serialization_result_t result;
1185
+ if (!file_)
1186
+ file_ = std::fopen(path_, "wb");
1187
+ if (!file_)
1188
+ return result.failed(std::strerror(errno));
1189
+ return result;
1190
+ }
1191
+ serialization_result_t write(void* begin, std::size_t length) noexcept {
1192
+ serialization_result_t result;
1193
+ std::size_t written = std::fwrite(begin, length, 1, file_);
1194
+ if (!written)
1195
+ result.failed(std::strerror(errno));
1196
+ return result;
1197
+ }
1198
+ void close() noexcept {
1199
+ if (file_)
1200
+ std::fclose(exchange(file_, nullptr));
1201
+ }
1202
+ };
1203
+
1204
+ /**
1205
+ * @brief Smart-pointer wrapping the LibC @b `FILE` for binary files @b inputs.
1206
+ *
1207
+ * This class raises no exceptions and corresponds errors through `serialization_result_t`.
1208
+ * The class automatically closes the file when the object is destroyed.
1209
+ */
1210
+ class input_file_t {
1211
+ char const* path_ = nullptr;
1212
+ std::FILE* file_ = nullptr;
1213
+
1214
+ public:
1215
+ input_file_t(char const* path) noexcept : path_(path) {}
1216
+ ~input_file_t() noexcept { close(); }
1217
+ input_file_t(input_file_t&& other) noexcept
1218
+ : path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
1219
+ input_file_t& operator=(input_file_t&& other) noexcept {
1220
+ std::swap(path_, other.path_);
1221
+ std::swap(file_, other.file_);
1222
+ return *this;
1223
+ }
1224
+
1225
+ serialization_result_t open_if_not() noexcept {
1226
+ serialization_result_t result;
1227
+ if (!file_)
1228
+ file_ = std::fopen(path_, "rb");
1229
+ if (!file_)
1230
+ return result.failed(std::strerror(errno));
1231
+ return result;
1232
+ }
1233
+ serialization_result_t read(void* begin, std::size_t length) noexcept {
1234
+ serialization_result_t result;
1235
+ std::size_t read = std::fread(begin, length, 1, file_);
1236
+ if (!read)
1237
+ result.failed(std::feof(file_) ? "End of file reached!" : std::strerror(errno));
1238
+ return result;
1239
+ }
1240
+ void close() noexcept {
1241
+ if (file_)
1242
+ std::fclose(exchange(file_, nullptr));
1243
+ }
1244
+
1245
+ explicit operator bool() const noexcept { return file_; }
1246
+ bool seek_to(std::size_t progress) noexcept { return std::fseek(file_, progress, SEEK_SET) == 0; }
1247
+ bool seek_to_end() noexcept { return std::fseek(file_, 0L, SEEK_END) == 0; }
1248
+ bool infer_progress(std::size_t& progress) noexcept {
1249
+ long int result = std::ftell(file_);
1250
+ if (result == -1L)
1251
+ return false;
1252
+ progress = static_cast<std::size_t>(result);
1253
+ return true;
1254
+ }
1255
+ };
1256
+
1257
+ /**
1258
+ * @brief Represents a memory-mapped file or a pre-allocated anonymous memory region.
1259
+ *
1260
+ * This class provides a convenient way to memory-map a file and access its contents as a block of
1261
+ * memory. The class handles platform-specific memory-mapping operations on Windows, Linux, and MacOS.
1262
+ * The class automatically closes the file when the object is destroyed.
1263
+ */
1264
+ class memory_mapped_file_t {
1265
+ char const* path_{}; /**< The path to the file to be memory-mapped. */
1266
+ void* ptr_{}; /**< A pointer to the memory-mapping. */
1267
+ size_t length_{}; /**< The length of the memory-mapped file in bytes. */
1268
+
1269
+ #if defined(USEARCH_DEFINED_WINDOWS)
1270
+ HANDLE file_handle_{}; /**< The file handle on Windows. */
1271
+ HANDLE mapping_handle_{}; /**< The mapping handle on Windows. */
1272
+ #else
1273
+ int file_descriptor_{}; /**< The file descriptor on Linux and MacOS. */
1274
+ #endif
1275
+
1276
+ public:
1277
+ explicit operator bool() const noexcept { return ptr_ != nullptr; }
1278
+ byte_t const* data() const noexcept { return reinterpret_cast<byte_t const*>(ptr_); }
1279
+ std::size_t size() const noexcept { return static_cast<std::size_t>(length_); }
1280
+
1281
+ memory_mapped_file_t() noexcept {}
1282
+ memory_mapped_file_t(char const* path) noexcept : path_(path) {}
1283
+ ~memory_mapped_file_t() noexcept { close(); }
1284
+ memory_mapped_file_t(memory_mapped_file_t&& other) noexcept
1285
+ : path_(exchange(other.path_, nullptr)), ptr_(exchange(other.ptr_, nullptr)),
1286
+ length_(exchange(other.length_, 0)),
1287
+ #if defined(USEARCH_DEFINED_WINDOWS)
1288
+ file_handle_(exchange(other.file_handle_, nullptr)), mapping_handle_(exchange(other.mapping_handle_, nullptr))
1289
+ #else
1290
+ file_descriptor_(exchange(other.file_descriptor_, 0))
1291
+ #endif
1292
+ {
1293
+ }
1294
+
1295
+ memory_mapped_file_t(byte_t* data, std::size_t length) noexcept : ptr_(data), length_(length) {}
1296
+
1297
+ memory_mapped_file_t& operator=(memory_mapped_file_t&& other) noexcept {
1298
+ std::swap(path_, other.path_);
1299
+ std::swap(ptr_, other.ptr_);
1300
+ std::swap(length_, other.length_);
1301
+ #if defined(USEARCH_DEFINED_WINDOWS)
1302
+ std::swap(file_handle_, other.file_handle_);
1303
+ std::swap(mapping_handle_, other.mapping_handle_);
1304
+ #else
1305
+ std::swap(file_descriptor_, other.file_descriptor_);
1306
+ #endif
1307
+ return *this;
1308
+ }
1309
+
1310
+ serialization_result_t open_if_not() noexcept {
1311
+ serialization_result_t result;
1312
+ if (!path_ || ptr_)
1313
+ return result;
1314
+
1315
+ #if defined(USEARCH_DEFINED_WINDOWS)
1316
+
1317
+ HANDLE file_handle =
1318
+ CreateFile(path_, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
1319
+ if (file_handle == INVALID_HANDLE_VALUE)
1320
+ return result.failed("Opening file failed!");
1321
+
1322
+ std::size_t file_length = GetFileSize(file_handle, 0);
1323
+ HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
1324
+ if (mapping_handle == 0) {
1325
+ CloseHandle(file_handle);
1326
+ return result.failed("Mapping file failed!");
1590
1327
  }
1591
- bool operator==(member_iterator_gt const& other) const noexcept {
1592
- return index_ == other.index_ && offset_ == other.offset_;
1328
+
1329
+ byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
1330
+ if (file == 0) {
1331
+ CloseHandle(mapping_handle);
1332
+ CloseHandle(file_handle);
1333
+ return result.failed("View the map failed!");
1593
1334
  }
1594
- bool operator!=(member_iterator_gt const& other) const noexcept {
1595
- return index_ != other.index_ || offset_ != other.offset_;
1335
+ file_handle_ = file_handle;
1336
+ mapping_handle_ = mapping_handle;
1337
+ ptr_ = file;
1338
+ length_ = file_length;
1339
+ #else
1340
+
1341
+ #if defined(USEARCH_DEFINED_LINUX)
1342
+ int descriptor = open(path_, O_RDONLY | O_NOATIME);
1343
+ #else
1344
+ int descriptor = open(path_, O_RDONLY);
1345
+ #endif
1346
+
1347
+ // Estimate the file size
1348
+ struct stat file_stat;
1349
+ int fstat_status = fstat(descriptor, &file_stat);
1350
+ if (fstat_status < 0) {
1351
+ ::close(descriptor);
1352
+ return result.failed(std::strerror(errno));
1353
+ }
1354
+
1355
+ // Map the entire file
1356
+ byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_SHARED, descriptor, 0);
1357
+ if (file == MAP_FAILED) {
1358
+ ::close(descriptor);
1359
+ return result.failed(std::strerror(errno));
1360
+ }
1361
+ file_descriptor_ = descriptor;
1362
+ ptr_ = file;
1363
+ length_ = file_stat.st_size;
1364
+ #endif // Platform specific code
1365
+ return result;
1366
+ }
1367
+
1368
+ void close() noexcept {
1369
+ if (!path_) {
1370
+ ptr_ = nullptr;
1371
+ length_ = 0;
1372
+ return;
1596
1373
  }
1374
+ #if defined(USEARCH_DEFINED_WINDOWS)
1375
+ UnmapViewOfFile(ptr_);
1376
+ CloseHandle(mapping_handle_);
1377
+ CloseHandle(file_handle_);
1378
+ mapping_handle_ = nullptr;
1379
+ file_handle_ = nullptr;
1380
+ #else
1381
+ munmap(ptr_, length_);
1382
+ ::close(file_descriptor_);
1383
+ file_descriptor_ = 0;
1384
+ #endif
1385
+ ptr_ = nullptr;
1386
+ length_ = 0;
1387
+ }
1388
+ };
1389
+
1390
+ struct index_serialized_header_t {
1391
+ std::uint64_t size = 0;
1392
+ std::uint64_t connectivity = 0;
1393
+ std::uint64_t connectivity_base = 0;
1394
+ std::uint64_t max_level = 0;
1395
+ std::uint64_t entry_slot = 0;
1396
+ };
1397
+
1398
+ using default_key_t = std::uint64_t;
1399
+ using default_slot_t = std::uint32_t;
1400
+ using default_distance_t = float;
1401
+
1402
+ template <typename key_at = default_key_t> struct member_gt {
1403
+ key_at key;
1404
+ std::size_t slot;
1405
+ };
1406
+
1407
+ template <typename key_at> inline std::size_t get_slot(member_gt<key_at> const& m) noexcept { return m.slot; }
1408
+ template <typename key_at> inline key_at get_key(member_gt<key_at> const& m) noexcept { return m.key; }
1409
+
1410
+ template <typename key_at = default_key_t> struct member_cref_gt {
1411
+ misaligned_ref_gt<key_at const> key;
1412
+ std::size_t slot;
1413
+ };
1414
+
1415
+ template <typename key_at> inline std::size_t get_slot(member_cref_gt<key_at> const& m) noexcept { return m.slot; }
1416
+ template <typename key_at> inline key_at get_key(member_cref_gt<key_at> const& m) noexcept { return m.key; }
1417
+
1418
+ template <typename key_at = default_key_t> struct member_ref_gt {
1419
+ misaligned_ref_gt<key_at> key;
1420
+ std::size_t slot;
1421
+
1422
+ inline operator member_cref_gt<key_at>() const noexcept { return {key.ptr(), slot}; }
1423
+ };
1424
+
1425
+ template <typename key_at> inline std::size_t get_slot(member_ref_gt<key_at> const& m) noexcept { return m.slot; }
1426
+ template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m) noexcept { return m.key; }
1427
+
1428
+ /**
1429
+ * @brief Approximate Nearest Neighbors Search @b index-structure using the
1430
+ * Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
1431
+ * If classical containers store @b Key->Value mappings, this one can
1432
+ * be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
1433
+ *
1434
+ * Unlike most implementations, this one is generic anc can be used for any search,
1435
+ * not just within equi-dimensional vectors. Examples range from texts to similar Chess
1436
+ * positions.
1437
+ *
1438
+ * @tparam key_at
1439
+ * The type of primary objects stored in the index.
1440
+ * The values, to which those map, are not managed by the same index structure.
1441
+ *
1442
+ * @tparam compressed_slot_at
1443
+ * The smallest unsigned integer type to address indexed elements.
1444
+ * It is used internally to maximize space-efficiency and is generally
1445
+ * up-casted to @b `std::size_t` in public interfaces.
1446
+ * Can be a built-in @b `uint32_t`, `uint64_t`, or our custom @b `uint40_t`.
1447
+ * Which makes the most sense for 4B+ entry indexes.
1448
+ *
1449
+ * @tparam dynamic_allocator_at
1450
+ * Dynamic memory allocator for temporary buffers, visits indicators, and
1451
+ * priority queues, needed during construction and traversals of graphs.
1452
+ * The allocated buffers may be uninitialized.
1453
+ *
1454
+ * @tparam tape_allocator_at
1455
+ * Potentially different memory allocator for primary allocations of nodes and vectors.
1456
+ * It would never `deallocate` separate entries, and would only free all the space at once.
1457
+ * The allocated buffers may be uninitialized.
1458
+ *
1459
+ * @section Features
1460
+ *
1461
+ * - Thread-safe for concurrent construction, search, and updates.
1462
+ * - Doesn't allocate new threads, and reuses the ones its called from.
1463
+ * - Allows storing value externally, managing just the similarity index.
1464
+ * - Joins.
1465
+
1466
+ * @section Usage
1467
+ *
1468
+ * @subsection Exceptions
1469
+ *
1470
+ * None of the methods throw exceptions in the "Release" compilation mode.
1471
+ * It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
1472
+ * safe to copy.
1473
+ *
1474
+ * @subsection Serialization
1475
+ *
1476
+ * When serialized, doesn't include any additional metadata.
1477
+ * It is just the multi-level proximity-graph. You may want to store metadata about
1478
+ * the used metric and key types somewhere else.
1479
+ *
1480
+ * @section Implementation Details
1481
+ *
1482
+ * Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
1483
+ * Every added vector forms a node in one or more levels of the graph.
1484
+ * Every node is present in the base level. Every following level contains a smaller
1485
+ * fraction of nodes. During search, the operation starts with the smaller levels
1486
+ * and zooms-in on every following iteration of larger graph traversals.
1487
+ *
1488
+ * Just one memory allocation is performed regardless of the number of levels.
1489
+ * The adjacency lists across all levels are concatenated into that single buffer.
1490
+ * That buffer starts with a "head", that stores the metadata, such as the
1491
+ * tallest "level" of the graph that it belongs to, the external "key", and the
1492
+ * number of "dimensions" in the vector.
1493
+ *
1494
+ * @section Metrics, Predicates and Callbacks
1495
+ *
1496
+ *
1497
+ * @section Smart References and Iterators
1498
+ *
1499
+ * - `member_citerator_t` and `member_iterator_t` have only slots, no indirections.
1500
+ *
1501
+ * - `member_cref_t` and `member_ref_t` contains the `slot` and a reference
1502
+ * to the key. So it passes through 1 level of visited_members in `nodes_`.
1503
+ * Retrieving the key via `get_key` will cause fetching yet another cache line.
1504
+ *
1505
+ * - `member_gt` contains an already prefetched copy of the key.
1506
+ *
1507
+ */
1508
+ template <typename distance_at = default_distance_t, //
1509
+ typename key_at = default_key_t, //
1510
+ typename compressed_slot_at = default_slot_t, //
1511
+ typename dynamic_allocator_at = std::allocator<byte_t>, //
1512
+ typename tape_allocator_at = dynamic_allocator_at> //
1513
+ class index_gt {
1514
+ public:
1515
+ using distance_t = distance_at;
1516
+ using key_t = key_at;
1517
+ using compressed_slot_t = compressed_slot_at;
1518
+ using dynamic_allocator_t = dynamic_allocator_at;
1519
+ using tape_allocator_t = tape_allocator_at;
1520
+ static_assert(sizeof(key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
1521
+
1522
+ using member_cref_t = member_cref_gt<key_t>;
1523
+ using member_ref_t = member_ref_gt<key_t>;
1524
+
1525
+ template <typename ref_at, typename index_at> class member_iterator_gt {
1526
+ using ref_t = ref_at;
1527
+ using index_t = index_at;
1528
+
1529
+ friend class index_gt;
1530
+ member_iterator_gt() noexcept {}
1531
+ member_iterator_gt(index_t* index, std::size_t slot) noexcept : index_(index), slot_(slot) {}
1532
+
1533
+ index_t* index_{};
1534
+ std::size_t slot_{};
1535
+
1536
+ public:
1537
+ using iterator_category = std::random_access_iterator_tag;
1538
+ using value_type = ref_t;
1539
+ using difference_type = std::ptrdiff_t;
1540
+ using pointer = void;
1541
+ using reference = ref_t;
1542
+
1543
+ reference operator*() const noexcept { return {index_->node_at_(slot_).key(), slot_}; }
1544
+ key_t key() const noexcept { return index_->node_at_(slot_).key(); }
1545
+
1546
+ friend inline std::size_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
1547
+ friend inline key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
1548
+
1549
+ member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
1550
+ member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
1551
+ member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
1552
+ member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
1553
+
1554
+ // clang-format off
1555
+ member_iterator_gt& operator++() noexcept { slot_ += 1; return *this; }
1556
+ member_iterator_gt& operator--() noexcept { slot_ -= 1; return *this; }
1557
+ member_iterator_gt& operator+=(difference_type d) noexcept { slot_ += d; return *this; }
1558
+ member_iterator_gt& operator-=(difference_type d) noexcept { slot_ -= d; return *this; }
1559
+ bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
1560
+ bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
1561
+ // clang-format on
1597
1562
  };
1598
1563
 
1599
1564
  using member_iterator_t = member_iterator_gt<member_ref_t, index_gt>;
1600
1565
  using member_citerator_t = member_iterator_gt<member_cref_t, index_gt const>;
1601
1566
 
1602
1567
  // STL compatibility:
1603
- using value_type = std::pair<label_t, distance_t>;
1604
- using allocator_type = allocator_t;
1568
+ using value_type = key_t;
1569
+ using allocator_type = dynamic_allocator_t;
1605
1570
  using size_type = std::size_t;
1606
1571
  using difference_type = std::ptrdiff_t;
1607
1572
  using reference = member_ref_t;
@@ -1613,109 +1578,113 @@ class index_gt {
1613
1578
  using reverse_iterator = std::reverse_iterator<member_iterator_t>;
1614
1579
  using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
1615
1580
 
1616
- private:
1617
- using neighbors_count_t = std::uint32_t;
1618
- using dim_t = std::uint32_t;
1619
- using level_t = std::int32_t;
1581
+ using dynamic_allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
1582
+ using byte_t = typename dynamic_allocator_t::value_type;
1583
+ static_assert( //
1584
+ sizeof(byte_t) == 1, //
1585
+ "Primary allocator must allocate separate addressable bytes");
1620
1586
 
1621
- using allocator_traits_t = std::allocator_traits<allocator_t>;
1622
- using byte_t = typename allocator_t::value_type;
1623
- static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
1624
-
1625
- using point_allocator_traits_t = std::allocator_traits<point_allocator_t>;
1626
- static_assert(sizeof(typename point_allocator_traits_t::value_type) == 1,
1627
- "Allocator must allocate separate addressable bytes");
1587
+ using tape_allocator_traits_t = std::allocator_traits<tape_allocator_t>;
1588
+ static_assert( //
1589
+ sizeof(typename tape_allocator_traits_t::value_type) == 1, //
1590
+ "Tape allocator must allocate separate addressable bytes");
1628
1591
 
1592
+ private:
1629
1593
  /**
1630
- * @brief How much larger (number of neighbors per node) will
1631
- * the base level be compared to other levels.
1594
+ * @brief Integer for the number of node neighbors at a specific level of the
1595
+ * multi-level graph. It's selected to be `std::uint32_t` to improve the
1596
+ * alignment in most common cases.
1632
1597
  */
1633
- static constexpr std::size_t base_level_multiple_() { return 2; }
1598
+ using neighbors_count_t = std::uint32_t;
1599
+ using level_t = std::int16_t;
1634
1600
 
1635
1601
  /**
1636
1602
  * @brief How many bytes of memory are needed to form the "head" of the node.
1637
1603
  */
1638
- static constexpr std::size_t node_head_bytes_() { return sizeof(label_t) + sizeof(dim_t) + sizeof(level_t); }
1604
+ static constexpr std::size_t node_head_bytes_() { return sizeof(key_t) + sizeof(level_t); }
1639
1605
 
1640
- using visits_bitset_t = visits_bitset_gt<allocator_t>;
1606
+ using visits_bitset_t = visits_bitset_gt<dynamic_allocator_t>;
1641
1607
 
1642
1608
  struct precomputed_constants_t {
1643
1609
  double inverse_log_connectivity{};
1644
- std::size_t connectivity_max_base{};
1645
1610
  std::size_t neighbors_bytes{};
1646
1611
  std::size_t neighbors_base_bytes{};
1647
1612
  };
1613
+ /// @brief A space-efficient internal data-structure used in graph traversal queues.
1648
1614
  struct candidate_t {
1649
1615
  distance_t distance;
1650
- id_t id;
1651
- };
1652
- struct compare_by_distance_t {
1653
- inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
1616
+ compressed_slot_t slot;
1617
+ inline bool operator<(candidate_t other) const noexcept { return distance < other.distance; }
1654
1618
  };
1655
1619
 
1656
1620
  using candidates_view_t = span_gt<candidate_t const>;
1657
- using candidates_allocator_t = typename allocator_traits_t::template rebind_alloc<candidate_t>;
1658
- using top_candidates_t = sorted_buffer_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
1659
- using next_candidates_t = max_heap_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
1621
+ using candidates_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<candidate_t>;
1622
+ using top_candidates_t = sorted_buffer_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
1623
+ using next_candidates_t = max_heap_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
1660
1624
 
1661
1625
  /**
1662
- * @brief A loosely-structured handle for every node. One such node is created for every vector.
1626
+ * @brief A loosely-structured handle for every node. One such node is created for every member.
1663
1627
  * To minimize memory usage and maximize the number of entries per cache-line, it only
1664
- * stores to pointers. To reinterpret those into a structured `node_t` more
1665
- * information is needed from the parent `index_gt`.
1628
+ * stores to pointers. The internal tape starts with a `key_t` @b key, then
1629
+ * a `level_t` for the number of graph @b levels in which this member appears,
1630
+ * then the { `neighbors_count_t`, `compressed_slot_t`, `compressed_slot_t` ... } sequences
1631
+ * for @b each-level.
1666
1632
  */
1667
1633
  class node_t {
1668
1634
  byte_t* tape_{};
1669
- scalar_t* vector_{};
1670
1635
 
1671
1636
  public:
1672
- explicit node_t(byte_t* tape, scalar_t* vector) noexcept : tape_(tape), vector_(vector) {}
1637
+ explicit node_t(byte_t* tape) noexcept : tape_(tape) {}
1673
1638
  byte_t* tape() const noexcept { return tape_; }
1674
- scalar_t* vector() const noexcept { return vector_; }
1675
1639
  byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
1640
+ explicit operator bool() const noexcept { return tape_; }
1676
1641
 
1677
1642
  node_t() = default;
1678
1643
  node_t(node_t const&) = default;
1679
1644
  node_t& operator=(node_t const&) = default;
1680
1645
 
1681
- misaligned_ref_gt<label_t> label() const noexcept { return {tape_}; }
1682
- misaligned_ref_gt<dim_t> dim() const noexcept { return {tape_ + sizeof(label_t)}; }
1683
- misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(label_t) + sizeof(dim_t)}; }
1646
+ misaligned_ref_gt<key_t const> ckey() const noexcept { return {tape_}; }
1647
+ misaligned_ref_gt<key_t> key() const noexcept { return {tape_}; }
1648
+ misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(key_t)}; }
1684
1649
 
1685
- void label(label_t v) noexcept { return misaligned_store<label_t>(tape_, v); }
1686
- void dim(dim_t v) noexcept { return misaligned_store<dim_t>(tape_ + sizeof(label_t), v); }
1687
- void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
1688
-
1689
- operator vector_view_t() const noexcept { return {vector(), dim()}; }
1690
- vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
1691
- explicit operator member_ref_t() noexcept { return {{tape_}, vector_view()}; }
1692
- explicit operator member_cref_t() const noexcept { return {label(), vector_view()}; }
1693
- explicit operator bool() const noexcept { return tape_; }
1650
+ void key(key_t v) noexcept { return misaligned_store<key_t>(tape_, v); }
1651
+ void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(key_t), v); }
1694
1652
  };
1695
1653
 
1654
+ static_assert(std::is_trivially_copy_constructible<node_t>::value, "Nodes must be light!");
1655
+ static_assert(std::is_trivially_destructible<node_t>::value, "Nodes must be light!");
1656
+
1696
1657
  /**
1697
1658
  * @brief A slice of the node's tape, containing a the list of neighbors
1698
1659
  * for a node in a single graph level. It's pre-allocated to fit
1699
- * as many neighbors IDs, as may be needed at the target level,
1660
+ * as many neighbors "slots", as may be needed at the target level,
1700
1661
  * and starts with a single integer `neighbors_count_t` counter.
1701
1662
  */
1702
1663
  class neighbors_ref_t {
1703
1664
  byte_t* tape_;
1704
1665
 
1705
- static constexpr std::size_t shift(std::size_t i = 0) { return sizeof(neighbors_count_t) + sizeof(id_t) * i; }
1666
+ static constexpr std::size_t shift(std::size_t i = 0) {
1667
+ return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
1668
+ }
1706
1669
 
1707
1670
  public:
1708
1671
  neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
1709
- misaligned_ptr_gt<id_t> begin() noexcept { return tape_ + shift(); }
1710
- misaligned_ptr_gt<id_t> end() noexcept { return begin() + size(); }
1711
- misaligned_ptr_gt<id_t const> begin() const noexcept { return tape_ + shift(); }
1712
- misaligned_ptr_gt<id_t const> end() const noexcept { return begin() + size(); }
1713
- id_t operator[](std::size_t i) const noexcept { return misaligned_load<id_t>(tape_ + shift(i)); }
1672
+ misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
1673
+ misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
1674
+ misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
1675
+ misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
1676
+ compressed_slot_t operator[](std::size_t i) const noexcept {
1677
+ return misaligned_load<compressed_slot_t>(tape_ + shift(i));
1678
+ }
1714
1679
  std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
1715
- void clear() noexcept { misaligned_store<neighbors_count_t>(tape_, 0); }
1716
- void push_back(id_t id) noexcept {
1680
+ void clear() noexcept {
1681
+ neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1682
+ std::memset(tape_, 0, shift(n));
1683
+ // misaligned_store<neighbors_count_t>(tape_, 0);
1684
+ }
1685
+ void push_back(compressed_slot_t slot) noexcept {
1717
1686
  neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1718
- misaligned_store<id_t>(tape_ + shift(n), id);
1687
+ misaligned_store<compressed_slot_t>(tape_ + shift(n), slot);
1719
1688
  misaligned_store<neighbors_count_t>(tape_, n + 1);
1720
1689
  }
1721
1690
  };
@@ -1730,44 +1699,72 @@ class index_gt {
1730
1699
  next_candidates_t next_candidates{};
1731
1700
  visits_bitset_t visits{};
1732
1701
  std::default_random_engine level_generator{};
1733
- metric_t metric{};
1734
1702
  std::size_t iteration_cycles{};
1735
- std::size_t measurements_count{};
1703
+ std::size_t computed_distances_count{};
1704
+
1705
+ template <typename value_at, typename metric_at, typename entry_at> //
1706
+ inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
1707
+ static_assert( //
1708
+ std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
1709
+ "Unexpected type");
1736
1710
 
1737
- inline distance_t measure(vector_view_t a, vector_view_t b) noexcept {
1738
- measurements_count++;
1739
- return metric(a, b);
1711
+ computed_distances_count++;
1712
+ return metric(first, second);
1713
+ }
1714
+
1715
+ template <typename metric_at, typename entry_at> //
1716
+ inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
1717
+ static_assert( //
1718
+ std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
1719
+ "Unexpected type");
1720
+
1721
+ computed_distances_count++;
1722
+ return metric(first, second);
1740
1723
  }
1741
1724
  };
1742
1725
 
1743
1726
  index_config_t config_{};
1744
1727
  index_limits_t limits_{};
1745
- metric_t metric_{};
1746
- mutable allocator_t allocator_{};
1747
- point_allocator_t point_allocator_{};
1728
+
1729
+ mutable dynamic_allocator_t dynamic_allocator_{};
1730
+ tape_allocator_t tape_allocator_{};
1731
+
1748
1732
  precomputed_constants_t pre_{};
1749
- viewed_file_t viewed_file_{};
1733
+ memory_mapped_file_t viewed_file_{};
1734
+
1735
+ /// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
1736
+ usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
1750
1737
 
1751
- usearch_align_m mutable std::atomic<std::size_t> capacity_{};
1752
- usearch_align_m mutable std::atomic<std::size_t> size_{};
1738
+ /// @brief Number of "slots" already storing non-null nodes.
1739
+ usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
1753
1740
 
1754
- /// @brief Controls access to `max_level_` and `entry_id_`.
1741
+ /// @brief Controls access to `max_level_` and `entry_slot_`.
1755
1742
  /// If any thread is updating those values, no other threads can `add()` or `search()`.
1756
1743
  std::mutex global_mutex_{};
1744
+
1745
+ /// @brief The level of the top-most graph in the index. Grows as the logarithm of size, starts from zero.
1757
1746
  level_t max_level_{};
1758
- id_t entry_id_{};
1759
1747
 
1760
- using node_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
1761
- node_t* nodes_{};
1748
+ /// @brief The slot in which the only node of the top-level graph is stored.
1749
+ std::size_t entry_slot_{};
1750
+
1751
+ using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
1752
+
1753
+ /// @brief C-style array of `node_t` smart-pointers.
1754
+ buffer_gt<node_t, nodes_allocator_t> nodes_{};
1755
+
1756
+ /// @brief Mutex, that limits concurrent access to `nodes_`.
1762
1757
  mutable visits_bitset_t nodes_mutexes_{};
1763
1758
 
1764
- using context_allocator_t = typename allocator_traits_t::template rebind_alloc<context_t>;
1765
- context_t* contexts_{};
1759
+ using contexts_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<context_t>;
1760
+
1761
+ /// @brief Array of thread-specific buffers for temporary data.
1762
+ mutable buffer_gt<context_t, contexts_allocator_t> contexts_{};
1766
1763
 
1767
1764
  public:
1768
1765
  std::size_t connectivity() const noexcept { return config_.connectivity; }
1769
- std::size_t capacity() const noexcept { return capacity_; }
1770
- std::size_t size() const noexcept { return size_; }
1766
+ std::size_t capacity() const noexcept { return nodes_capacity_; }
1767
+ std::size_t size() const noexcept { return nodes_count_; }
1771
1768
  std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
1772
1769
  index_config_t const& config() const noexcept { return config_; }
1773
1770
  index_limits_t const& limits() const noexcept { return limits_; }
@@ -1777,16 +1774,17 @@ class index_gt {
1777
1774
  * @section Exceptions
1778
1775
  * Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
1779
1776
  */
1780
- explicit index_gt(index_config_t config = {}, metric_t metric = {}, allocator_t allocator = {},
1781
- point_allocator_t point_allocator = {}) noexcept
1782
- : config_(config), limits_(0, 0), metric_(metric), allocator_(std::move(allocator)),
1783
- point_allocator_(std::move(point_allocator)), pre_(precompute_(config)), size_(0u), max_level_(-1),
1784
- entry_id_(0u), nodes_(nullptr), nodes_mutexes_(), contexts_(nullptr) {}
1777
+ explicit index_gt( //
1778
+ index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
1779
+ tape_allocator_t tape_allocator = {}) noexcept
1780
+ : config_(config), limits_(0, 0), dynamic_allocator_(std::move(dynamic_allocator)),
1781
+ tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), nodes_count_(0u), max_level_(-1),
1782
+ entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
1785
1783
 
1786
1784
  /**
1787
1785
  * @brief Clones the structure with the same hyper-parameters, but without contents.
1788
1786
  */
1789
- index_gt fork() noexcept { return index_gt{config_, metric_, allocator_}; }
1787
+ index_gt fork() noexcept { return index_gt{config_, dynamic_allocator_, tape_allocator_}; }
1790
1788
 
1791
1789
  ~index_gt() noexcept { reset(); }
1792
1790
 
@@ -1808,21 +1806,21 @@ class index_gt {
1808
1806
  }
1809
1807
  };
1810
1808
 
1811
- copy_result_t copy(copy_config_t /*config*/ = {}) const noexcept {
1809
+ copy_result_t copy(index_copy_config_t /*config*/ = {}) const noexcept {
1812
1810
  copy_result_t result;
1813
1811
  index_gt& other = result.index;
1814
- other = index_gt(config_, metric_, allocator_, point_allocator_);
1812
+ other = index_gt(config_, dynamic_allocator_, tape_allocator_);
1815
1813
  if (!other.reserve(limits_))
1816
1814
  return result.failed("Failed to reserve the contexts");
1817
1815
 
1818
1816
  // Now all is left - is to allocate new `node_t` instances and populate
1819
1817
  // the `other.nodes_` array into it.
1820
- for (std::size_t i = 0; i != size_; ++i)
1821
- other.nodes_[i] = other.node_make_copy_(node_bytes_split_(nodes_[i]));
1818
+ for (std::size_t i = 0; i != nodes_count_; ++i)
1819
+ other.nodes_[i] = other.node_make_copy_(node_bytes_(nodes_[i]));
1822
1820
 
1823
- other.size_ = size_.load();
1821
+ other.nodes_count_ = nodes_count_.load();
1824
1822
  other.max_level_ = max_level_;
1825
- other.entry_id_ = entry_id_;
1823
+ other.entry_slot_ = entry_slot_;
1826
1824
  return result;
1827
1825
  }
1828
1826
 
@@ -1833,43 +1831,51 @@ class index_gt {
1833
1831
  member_iterator_t begin() noexcept { return {this, 0}; }
1834
1832
  member_iterator_t end() noexcept { return {this, size()}; }
1835
1833
 
1836
- member_cref_t at(std::size_t i) const noexcept { return member_cref_t(node_with_id_(i)); }
1837
- member_ref_t at(std::size_t i) noexcept { return member_ref_t(node_with_id_(i)); }
1834
+ member_ref_t at(std::size_t slot) noexcept { return {nodes_[slot].key(), slot}; }
1835
+ member_cref_t at(std::size_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
1836
+ member_iterator_t iterator_at(std::size_t slot) noexcept { return {this, slot}; }
1837
+ member_citerator_t citerator_at(std::size_t slot) const noexcept { return {this, slot}; }
1838
+
1839
+ dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
1840
+ tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
1838
1841
 
1839
1842
  #pragma region Adjusting Configuration
1840
1843
 
1841
1844
  /**
1842
- * @brief Erases all the vectors from the index.
1843
- * Will change `size()` to zero, but will keep the same `capacity()`.
1844
- * Will keep the number of available threads/contexts the same as it was.
1845
+ * @brief Erases all the vectors from the index.
1846
+ *
1847
+ * Will change `size()` to zero, but will keep the same `capacity()`.
1848
+ * Will keep the number of available threads/contexts the same as it was.
1845
1849
  */
1846
1850
  void clear() noexcept {
1847
- std::size_t n = size_;
1848
- for (std::size_t i = 0; i != n; ++i)
1849
- node_free_(i);
1850
- size_ = 0;
1851
+ if (!has_reset<tape_allocator_t>()) {
1852
+ std::size_t n = nodes_count_;
1853
+ for (std::size_t i = 0; i != n; ++i)
1854
+ node_free_(i);
1855
+ } else
1856
+ tape_allocator_.deallocate(nullptr, 0);
1857
+ nodes_count_ = 0;
1851
1858
  max_level_ = -1;
1852
- entry_id_ = 0u;
1859
+ entry_slot_ = 0u;
1853
1860
  }
1854
1861
 
1855
1862
  /**
1856
- * @brief Erases all the vectors from the index, also deallocating the registry.
1857
- * Will change both `size()` and `capacity()` to zero.
1858
- * Will deallocate all threads/contexts.
1863
+ * @brief Erases all members from index, closing files, and returning RAM to OS.
1864
+ *
1865
+ * Will change both `size()` and `capacity()` to zero.
1866
+ * Will deallocate all threads/contexts.
1867
+ * If the index is memory-mapped - releases the mapping and the descriptor.
1859
1868
  */
1860
1869
  void reset() noexcept {
1861
1870
  clear();
1862
1871
 
1863
- if (nodes_)
1864
- node_allocator_t{}.deallocate(exchange(nodes_, nullptr), limits_.members);
1865
- if (contexts_) {
1866
- for (std::size_t i = 0; i != limits_.threads(); ++i)
1867
- contexts_[i].~context_t();
1868
- context_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
1869
- }
1872
+ nodes_ = {};
1873
+ contexts_ = {};
1874
+ nodes_mutexes_ = {};
1870
1875
  limits_ = index_limits_t{0, 0};
1871
- capacity_ = 0;
1872
- reset_view_();
1876
+ nodes_capacity_ = 0;
1877
+ viewed_file_ = memory_mapped_file_t{};
1878
+ tape_allocator_ = {};
1873
1879
  }
1874
1880
 
1875
1881
  /**
@@ -1878,24 +1884,23 @@ class index_gt {
1878
1884
  void swap(index_gt& other) noexcept {
1879
1885
  std::swap(config_, other.config_);
1880
1886
  std::swap(limits_, other.limits_);
1881
- std::swap(metric_, other.metric_);
1882
- std::swap(allocator_, other.allocator_);
1883
- std::swap(point_allocator_, other.point_allocator_);
1887
+ std::swap(dynamic_allocator_, other.dynamic_allocator_);
1888
+ std::swap(tape_allocator_, other.tape_allocator_);
1884
1889
  std::swap(pre_, other.pre_);
1885
1890
  std::swap(viewed_file_, other.viewed_file_);
1886
1891
  std::swap(max_level_, other.max_level_);
1887
- std::swap(entry_id_, other.entry_id_);
1892
+ std::swap(entry_slot_, other.entry_slot_);
1888
1893
  std::swap(nodes_, other.nodes_);
1889
1894
  std::swap(nodes_mutexes_, other.nodes_mutexes_);
1890
1895
  std::swap(contexts_, other.contexts_);
1891
1896
 
1892
1897
  // Non-atomic parts.
1893
- std::size_t capacity_copy = capacity_;
1894
- std::size_t size_copy = size_;
1895
- capacity_ = other.capacity_.load();
1896
- size_ = other.size_.load();
1897
- other.capacity_ = capacity_copy;
1898
- other.size_ = size_copy;
1898
+ std::size_t capacity_copy = nodes_capacity_;
1899
+ std::size_t count_copy = nodes_count_;
1900
+ nodes_capacity_ = other.nodes_capacity_.load();
1901
+ nodes_count_ = other.nodes_count_.load();
1902
+ other.nodes_capacity_ = capacity_copy;
1903
+ other.nodes_count_ = count_copy;
1899
1904
  }
1900
1905
 
1901
1906
  /**
@@ -1909,72 +1914,30 @@ class index_gt {
1909
1914
  && limits.members <= limits_.members)
1910
1915
  return true;
1911
1916
 
1912
- if (!nodes_mutexes_.resize(limits.members))
1917
+ visits_bitset_t new_mutexes(limits.members);
1918
+ buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
1919
+ buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
1920
+ if (!new_nodes || !new_contexts || !new_mutexes)
1913
1921
  return false;
1914
1922
 
1915
- node_allocator_t node_allocator;
1916
- node_t* new_nodes = node_allocator.allocate(limits.members);
1917
- if (!new_nodes)
1918
- return false;
1919
-
1920
- std::size_t limits_threads = limits.threads();
1921
- context_allocator_t context_allocator;
1922
- context_t* new_contexts = context_allocator.allocate(limits_threads);
1923
- if (!new_contexts) {
1924
- node_allocator.deallocate(new_nodes, limits.members);
1925
- return false;
1926
- }
1927
- for (std::size_t i = 0; i != limits_threads; ++i) {
1928
- context_t& context = new_contexts[i];
1929
- new (&context) context_t();
1930
- context.metric = metric_;
1931
- if (!context.visits.resize(limits.members)) {
1932
- for (std::size_t j = 0; j != i; ++j)
1933
- context.visits.reset();
1934
- node_allocator.deallocate(new_nodes, limits.members);
1935
- context_allocator.deallocate(new_contexts, limits_threads);
1923
+ for (context_t& context : new_contexts) {
1924
+ context.visits = visits_bitset_t(limits.members);
1925
+ if (!context.visits)
1936
1926
  return false;
1937
- }
1938
- }
1939
-
1940
- // We have passed all the require memory allocations.
1941
- // The remaining code can't fail. Let's just reuse some of our existing buffers.
1942
- for (std::size_t i = 0; i != limits_.threads(); ++i) {
1943
- context_t& old_context = contexts_[i];
1944
- context_t& context = new_contexts[i];
1945
- std::swap(old_context.top_candidates, context.top_candidates);
1946
- std::swap(old_context.next_candidates, context.next_candidates);
1947
- std::swap(old_context.iteration_cycles, context.iteration_cycles);
1948
- std::swap(old_context.measurements_count, context.measurements_count);
1949
- old_context.visits.reset();
1950
1927
  }
1951
1928
 
1952
1929
  // Move the nodes info, and deallocate previous buffers.
1953
1930
  if (nodes_)
1954
- std::memcpy(new_nodes, nodes_, sizeof(node_t) * size()), node_allocator.deallocate(nodes_, limits_.members);
1955
- if (contexts_)
1956
- context_allocator.deallocate(contexts_, limits_.threads());
1931
+ std::memcpy(new_nodes.data(), nodes_.data(), sizeof(node_t) * size());
1957
1932
 
1958
1933
  limits_ = limits;
1959
- capacity_ = limits.members;
1960
- nodes_ = new_nodes;
1961
- contexts_ = new_contexts;
1934
+ nodes_capacity_ = limits.members;
1935
+ nodes_ = std::move(new_nodes);
1936
+ contexts_ = std::move(new_contexts);
1937
+ nodes_mutexes_ = std::move(new_mutexes);
1962
1938
  return true;
1963
1939
  }
1964
1940
 
1965
- /**
1966
- * @brief Optimizes configuration options to fit the maximum number
1967
- * of neighbors in CPU-cache-aligned buffers.
1968
- */
1969
- static index_config_t optimize(index_config_t config) noexcept {
1970
- precomputed_constants_t pre = precompute_(config);
1971
- std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
1972
- std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
1973
- std::size_t added_connections = (rounded_size - bytes_per_node_base) / sizeof(id_t);
1974
- config.connectivity = config.connectivity + added_connections / base_level_multiple_();
1975
- return config;
1976
- }
1977
-
1978
1941
  #pragma endregion
1979
1942
 
1980
1943
  #pragma region Construction and Search
@@ -1982,9 +1945,9 @@ class index_gt {
1982
1945
  struct add_result_t {
1983
1946
  error_t error{};
1984
1947
  std::size_t new_size{};
1985
- std::size_t cycles{};
1986
- std::size_t measurements{};
1987
- id_t id{};
1948
+ std::size_t visited_members{};
1949
+ std::size_t computed_distances{};
1950
+ std::size_t slot{};
1988
1951
 
1989
1952
  explicit operator bool() const noexcept { return !error; }
1990
1953
  add_result_t failed(error_t message) noexcept {
@@ -1993,29 +1956,31 @@ class index_gt {
1993
1956
  }
1994
1957
  };
1995
1958
 
1959
+ /// @brief Describes a matched search result, augmenting `member_cref_t`
1960
+ /// contents with `distance` to the query object.
1996
1961
  struct match_t {
1997
1962
  member_cref_t member;
1998
1963
  distance_t distance;
1999
1964
  };
2000
1965
 
2001
1966
  class search_result_t {
2002
- index_gt const& index_;
2003
- top_candidates_t& top_;
1967
+ node_t const* nodes_{};
1968
+ top_candidates_t const* top_{};
2004
1969
 
2005
1970
  friend class index_gt;
2006
- inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
1971
+ inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
1972
+ : nodes_(index.nodes_), top_(&top) {}
2007
1973
 
2008
1974
  public:
1975
+ /** @brief Number of search results found. */
2009
1976
  std::size_t count{};
2010
- std::size_t cycles{};
2011
- std::size_t measurements{};
1977
+ /** @brief Number of graph nodes traversed. */
1978
+ std::size_t visited_members{};
1979
+ /** @brief Number of times the distances were computed. */
1980
+ std::size_t computed_distances{};
2012
1981
  error_t error{};
2013
1982
 
2014
- // todo: this is a hack to get a handle to a failed search_result_t without changing its member types
2015
- // given that search_result_t can represent a failed result (which implies no valid top_candidates_t &top_),
2016
- // the top_ probably needs to be a pointer.
2017
- inline search_result_t(index_gt const& index) noexcept
2018
- : index_(index), top_(index.contexts_[0].top_candidates) {}
1983
+ inline search_result_t() noexcept {}
2019
1984
  inline search_result_t(search_result_t&&) = default;
2020
1985
  inline search_result_t& operator=(search_result_t&&) = default;
2021
1986
 
@@ -2031,46 +1996,88 @@ class index_gt {
2031
1996
  inline match_t operator[](std::size_t i) const noexcept { return at(i); }
2032
1997
  inline match_t front() const noexcept { return at(0); }
2033
1998
  inline match_t back() const noexcept { return at(count - 1); }
2034
- inline bool contains(label_t label) const noexcept {
1999
+ inline bool contains(key_t key) const noexcept {
2035
2000
  for (std::size_t i = 0; i != count; ++i)
2036
- if (at(i).member.label == label)
2001
+ if (at(i).member.key == key)
2037
2002
  return true;
2038
2003
  return false;
2039
2004
  }
2040
2005
  inline match_t at(std::size_t i) const noexcept {
2041
- candidate_t const* top_ordered = top_.data();
2006
+ candidate_t const* top_ordered = top_->data();
2042
2007
  candidate_t candidate = top_ordered[i];
2043
- node_t node = index_.node_with_id_(candidate.id);
2044
- return {member_cref_t{node.label(), node.vector_view(), candidate.id}, candidate.distance};
2008
+ node_t node = nodes_[candidate.slot];
2009
+ return {member_cref_t{node.ckey(), candidate.slot}, candidate.distance};
2010
+ }
2011
+ inline std::size_t merge_into( //
2012
+ key_t* keys, distance_t* distances, //
2013
+ std::size_t old_count, std::size_t max_count) const noexcept {
2014
+
2015
+ std::size_t merged_count = old_count;
2016
+ for (std::size_t i = 0; i != count; ++i) {
2017
+ match_t result = operator[](i);
2018
+ distance_t* merged_end = distances + merged_count;
2019
+ std::size_t offset = std::lower_bound(distances, merged_end, result.distance) - distances;
2020
+ if (offset == max_count)
2021
+ continue;
2022
+
2023
+ std::size_t count_worse = merged_count - offset - (max_count == merged_count);
2024
+ std::memmove(keys + offset + 1, keys + offset, count_worse * sizeof(key_t));
2025
+ std::memmove(distances + offset + 1, distances + offset, count_worse * sizeof(distance_t));
2026
+ keys[offset] = result.member.key;
2027
+ distances[offset] = result.distance;
2028
+ merged_count += merged_count != max_count;
2029
+ }
2030
+ return merged_count;
2045
2031
  }
2046
- inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
2032
+ inline std::size_t dump_to(key_t* keys, distance_t* distances) const noexcept {
2047
2033
  for (std::size_t i = 0; i != count; ++i) {
2048
2034
  match_t result = operator[](i);
2049
- labels[i] = result.member.label;
2035
+ keys[i] = result.member.key;
2050
2036
  distances[i] = result.distance;
2051
2037
  }
2052
2038
  return count;
2053
2039
  }
2054
- inline std::size_t dump_to(label_t* labels) const noexcept {
2040
+ inline std::size_t dump_to(key_t* keys) const noexcept {
2055
2041
  for (std::size_t i = 0; i != count; ++i) {
2056
2042
  match_t result = operator[](i);
2057
- labels[i] = result.member.label;
2043
+ keys[i] = result.member.key;
2058
2044
  }
2059
2045
  return count;
2060
2046
  }
2061
2047
  };
2062
2048
 
2063
2049
  /**
2064
- * @brief Inserts a new vector into the index. Thread-safe.
2050
+ * @brief Inserts a new entry into the index. Thread-safe. Supports @b heterogeneous lookups.
2051
+ * Expects needed capacity to be reserved ahead of time: `size() < capacity()`.
2052
+ *
2053
+ * @tparam metric_at
2054
+ * A function responsible for computing the distance @b (dis-similarity) between two objects.
2055
+ * It should be callable into distinctly different scenarios:
2056
+ * - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
2057
+ * - `distance_t operator() (entry_at, entry_at)` - between existing entries.
2058
+ * Where any possible `entry_at` has both two interfaces: `std::size_t slot()`, `key_t key()`.
2065
2059
  *
2066
- * @param[in] label External identifier/name/descriptor for the vector.
2067
- * @param[in] vector Contiguous range of scalars forming a vector view.
2060
+ * @param[in] key External identifier/name/descriptor for the new entry.
2061
+ * @param[in] value Content that will be compared against other entries to index.
2062
+ * @param[in] metric Callable object measuring distance between ::value and present objects.
2068
2063
  * @param[in] config Configuration options for this specific operation.
2064
+ * @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
2069
2065
  */
2070
- add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) usearch_noexcept_m {
2066
+ template < //
2067
+ typename value_at, //
2068
+ typename metric_at, //
2069
+ typename callback_at = dummy_callback_t, //
2070
+ typename prefetch_at = dummy_prefetch_t //
2071
+ >
2072
+ add_result_t add( //
2073
+ key_t key, value_at&& value, metric_at&& metric, //
2074
+ index_update_config_t config = {}, //
2075
+ callback_at&& callback = callback_at{}, //
2076
+ prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
2071
2077
 
2072
- usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
2073
2078
  add_result_t result;
2079
+ if (is_immutable())
2080
+ return result.failed("Can't add to an immutable index");
2074
2081
 
2075
2082
  // Make sure we have enough local memory to perform this request
2076
2083
  context_t& context = contexts_[config.thread];
@@ -2081,7 +2088,8 @@ class index_gt {
2081
2088
 
2082
2089
  // The top list needs one more slot than the connectivity of the base level
2083
2090
  // for the heuristic, that tries to squeeze one more element into saturated list.
2084
- std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config.expansion);
2091
+ std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
2092
+ std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
2085
2093
  if (!top.reserve(top_limit))
2086
2094
  return result.failed("Out of memory!");
2087
2095
  if (!next.reserve(config.expansion))
@@ -2089,60 +2097,98 @@ class index_gt {
2089
2097
 
2090
2098
  // Determining how much memory to allocate for the node depends on the target level
2091
2099
  std::unique_lock<std::mutex> new_level_lock(global_mutex_);
2092
- level_t max_level_copy = max_level_; // Copy under lock
2093
- id_t entry_id_copy = entry_id_; // Copy under lock
2100
+ level_t max_level_copy = max_level_; // Copy under lock
2101
+ std::size_t entry_idx_copy = entry_slot_; // Copy under lock
2094
2102
  level_t target_level = choose_random_level_(context.level_generator);
2095
- if (target_level <= max_level_copy)
2096
- new_level_lock.unlock();
2103
+
2104
+ // Make sure we are not overflowing
2105
+ std::size_t capacity = nodes_capacity_.load();
2106
+ std::size_t new_slot = nodes_count_.fetch_add(1);
2107
+ if (new_slot >= capacity) {
2108
+ nodes_count_.fetch_sub(1);
2109
+ return result.failed("Reserve capacity ahead of insertions!");
2110
+ }
2097
2111
 
2098
2112
  // Allocate the neighbors
2099
- node_t node = node_make_(label, vector, target_level, config.store_vector);
2100
- if (!node)
2113
+ node_t node = node_make_(key, target_level);
2114
+ if (!node) {
2115
+ nodes_count_.fetch_sub(1);
2101
2116
  return result.failed("Out of memory!");
2102
- std::size_t old_size = size_.fetch_add(1);
2103
- id_t new_id = static_cast<id_t>(old_size);
2104
- nodes_[old_size] = node;
2105
- result.new_size = old_size + 1;
2106
- result.id = new_id;
2107
- node_lock_t new_lock = node_lock_(old_size);
2117
+ }
2118
+ if (target_level <= max_level_copy)
2119
+ new_level_lock.unlock();
2120
+
2121
+ nodes_[new_slot] = node;
2122
+ result.new_size = new_slot + 1;
2123
+ result.slot = new_slot;
2124
+ callback(at(new_slot));
2125
+ node_lock_t new_lock = node_lock_(new_slot);
2108
2126
 
2109
2127
  // Do nothing for the first element
2110
- if (!new_id) {
2111
- entry_id_ = new_id;
2128
+ if (!new_slot) {
2129
+ entry_slot_ = new_slot;
2112
2130
  max_level_ = target_level;
2113
2131
  return result;
2114
2132
  }
2115
2133
 
2116
2134
  // Pull stats
2117
- result.measurements = context.measurements_count;
2118
- result.cycles = context.iteration_cycles;
2135
+ result.computed_distances = context.computed_distances_count;
2136
+ result.visited_members = context.iteration_cycles;
2119
2137
 
2120
- connect_node_across_levels_(new_id, vector, entry_id_copy, max_level_copy, target_level, config, context);
2138
+ connect_node_across_levels_( //
2139
+ value, metric, prefetch, //
2140
+ new_slot, entry_idx_copy, max_level_copy, target_level, //
2141
+ config, context);
2121
2142
 
2122
2143
  // Normalize stats
2123
- result.measurements = context.measurements_count - result.measurements;
2124
- result.cycles = context.iteration_cycles - result.cycles;
2144
+ result.computed_distances = context.computed_distances_count - result.computed_distances;
2145
+ result.visited_members = context.iteration_cycles - result.visited_members;
2125
2146
 
2126
2147
  // Updating the entry point if needed
2127
2148
  if (target_level > max_level_copy) {
2128
- entry_id_ = new_id;
2149
+ entry_slot_ = new_slot;
2129
2150
  max_level_ = target_level;
2130
2151
  }
2131
2152
  return result;
2132
2153
  }
2133
2154
 
2134
2155
  /**
2135
- * @brief Update an existing entry, replacing a vector and a label. Thread-safe.
2156
+ * @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
2136
2157
  *
2137
- * @param[in] old_id Existing internal identifier for a node to be replaced.
2138
- * @param[in] label External identifier/name/descriptor for the vector.
2139
- * @param[in] vector Contiguous range of scalars forming a vector view.
2158
+ * @tparam metric_at
2159
+ * A function responsible for computing the distance @b (dis-similarity) between two objects.
2160
+ * It should be callable into distinctly different scenarios:
2161
+ * - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
2162
+ * - `distance_t operator() (entry_at, entry_at)` - between existing entries.
2163
+ * For any possible `entry_at` following interfaces will work:
2164
+ * - `std::size_t get_slot(entry_at const &)`
2165
+ * - `key_t get_key(entry_at const &)`
2166
+ *
2167
+ * @param[in] iterator Iterator pointing to an existing entry to be replaced.
2168
+ * @param[in] key External identifier/name/descriptor for the entry.
2169
+ * @param[in] value Content that will be compared against other entries in the index.
2170
+ * @param[in] metric Callable object measuring distance between ::value and present objects.
2140
2171
  * @param[in] config Configuration options for this specific operation.
2172
+ * @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
2141
2173
  */
2142
- add_result_t update(id_t old_id, label_t label, vector_view_t vector, add_config_t config = {}) usearch_noexcept_m {
2174
+ template < //
2175
+ typename value_at, //
2176
+ typename metric_at, //
2177
+ typename callback_at = dummy_callback_t, //
2178
+ typename prefetch_at = dummy_prefetch_t //
2179
+ >
2180
+ add_result_t update( //
2181
+ member_iterator_t iterator, //
2182
+ key_t key, //
2183
+ value_at&& value, //
2184
+ metric_at&& metric, //
2185
+ index_update_config_t config = {}, //
2186
+ callback_at&& callback = callback_at{}, //
2187
+ prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
2143
2188
 
2144
2189
  usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
2145
2190
  add_result_t result;
2191
+ std::size_t old_slot = iterator.slot_;
2146
2192
 
2147
2193
  // Make sure we have enough local memory to perform this request
2148
2194
  context_t& context = contexts_[config.thread];
@@ -2153,57 +2199,77 @@ class index_gt {
2153
2199
 
2154
2200
  // The top list needs one more slot than the connectivity of the base level
2155
2201
  // for the heuristic, that tries to squeeze one more element into saturated list.
2156
- std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config.expansion);
2202
+ std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
2203
+ std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
2157
2204
  if (!top.reserve(top_limit))
2158
2205
  return result.failed("Out of memory!");
2159
2206
  if (!next.reserve(config.expansion))
2160
2207
  return result.failed("Out of memory!");
2161
2208
 
2162
- node_lock_t new_lock = node_lock_(old_id);
2163
- node_t node = node_with_id_(old_id);
2209
+ node_lock_t new_lock = node_lock_(old_slot);
2210
+ node_t node = node_at_(old_slot);
2211
+
2212
+ level_t node_level = node.level();
2213
+ span_bytes_t node_bytes = node_bytes_(node);
2214
+ std::memset(node_bytes.data(), 0, node_bytes.size());
2215
+ node.level(node_level);
2164
2216
 
2165
2217
  // Pull stats
2166
- result.measurements = context.measurements_count;
2167
- result.cycles = context.iteration_cycles;
2218
+ result.computed_distances = context.computed_distances_count;
2219
+ result.visited_members = context.iteration_cycles;
2168
2220
 
2169
- connect_node_across_levels_(old_id, vector, entry_id_, max_level_, node.level(), config, context);
2170
- node.label(label);
2221
+ connect_node_across_levels_( //
2222
+ value, metric, prefetch, //
2223
+ old_slot, entry_slot_, max_level_, node_level, //
2224
+ config, context);
2225
+ node.key(key);
2171
2226
 
2172
2227
  // Normalize stats
2173
- result.measurements = context.measurements_count - result.measurements;
2174
- result.cycles = context.iteration_cycles - result.cycles;
2228
+ result.computed_distances = context.computed_distances_count - result.computed_distances;
2229
+ result.visited_members = context.iteration_cycles - result.visited_members;
2230
+ result.slot = old_slot;
2175
2231
 
2232
+ callback(at(old_slot));
2176
2233
  return result;
2177
2234
  }
2178
2235
 
2179
2236
  /**
2180
2237
  * @brief Searches for the closest elements to the given ::query. Thread-safe.
2181
2238
  *
2182
- * @param[in] query Contiguous range of scalars forming a vector view.
2239
+ * @param[in] query Content that will be compared against other entries in the index.
2183
2240
  * @param[in] wanted The upper bound for the number of results to return.
2184
2241
  * @param[in] config Configuration options for this specific operation.
2185
2242
  * @param[in] predicate Optional filtering predicate for `member_cref_t`.
2186
2243
  * @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
2187
2244
  */
2188
- template <typename predicate_at = dummy_predicate_t>
2189
- search_result_t search( //
2190
- vector_view_t query, std::size_t wanted, search_config_t config = {},
2191
- predicate_at&& predicate = dummy_predicate_t{}) const noexcept {
2245
+ template < //
2246
+ typename value_at, //
2247
+ typename metric_at, //
2248
+ typename predicate_at = dummy_predicate_t, //
2249
+ typename prefetch_at = dummy_prefetch_t //
2250
+ >
2251
+ search_result_t search( //
2252
+ value_at&& query, //
2253
+ std::size_t wanted, //
2254
+ metric_at&& metric, //
2255
+ index_search_config_t config = {}, //
2256
+ predicate_at&& predicate = predicate_at{}, //
2257
+ prefetch_at&& prefetch = prefetch_at{}) const noexcept {
2192
2258
 
2193
2259
  context_t& context = contexts_[config.thread];
2194
2260
  top_candidates_t& top = context.top_candidates;
2195
2261
  search_result_t result{*this, top};
2196
- if (!size_)
2262
+ if (!nodes_count_)
2197
2263
  return result;
2198
2264
 
2199
2265
  // Go down the level, tracking only the closest match
2200
- result.measurements = context.measurements_count;
2201
- result.cycles = context.iteration_cycles;
2266
+ result.computed_distances = context.computed_distances_count;
2267
+ result.visited_members = context.iteration_cycles;
2202
2268
 
2203
2269
  if (config.exact) {
2204
2270
  if (!top.reserve(wanted))
2205
2271
  return result.failed("Out of memory!");
2206
- search_exact_(query, wanted, context, std::forward<predicate_at>(predicate));
2272
+ search_exact_(query, metric, predicate, wanted, context);
2207
2273
  } else {
2208
2274
  next_candidates_t& next = context.next_candidates;
2209
2275
  std::size_t expansion = (std::max)(config.expansion, wanted);
@@ -2212,9 +2278,10 @@ class index_gt {
2212
2278
  if (!top.reserve(expansion))
2213
2279
  return result.failed("Out of memory!");
2214
2280
 
2215
- id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
2281
+ std::size_t closest_slot = search_for_one_(query, metric, prefetch, entry_slot_, max_level_, 0, context);
2282
+
2216
2283
  // For bottom layer we need a more optimized procedure
2217
- if (!search_to_find_in_base_(closest_id, query, expansion, context, std::forward<predicate_at>(predicate)))
2284
+ if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
2218
2285
  return result.failed("Out of memory!");
2219
2286
  }
2220
2287
 
@@ -2222,8 +2289,8 @@ class index_gt {
2222
2289
  top.shrink(wanted);
2223
2290
 
2224
2291
  // Normalize stats
2225
- result.measurements = context.measurements_count - result.measurements;
2226
- result.cycles = context.iteration_cycles - result.cycles;
2292
+ result.computed_distances = context.computed_distances_count - result.computed_distances;
2293
+ result.visited_members = context.iteration_cycles - result.visited_members;
2227
2294
  result.count = top.size();
2228
2295
  return result;
2229
2296
  }
@@ -2243,13 +2310,13 @@ class index_gt {
2243
2310
  stats_t result{};
2244
2311
  result.nodes = size();
2245
2312
  for (std::size_t i = 0; i != result.nodes; ++i) {
2246
- node_t node = node_with_id_(i);
2247
- std::size_t max_edges = node.level() * config_.connectivity + base_level_multiple_() * config_.connectivity;
2313
+ node_t node = node_at_(i);
2314
+ std::size_t max_edges = node.level() * config_.connectivity + config_.connectivity_base;
2248
2315
  std::size_t edges = 0;
2249
2316
  for (level_t level = 0; level <= node.level(); ++level)
2250
2317
  edges += neighbors_(node, level).size();
2251
2318
 
2252
- result.allocated_bytes += node_bytes_(node);
2319
+ result.allocated_bytes += node_bytes_(node).size();
2253
2320
  result.edges += edges;
2254
2321
  result.max_edges += max_edges;
2255
2322
  }
@@ -2262,15 +2329,15 @@ class index_gt {
2262
2329
 
2263
2330
  std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
2264
2331
  for (std::size_t i = 0; i != result.nodes; ++i) {
2265
- node_t node = node_with_id_(i);
2332
+ node_t node = node_at_(i);
2266
2333
  if (static_cast<std::size_t>(node.level()) < level)
2267
2334
  continue;
2268
2335
 
2269
2336
  result.edges += neighbors_(node, level).size();
2270
- result.allocated_bytes += node_head_bytes_() + node_vector_bytes_(node) + neighbors_bytes;
2337
+ result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
2271
2338
  }
2272
2339
 
2273
- std::size_t max_edges_per_node = level ? config_.connectivity : base_level_multiple_() * config_.connectivity;
2340
+ std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
2274
2341
  result.max_edges = result.nodes * max_edges_per_node;
2275
2342
  return result;
2276
2343
  }
@@ -2278,6 +2345,8 @@ class index_gt {
2278
2345
  /**
2279
2346
  * @brief A relatively accurate lower bound on the amount of memory consumed by the system.
2280
2347
  * In practice it's error will be below 10%.
2348
+ *
2349
+ * @see `stream_length` for the length of the binary serialized representation.
2281
2350
  */
2282
2351
  std::size_t memory_usage(std::size_t allocator_entry_bytes = default_allocator_entry_bytes()) const noexcept {
2283
2352
  std::size_t total = 0;
@@ -2295,358 +2364,297 @@ class index_gt {
2295
2364
  return total;
2296
2365
  }
2297
2366
 
2298
- std::size_t memory_usage_per_node(dim_t dim, level_t level) const noexcept { return node_bytes_(dim, level); }
2299
-
2300
- void change_metric(metric_t const& m) noexcept {
2301
- metric_ = m;
2302
- for (std::size_t i = 0; i != limits_.threads(); ++i)
2303
- contexts_[i].metric = metric_;
2304
- }
2367
+ std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
2305
2368
 
2306
2369
  #pragma endregion
2307
2370
 
2308
2371
  #pragma region Serialization
2309
2372
 
2310
- struct serialization_result_t {
2311
- error_t error;
2312
-
2313
- explicit operator bool() const noexcept { return !error; }
2314
- serialization_result_t failed(error_t message) noexcept {
2315
- error = std::move(message);
2316
- return std::move(*this);
2317
- }
2318
- };
2319
-
2320
2373
  /**
2321
- * @brief Saves serialized binary index representation to disk,
2322
- * co-locating vectors and neighbors lists.
2323
- * Available on Linux, MacOS, Windows.
2374
+ * @brief Saves serialized binary index representation to a file, generally on disk.
2324
2375
  */
2325
2376
  template <typename progress_at = dummy_progress_t>
2326
- serialization_result_t save(char const* file_path, progress_at&& progress = {}) const noexcept {
2377
+ serialization_result_t save(output_file_t file, progress_at&& progress = {}) const noexcept {
2378
+
2379
+ serialization_result_t result = file.open_if_not();
2380
+ if (result)
2381
+ stream(
2382
+ [&](void* buffer, std::size_t length) {
2383
+ result = file.write(buffer, length);
2384
+ return !!result;
2385
+ },
2386
+ std::forward<progress_at>(progress));
2387
+ return result;
2388
+ }
2389
+
2390
+ /**
2391
+ * @brief Saves serialized binary index representation to a stream.
2392
+ */
2393
+ template <typename output_callback_at, typename progress_at = dummy_progress_t>
2394
+ serialization_result_t stream(output_callback_at&& callback, progress_at&& progress = {}) const noexcept {
2327
2395
 
2328
- // Make sure we have right to write to that file
2329
2396
  serialization_result_t result;
2330
- std::FILE* file = std::fopen(file_path, "wb");
2331
- if (!file)
2332
- return result.failed(std::strerror(errno));
2333
2397
 
2334
- // Prepare the header with metadata
2335
- file_header_t state_buffer{};
2336
- std::memset(state_buffer, 0, sizeof(file_header_t));
2337
- file_head_t state{state_buffer};
2338
- std::memcpy(state_buffer, default_magic(), std::strlen(default_magic()));
2339
-
2340
- // Mark compatibility
2341
- state.version_major = USEARCH_VERSION_MAJOR;
2342
- state.version_minor = USEARCH_VERSION_MINOR;
2343
- state.version_patch = USEARCH_VERSION_PATCH;
2344
- state.metric = metric_.kind();
2345
-
2346
- // Describe state
2347
- state.connectivity = config_.connectivity;
2348
- state.max_level = max_level_;
2349
- state.vector_alignment = config_.vector_alignment;
2350
- state.bytes_per_label = sizeof(label_t);
2351
- state.bytes_per_id = sizeof(id_t);
2352
- state.scalar_kind = metric_.scalar_kind();
2353
- state.size = size_;
2354
- state.entry_idx = entry_id_;
2355
-
2356
- // Augment with metadata
2357
- std::size_t graphs_bytes = 0;
2358
- std::size_t vectors_bytes = 0;
2359
- for (std::size_t i = 0; i != state.size; ++i) {
2360
- node_t node = node_with_id_(i);
2361
- std::size_t node_bytes = node_bytes_(node);
2362
- std::size_t node_vector_bytes = node_vector_bytes_(node);
2363
- graphs_bytes += node_bytes - node_vector_bytes;
2364
- vectors_bytes += node_vector_bytes;
2398
+ // Export some basic metadata
2399
+ index_serialized_header_t header;
2400
+ header.size = nodes_count_;
2401
+ header.connectivity = config_.connectivity;
2402
+ header.connectivity_base = config_.connectivity_base;
2403
+ header.max_level = max_level_;
2404
+ header.entry_slot = entry_slot_;
2405
+ if (!callback(&header, sizeof(header)))
2406
+ return result.failed("Failed to serialize into stream");
2407
+
2408
+ // Export the number of levels per node
2409
+ // That is both enough to estimate the overall memory consumption,
2410
+ // and to be able to estimate the offsets of every entry in the file.
2411
+ for (std::size_t i = 0; i != header.size; ++i) {
2412
+ node_t node = node_at_(i);
2413
+ level_t level = node.level();
2414
+ if (!callback(&level, sizeof(level)))
2415
+ return result.failed("Failed to serialize into stream");
2365
2416
  }
2366
- state.bytes_for_graphs = graphs_bytes;
2367
- state.bytes_for_vectors = vectors_bytes;
2368
- state.bytes_checksum = 0;
2369
-
2370
- // Perform serialization
2371
- auto write_chunk = [&](void* begin, std::size_t length) {
2372
- std::size_t written = std::fwrite(begin, length, 1, file);
2373
- if (!written) {
2374
- std::fclose(file);
2375
- result.failed(std::strerror(errno));
2376
- }
2377
- };
2378
-
2379
- // Write the header
2380
- write_chunk(&state_buffer[0], sizeof(file_header_t));
2381
- if (result.error)
2382
- return result;
2383
2417
 
2384
- // Serialize nodes one by one
2385
- for (std::size_t i = 0; i != state.size; ++i) {
2386
- node_t node = node_with_id_(i);
2387
- std::size_t node_bytes = node_bytes_(node);
2388
- std::size_t node_vector_bytes = node_vector_bytes_(node);
2389
- // Dump neighbors and vectors, as vectors may be in a disjoint location
2390
- write_chunk(node.tape(), node_bytes - node_vector_bytes);
2391
- if (result.error)
2392
- return result;
2393
- write_chunk(node.vector(), node_vector_bytes);
2394
- if (result.error)
2395
- return result;
2396
- progress(i, state.size);
2418
+ // After that dump the nodes themselves
2419
+ for (std::size_t i = 0; i != header.size; ++i) {
2420
+ span_bytes_t node_bytes = node_bytes_(node_at_(i));
2421
+ if (!callback(node_bytes.data(), node_bytes.size()))
2422
+ return result.failed("Failed to serialize into stream");
2423
+ progress(i, header.size);
2397
2424
  }
2398
2425
 
2399
- std::fclose(file);
2400
2426
  return {};
2401
2427
  }
2402
2428
 
2403
2429
  /**
2404
- * @brief Loads the serialized binary index representation from disk,
2405
- * copying both vectors and neighbors lists into RAM.
2406
- * Available on Linux, MacOS, Windows.
2430
+ * @brief Estimate the binary length (in bytes) of the serialized index.
2407
2431
  */
2408
- template <typename progress_at = dummy_progress_t>
2409
- serialization_result_t load(char const* file_path, progress_at&& progress = {}) noexcept {
2432
+ std::size_t stream_length() const noexcept {
2433
+ std::size_t neighbors_length = 0;
2434
+ for (std::size_t i = 0; i != size(); ++i)
2435
+ neighbors_length += node_bytes_(node_at_(i).level());
2436
+ return sizeof(index_serialized_header_t) + neighbors_length;
2437
+ }
2410
2438
 
2411
- serialization_result_t result;
2412
- file_header_t state_buffer{};
2413
- std::FILE* file = std::fopen(file_path, "rb");
2414
- if (!file)
2415
- return result.failed(std::strerror(errno));
2439
+ /**
2440
+ * @brief Loads the serialized binary index representation from disk to RAM.
2441
+ * Adjusts the configuration properties of the constructed index to
2442
+ * match the settings in the file.
2443
+ */
2444
+ template <typename progress_at = dummy_progress_t>
2445
+ serialization_result_t load(input_file_t file, progress_at&& progress = {}) noexcept {
2416
2446
 
2417
- auto read_chunk = [&](void* begin, std::size_t length) {
2418
- std::size_t read = std::fread(begin, length, 1, file);
2419
- if (!read) {
2420
- std::fclose(file);
2421
- result.failed(std::strerror(errno));
2422
- }
2423
- };
2447
+ // Remove previously stored objects
2448
+ reset();
2424
2449
 
2425
- // Read the header
2426
- {
2427
- read_chunk(&state_buffer[0], sizeof(file_header_t));
2428
- if (result.error)
2429
- return result;
2450
+ serialization_result_t result = file.open_if_not();
2451
+ if (!result)
2452
+ return result;
2430
2453
 
2431
- file_head_t state{state_buffer};
2432
- if (state.bytes_per_label != sizeof(label_t)) {
2433
- std::fclose(file);
2434
- return result.failed("Incompatible label type!");
2435
- }
2436
- if (state.bytes_per_id != sizeof(id_t)) {
2437
- std::fclose(file);
2438
- return result.failed("Incompatible ID type!");
2439
- }
2454
+ // Pull basic metadata
2455
+ index_serialized_header_t header;
2456
+ result = file.read(&header, sizeof(header));
2457
+ if (!result)
2458
+ return result;
2440
2459
 
2441
- config_.connectivity = state.connectivity;
2442
- config_.vector_alignment = state.vector_alignment;
2443
- pre_ = precompute_(config_);
2460
+ // Allocate some dynamic memory to read all the levels
2461
+ using levels_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<level_t>;
2462
+ buffer_gt<level_t, levels_allocator_t> levels(header.size);
2463
+ if (!levels)
2464
+ return result.failed("Out of memory");
2465
+ result = file.read(levels, header.size * sizeof(level_t));
2466
+ if (!result)
2467
+ return result;
2444
2468
 
2445
- index_limits_t limits;
2446
- limits.members = state.size;
2447
- if (!reserve(limits)) {
2448
- std::fclose(file);
2449
- return result.failed("Out of memory");
2450
- }
2451
- size_ = state.size;
2452
- max_level_ = static_cast<level_t>(state.max_level);
2453
- entry_id_ = static_cast<id_t>(state.entry_idx);
2469
+ // Submit metadata
2470
+ config_.connectivity = header.connectivity;
2471
+ config_.connectivity_base = header.connectivity_base;
2472
+ pre_ = precompute_(config_);
2473
+ index_limits_t limits;
2474
+ limits.members = header.size;
2475
+ if (!reserve(limits)) {
2476
+ reset();
2477
+ return result.failed("Out of memory");
2454
2478
  }
2455
-
2456
- // Load nodes one by one
2457
- std::size_t const size = size_;
2458
- for (std::size_t i = 0; i != size; ++i) {
2459
- label_t label;
2460
- dim_t dim;
2461
- level_t level;
2462
- read_chunk(&label, sizeof(label));
2463
- if (result.error)
2464
- return result;
2465
- read_chunk(&dim, sizeof(dim));
2466
- if (result.error)
2479
+ nodes_count_ = header.size;
2480
+ max_level_ = static_cast<level_t>(header.max_level);
2481
+ entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
2482
+
2483
+ // Load the nodes
2484
+ for (std::size_t i = 0; i != header.size; ++i) {
2485
+ span_bytes_t node_bytes = node_malloc_(levels[i]);
2486
+ result = file.read(node_bytes.data(), node_bytes.size());
2487
+ if (!result) {
2488
+ reset();
2467
2489
  return result;
2468
- read_chunk(&level, sizeof(level));
2469
- if (result.error)
2470
- return result;
2471
-
2472
- std::size_t node_bytes = node_bytes_(dim, level);
2473
- node_t node = node_malloc_(dim, level);
2474
- node.label(label);
2475
- node.dim(dim);
2476
- node.level(level);
2477
- read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
2478
- if (result.error)
2479
- return result;
2480
- nodes_[i] = node;
2481
- progress(i, size);
2490
+ }
2491
+ nodes_[i] = node_t{node_bytes.data()};
2492
+ progress(i, header.size);
2482
2493
  }
2483
-
2484
- std::fclose(file);
2485
- reset_view_();
2486
2494
  return {};
2487
2495
  }
2488
2496
 
2489
2497
  /**
2490
2498
  * @brief Memory-maps the serialized binary index representation from disk,
2491
- * @b without copying the vectors and neighbors lists into RAM.
2492
- * Available on Linux, MacOS, but @b not on Windows.
2499
+ * @b without copying data into RAM, and fetching it on-demand.
2493
2500
  */
2494
2501
  template <typename progress_at = dummy_progress_t>
2495
- serialization_result_t view(char const* file_path, progress_at&& progress = {}) noexcept {
2496
- serialization_result_t result;
2502
+ serialization_result_t view(memory_mapped_file_t file, std::size_t offset = 0,
2503
+ progress_at&& progress = {}) noexcept {
2497
2504
 
2498
- #if defined(USEARCH_DEFINED_WINDOWS)
2505
+ // Remove previously stored objects
2506
+ reset();
2499
2507
 
2500
- HANDLE file_handle =
2501
- CreateFile(file_path, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
2502
- if (file_handle == INVALID_HANDLE_VALUE)
2503
- return result.failed("Opening file failed!");
2508
+ serialization_result_t result = file.open_if_not();
2509
+ if (!result)
2510
+ return result;
2504
2511
 
2505
- size_t file_length = GetFileSize(file_handle, 0);
2506
- HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
2507
- if (mapping_handle == 0) {
2508
- CloseHandle(file_handle);
2509
- return result.failed("Mapping file failed!");
2512
+ // Pull basic metadata
2513
+ index_serialized_header_t header;
2514
+ if (file.size() - offset < sizeof(header))
2515
+ return result.failed("File is corrupted and lacks a header");
2516
+ std::memcpy(&header, file.data() + offset, sizeof(header));
2517
+
2518
+ // Precompute offsets of every node, but before that we need to update the configs
2519
+ // This could have been done with `std::exclusive_scan`, but it's only available from C++17.
2520
+ using offsets_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
2521
+ buffer_gt<std::size_t, offsets_allocator_t> offsets(header.size);
2522
+ if (!offsets)
2523
+ return result.failed("Out of memory");
2524
+
2525
+ config_.connectivity = header.connectivity;
2526
+ config_.connectivity_base = header.connectivity_base;
2527
+ pre_ = precompute_(config_);
2528
+ misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
2529
+ offsets[0] = offset + sizeof(header) + sizeof(level_t) * header.size;
2530
+ for (std::size_t i = 1; i < header.size; ++i)
2531
+ offsets[i] = offsets[i - 1] + node_bytes_(levels[i - 1]);
2532
+
2533
+ std::size_t total_bytes = offsets[header.size - 1] + node_bytes_(levels[header.size - 1]);
2534
+ if (file.size() < total_bytes) {
2535
+ reset();
2536
+ return result.failed("File is corrupted and can't fit all the nodes");
2510
2537
  }
2511
2538
 
2512
- byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
2513
- if (file == 0) {
2514
- CloseHandle(mapping_handle);
2515
- CloseHandle(file_handle);
2516
- return result.failed("View the map failed!");
2539
+ // Submit metadata and reserve memory
2540
+ index_limits_t limits;
2541
+ limits.members = header.size;
2542
+ if (!reserve(limits)) {
2543
+ reset();
2544
+ return result.failed("Out of memory");
2517
2545
  }
2518
- viewed_file_.file_handle = file_handle;
2519
- viewed_file_.mapping_handle = mapping_handle;
2520
- viewed_file_.ptr = file;
2521
- viewed_file_.length = file_length;
2522
- #else
2523
-
2524
- #if defined(USEARCH_DEFINED_LINUX)
2525
- int descriptor = open(file_path, O_RDONLY | O_NOATIME);
2526
- #else
2527
- int descriptor = open(file_path, O_RDONLY);
2528
- #endif
2529
-
2530
- // Estimate the file size
2531
- struct stat file_stat;
2532
- int fstat_status = fstat(descriptor, &file_stat);
2533
- if (fstat_status < 0) {
2534
- close(descriptor);
2535
- return result.failed(std::strerror(errno));
2546
+ nodes_count_ = header.size;
2547
+ max_level_ = static_cast<level_t>(header.max_level);
2548
+ entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
2549
+
2550
+ // Rapidly address all the nodes
2551
+ for (std::size_t i = 0; i != header.size; ++i) {
2552
+ nodes_[i] = node_t{(byte_t*)file.data() + offsets[i]};
2553
+ progress(i, header.size);
2536
2554
  }
2555
+ viewed_file_ = std::move(file);
2556
+ return {};
2557
+ }
2537
2558
 
2538
- // Map the entire file
2539
- byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_PRIVATE, descriptor, 0);
2540
- if (file == MAP_FAILED) {
2541
- close(descriptor);
2542
- return result.failed(std::strerror(errno));
2543
- }
2544
- viewed_file_.file_descriptor = descriptor;
2545
- viewed_file_.ptr = file;
2546
- viewed_file_.length = file_stat.st_size;
2547
- #endif // Platform specific code
2559
+ #pragma endregion
2548
2560
 
2549
- // Read the header
2550
- {
2551
- file_head_t state{file};
2552
- if (state.bytes_per_label != sizeof(label_t)) {
2553
- reset_view_();
2554
- return result.failed("Incompatible label type!");
2555
- }
2556
- if (state.bytes_per_id != sizeof(id_t)) {
2557
- reset_view_();
2558
- return result.failed("Incompatible ID type!");
2559
- }
2561
+ /**
2562
+ * @brief Performs compaction on the whole HNSW index, purging some entries
2563
+ * and links to them, while also generating a more efficient mapping,
2564
+ * putting the more frequently used entries closer together.
2565
+ *
2566
+ *
2567
+ * Scans the whole collection, removing the links leading towards
2568
+ * banned entries. This essentially isolates some nodes from the rest
2569
+ * of the graph, while keeping their outgoing links, in case the node
2570
+ * is structurally relevant and has a crucial role in the index.
2571
+ * It won't reclaim the memory.
2572
+ *
2573
+ * @param[in] allow_member Predicate to mark nodes for isolation.
2574
+ * @param[in] executor Thread-pool to execute the job in parallel.
2575
+ * @param[in] progress Callback to report the execution progress.
2576
+ */
2577
+ template <typename values_at, typename metric_at, //
2578
+ typename slot_transition_at = dummy_label_to_label_mapping_t, //
2579
+ typename executor_at = dummy_executor_t, //
2580
+ typename progress_at = dummy_progress_t, //
2581
+ typename prefetch_at = dummy_prefetch_t>
2582
+ void compact( //
2583
+ values_at&& values, //
2584
+ metric_at&& metric, //
2585
+ slot_transition_at&& slot_transition, //
2560
2586
 
2561
- config_.connectivity = state.connectivity;
2562
- config_.vector_alignment = state.vector_alignment;
2563
- pre_ = precompute_(config_);
2587
+ executor_at&& executor = executor_at{}, //
2588
+ progress_at&& progress = progress_at{}, //
2589
+ prefetch_at&& prefetch = prefetch_at{}) noexcept {
2590
+
2591
+ // Export all the keys, slots, and levels.
2592
+ // Partition them with the predicate.
2593
+ // Sort the allowed entries in descending order of their level.
2594
+ // Create a new array mapping old slots to the new ones (INT_MAX for deleted items).
2595
+ struct slot_level_t {
2596
+ compressed_slot_t old_slot;
2597
+ compressed_slot_t cluster;
2598
+ level_t level;
2599
+ };
2600
+ using slot_level_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<slot_level_t>;
2601
+ buffer_gt<slot_level_t, slot_level_allocator_t> slots_and_levels(size());
2602
+
2603
+ // For every bottom level node, determine its parent cluster
2604
+ executor.execute_bulk(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
2605
+ context_t& context = contexts_[thread_idx];
2606
+ std::size_t cluster = search_for_one_( //
2607
+ values[citerator_at(old_slot)], //
2608
+ metric, prefetch, //
2609
+ entry_slot_, max_level_, 0, context);
2610
+ slots_and_levels[old_slot] = { //
2611
+ static_cast<compressed_slot_t>(old_slot), //
2612
+ static_cast<compressed_slot_t>(cluster), //
2613
+ node_at_(old_slot).level()};
2614
+ });
2564
2615
 
2565
- index_limits_t limits;
2566
- limits.members = state.size;
2567
- limits.threads_add = 0;
2568
- if (!reserve(limits))
2569
- return result.failed("Out of memory!");
2616
+ // Where the actual permutation happens:
2617
+ std::sort(slots_and_levels.begin(), slots_and_levels.end(), [](slot_level_t const& a, slot_level_t const& b) {
2618
+ return a.level == b.level ? a.cluster < b.cluster : a.level > b.level;
2619
+ });
2570
2620
 
2571
- size_ = state.size;
2572
- max_level_ = static_cast<level_t>(state.max_level);
2573
- entry_id_ = static_cast<id_t>(state.entry_idx);
2574
- }
2621
+ using size_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
2622
+ buffer_gt<std::size_t, size_allocator_t> old_slot_to_new(slots_and_levels.size());
2623
+ for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot)
2624
+ old_slot_to_new[slots_and_levels[new_slot].old_slot] = new_slot;
2575
2625
 
2576
- // Locate every node packed into file
2577
- std::size_t progress_bytes = sizeof(file_header_t);
2578
- std::size_t const size = size_;
2579
- for (std::size_t i = 0; i != size; ++i) {
2580
- byte_t* tape = (byte_t*)(file + progress_bytes);
2581
- dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
2582
- level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
2583
-
2584
- std::size_t node_bytes = node_bytes_(dim, level);
2585
- std::size_t node_vector_bytes = dim * sizeof(scalar_t);
2586
- nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
2587
- progress_bytes += node_bytes;
2588
- progress(i, size);
2589
- }
2626
+ // Erase all the incoming links
2627
+ buffer_gt<node_t, nodes_allocator_t> reordered_nodes(slots_and_levels.size());
2628
+ tape_allocator_t reordered_tape;
2590
2629
 
2591
- return {};
2592
- }
2630
+ for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
2631
+ std::size_t old_slot = slots_and_levels[new_slot].old_slot;
2632
+ node_t old_node = node_at_(old_slot);
2593
2633
 
2594
- #pragma endregion
2634
+ std::size_t node_bytes = node_bytes_(old_node.level());
2635
+ byte_t* new_data = (byte_t*)reordered_tape.allocate(node_bytes);
2636
+ node_t new_node{new_data};
2637
+ std::memcpy(new_data, old_node.tape(), node_bytes);
2595
2638
 
2596
- struct join_result_t {
2597
- error_t error{};
2598
- std::size_t intersection_size{};
2599
- std::size_t engagements{};
2600
- std::size_t cycles{};
2601
- std::size_t measurements{};
2639
+ for (level_t level = 0; level <= old_node.level(); ++level)
2640
+ for (misaligned_ref_gt<compressed_slot_t> neighbor : neighbors_(new_node, level))
2641
+ neighbor = static_cast<compressed_slot_t>(old_slot_to_new[compressed_slot_t(neighbor)]);
2602
2642
 
2603
- explicit operator bool() const noexcept { return !error; }
2604
- join_result_t failed(error_t message) noexcept {
2605
- error = std::move(message);
2606
- return std::move(*this);
2643
+ reordered_nodes[new_slot] = new_node;
2644
+
2645
+ progress(new_slot, slots_and_levels.size());
2607
2646
  }
2608
- };
2609
2647
 
2610
- /**
2611
- * @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
2612
- * to perform fast one-to-one matching between two large collections
2613
- * of vectors, using approximate nearest neighbors search.
2614
- *
2615
- * @param[inout] first_to_second Container to map ::first labels to ::second.
2616
- * @param[inout] second_to_first Container to map ::second labels to ::first.
2617
- * @param[in] executor Thread-pool to execute the job in parallel.
2618
- * @param[in] progress Callback to report the execution progress.
2619
- */
2620
- template < //
2621
- typename first_to_second_at = dummy_label_to_label_mapping_t, //
2622
- typename second_to_first_at = dummy_label_to_label_mapping_t, //
2623
- typename executor_at = dummy_executor_t, //
2624
- typename progress_at = dummy_progress_t //
2625
- >
2626
- static join_result_t join( //
2627
- index_gt const& first, index_gt const& second, //
2628
- join_config_t config = {}, //
2629
- first_to_second_at&& first_to_second = first_to_second_at{}, //
2630
- second_to_first_at&& second_to_first = second_to_first_at{}, //
2631
- executor_at&& executor = executor_at{}, //
2632
- progress_at&& progress = progress_at{}) noexcept {
2648
+ for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
2649
+ std::size_t old_slot = slots_and_levels[new_slot].old_slot;
2650
+ slot_transition(node_at_(old_slot).ckey(), //
2651
+ static_cast<compressed_slot_t>(old_slot), //
2652
+ static_cast<compressed_slot_t>(new_slot));
2653
+ }
2633
2654
 
2634
- if (second.size() < first.size())
2635
- return join_small_and_big_( //
2636
- second, first, //
2637
- config, //
2638
- std::forward<second_to_first_at>(second_to_first), //
2639
- std::forward<first_to_second_at>(first_to_second), //
2640
- std::forward<executor_at>(executor), //
2641
- std::forward<progress_at>(progress));
2642
- else
2643
- return join_small_and_big_( //
2644
- first, second, //
2645
- config, //
2646
- std::forward<first_to_second_at>(first_to_second), //
2647
- std::forward<second_to_first_at>(second_to_first), //
2648
- std::forward<executor_at>(executor), //
2649
- std::forward<progress_at>(progress));
2655
+ nodes_ = std::move(reordered_nodes);
2656
+ tape_allocator_ = std::move(reordered_tape);
2657
+ entry_slot_ = old_slot_to_new[entry_slot_];
2650
2658
  }
2651
2659
 
2652
2660
  /**
@@ -2671,293 +2679,80 @@ class index_gt {
2671
2679
  progress_at&& progress = progress_at{}) noexcept {
2672
2680
 
2673
2681
  // Erase all the incoming links
2674
- executor.execute_bulk(size(), [&](std::size_t node_idx, std::size_t) {
2675
- node_t node = node_with_id_(node_idx);
2682
+ std::size_t nodes_count = size();
2683
+ executor.execute_bulk(nodes_count, [&](std::size_t, std::size_t node_idx) {
2684
+ node_t node = node_at_(node_idx);
2676
2685
  for (level_t level = 0; level <= node.level(); ++level) {
2677
2686
  neighbors_ref_t neighbors = neighbors_(node, level);
2678
2687
  std::size_t old_size = neighbors.size();
2679
2688
  neighbors.clear();
2680
2689
  for (std::size_t i = 0; i != old_size; ++i) {
2681
- id_t neighbor_id = neighbors[i];
2682
- node_t neighbor = node_with_id_(neighbor_id);
2683
- if (allow_member(member_cref_t{neighbor.label(), neighbor.vector_view(), neighbor_id}))
2684
- neighbors.push_back(neighbor_id);
2690
+ compressed_slot_t neighbor_slot = neighbors[i];
2691
+ node_t neighbor = node_at_(neighbor_slot);
2692
+ if (allow_member(member_cref_t{neighbor.ckey(), neighbor_slot}))
2693
+ neighbors.push_back(neighbor_slot);
2685
2694
  }
2686
2695
  }
2696
+ progress(node_idx, nodes_count);
2687
2697
  });
2688
2698
  }
2689
2699
 
2690
2700
  private:
2691
- template <typename first_to_second_at, typename second_to_first_at, typename executor_at, typename progress_at>
2692
- static join_result_t join_small_and_big_( //
2693
- index_gt const& men, index_gt const& women, //
2694
- join_config_t config, //
2695
- first_to_second_at&& man_to_woman, //
2696
- second_to_first_at&& woman_to_man, //
2697
- executor_at&& executor, //
2698
- progress_at&& progress) noexcept {
2699
-
2700
- join_result_t result;
2701
-
2702
- // Sanity checks and argument validation:
2703
- if (&men == &women)
2704
- return result.failed("Can't join with itself, consider copying");
2705
- if (config.max_proposals == 0)
2706
- config.max_proposals = std::log(men.size()) + executor.size();
2707
- config.max_proposals = (std::min)(men.size(), config.max_proposals);
2708
-
2709
- using proposals_count_t = std::uint16_t;
2710
- using ids_allocator_t = typename allocator_traits_t::template rebind_alloc<id_t>;
2711
- allocator_t& alloc = men.allocator_;
2712
-
2713
- // Create an atomic queue, as a ring structure, from/to which
2714
- // free men will be added/pulled.
2715
- std::mutex free_men_mutex{};
2716
- ring_gt<id_t, ids_allocator_t> free_men;
2717
- free_men.reserve(men.size());
2718
- for (std::size_t i = 0; i != men.size(); ++i)
2719
- free_men.push(static_cast<id_t>(i));
2720
-
2721
- // We are gonna need some temporary memory.
2722
- proposals_count_t* proposal_counts = (proposals_count_t*)alloc.allocate(sizeof(proposals_count_t) * men.size());
2723
- id_t* man_to_woman_ids = (id_t*)alloc.allocate(sizeof(id_t) * men.size());
2724
- id_t* woman_to_man_ids = (id_t*)alloc.allocate(sizeof(id_t) * women.size());
2725
- if (!proposal_counts || !man_to_woman_ids || !woman_to_man_ids)
2726
- return result.failed("Can't temporary mappings");
2727
-
2728
- id_t missing_id;
2729
- std::memset((void*)&missing_id, 0xFF, sizeof(id_t));
2730
- std::memset((void*)man_to_woman_ids, 0xFF, sizeof(id_t) * men.size());
2731
- std::memset((void*)woman_to_man_ids, 0xFF, sizeof(id_t) * women.size());
2732
- std::memset(proposal_counts, 0, sizeof(proposals_count_t) * men.size());
2733
-
2734
- // Define locks, to limit concurrent accesses to `man_to_woman_ids` and `woman_to_man_ids`.
2735
- visits_bitset_t men_locks, women_locks;
2736
- if (!men_locks.resize(men.size()) || !women_locks.resize(women.size()))
2737
- return result.failed("Can't allocate locks");
2738
-
2739
- // Accumulate statistics from all the contexts,
2740
- // to have a baseline to compare with, by the time the `join` is finished.
2741
- std::size_t old_measurements{};
2742
- std::size_t old_cycles{};
2743
- for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
2744
- old_measurements += women.contexts_[thread_idx].measurements_count;
2745
- old_cycles += women.contexts_[thread_idx].iteration_cycles;
2746
- }
2747
- std::atomic<std::size_t> rounds{0};
2748
- std::atomic<std::size_t> engagements{0};
2749
-
2750
- // Concurrently process all the men
2751
- executor.execute_bulk([&](std::size_t thread_idx) {
2752
- context_t& context = women.contexts_[thread_idx];
2753
- search_config_t search_config;
2754
- search_config.expansion = config.expansion;
2755
- search_config.exact = config.exact;
2756
- search_config.thread = thread_idx;
2757
- id_t free_man_id;
2758
-
2759
- // While there exist a free man who still has a woman to propose to.
2760
- while (true) {
2761
- std::size_t passed_rounds = 0;
2762
- std::size_t total_rounds = 0;
2763
- {
2764
- std::unique_lock<std::mutex> pop_lock(free_men_mutex);
2765
- if (!free_men.try_pop(free_man_id))
2766
- // Primary exit path, we have exhausted the list of candidates
2767
- break;
2768
- passed_rounds = ++rounds;
2769
- total_rounds = passed_rounds + free_men.size();
2770
- }
2771
- progress(passed_rounds, total_rounds);
2772
- while (men_locks.atomic_set(free_man_id))
2773
- ;
2774
-
2775
- node_t free_man = men.node_with_id_(free_man_id);
2776
- proposals_count_t& free_man_proposals = proposal_counts[free_man_id];
2777
- if (free_man_proposals >= config.max_proposals)
2778
- continue;
2779
-
2780
- // Find the closest woman, to whom this man hasn't proposed yet.
2781
- free_man_proposals++;
2782
- search_result_t candidates = women.search(free_man.vector_view(), free_man_proposals, search_config);
2783
- if (!candidates) {
2784
- // TODO:
2785
- }
2786
-
2787
- match_t match = candidates.back();
2788
- member_cref_t woman = match.member;
2789
- while (women_locks.atomic_set(woman.id))
2790
- ;
2791
-
2792
- id_t husband_id = woman_to_man_ids[woman.id];
2793
- bool woman_is_free = husband_id == missing_id;
2794
- if (woman_is_free) {
2795
- // Engagement
2796
- man_to_woman_ids[free_man_id] = woman.id;
2797
- woman_to_man_ids[woman.id] = free_man_id;
2798
- engagements++;
2799
- } else {
2800
- distance_t distance_from_husband = context.measure(woman.vector, men.node_with_id_(husband_id));
2801
- distance_t distance_from_candidate = match.distance;
2802
- if (distance_from_husband > distance_from_candidate) {
2803
- // Break-up
2804
- while (men_locks.atomic_set(husband_id))
2805
- ;
2806
- man_to_woman_ids[husband_id] = missing_id;
2807
- men_locks.atomic_reset(husband_id);
2808
-
2809
- // New Engagement
2810
- man_to_woman_ids[free_man_id] = woman.id;
2811
- woman_to_man_ids[woman.id] = free_man_id;
2812
- engagements++;
2813
-
2814
- std::unique_lock<std::mutex> push_lock(free_men_mutex);
2815
- free_men.push(husband_id);
2816
- } else {
2817
- std::unique_lock<std::mutex> push_lock(free_men_mutex);
2818
- free_men.push(free_man_id);
2819
- }
2820
- }
2821
-
2822
- men_locks.atomic_reset(free_man_id);
2823
- women_locks.atomic_reset(woman.id);
2824
- }
2825
- });
2826
-
2827
- // Export the IDs into labels:
2828
- std::size_t intersection_size = 0;
2829
- for (std::size_t i = 0; i != men.size(); ++i) {
2830
- id_t woman_id = man_to_woman_ids[i];
2831
- if (woman_id != missing_id) {
2832
- label_t man = men.node_with_id_(i).label();
2833
- label_t woman = women.node_with_id_(woman_id).label();
2834
- man_to_woman[man] = woman;
2835
- woman_to_man[woman] = man;
2836
- intersection_size++;
2837
- }
2838
- }
2839
-
2840
- // Deallocate memory
2841
- alloc.deallocate((byte_t*)proposal_counts, sizeof(proposals_count_t) * men.size());
2842
- alloc.deallocate((byte_t*)man_to_woman_ids, sizeof(id_t) * men.size());
2843
- alloc.deallocate((byte_t*)woman_to_man_ids, sizeof(id_t) * women.size());
2844
-
2845
- // Export stats
2846
- result.engagements = engagements;
2847
- result.intersection_size = intersection_size;
2848
- for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
2849
- result.measurements += women.contexts_[thread_idx].measurements_count;
2850
- result.cycles += women.contexts_[thread_idx].iteration_cycles;
2851
- }
2852
- result.measurements -= old_measurements;
2853
- result.cycles -= old_cycles;
2854
- return result;
2855
- }
2856
-
2857
- void reset_view_() noexcept {
2858
- if (!viewed_file_)
2859
- return;
2860
- #if defined(USEARCH_DEFINED_WINDOWS)
2861
- UnmapViewOfFile(viewed_file_.ptr);
2862
- CloseHandle(viewed_file_.mapping_handle);
2863
- CloseHandle(viewed_file_.file_handle);
2864
-
2865
- #else
2866
- munmap(viewed_file_.ptr, viewed_file_.length);
2867
- close(viewed_file_.file_descriptor);
2868
- #endif
2869
- viewed_file_ = {};
2870
- }
2871
-
2872
2701
  inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
2873
2702
  precomputed_constants_t pre;
2874
- pre.connectivity_max_base = config.connectivity * base_level_multiple_();
2875
2703
  pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
2876
- pre.neighbors_bytes = config.connectivity * sizeof(id_t) + sizeof(neighbors_count_t);
2877
- pre.neighbors_base_bytes = pre.connectivity_max_base * sizeof(id_t) + sizeof(neighbors_count_t);
2704
+ pre.neighbors_bytes = config.connectivity * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
2705
+ pre.neighbors_base_bytes = config.connectivity_base * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
2878
2706
  return pre;
2879
2707
  }
2880
2708
 
2881
- inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
2882
- inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
2883
- return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
2884
- }
2885
-
2886
2709
  using span_bytes_t = span_gt<byte_t>;
2887
- struct node_bytes_split_t {
2888
- span_bytes_t tape{};
2889
- span_bytes_t vector{};
2890
2710
 
2891
- node_bytes_split_t() {}
2892
- node_bytes_split_t(span_bytes_t tape, span_bytes_t vector) noexcept : tape(tape), vector(vector) {}
2893
-
2894
- std::size_t memory_usage() const noexcept { return tape.size() + vector.size(); }
2895
- bool colocated() const noexcept { return tape.end() == vector.begin(); }
2896
- operator node_t() const noexcept { return node_t{tape.begin(), reinterpret_cast<scalar_t*>(vector.begin())}; }
2897
- };
2898
-
2899
- inline node_bytes_split_t node_bytes_split_(node_t node) const noexcept {
2900
- std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
2901
- std::size_t bytes_in_tape = node_head_bytes_() + levels_bytes;
2902
- return {{node.tape(), bytes_in_tape}, {(byte_t*)node.vector(), node_vector_bytes_(node)}};
2711
+ inline span_bytes_t node_bytes_(node_t node) const noexcept { return {node.tape(), node_bytes_(node.level())}; }
2712
+ inline std::size_t node_bytes_(level_t level) const noexcept {
2713
+ return node_head_bytes_() + node_neighbors_bytes_(level);
2714
+ }
2715
+ inline std::size_t node_neighbors_bytes_(node_t node) const noexcept { return node_neighbors_bytes_(node.level()); }
2716
+ inline std::size_t node_neighbors_bytes_(level_t level) const noexcept {
2717
+ return pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
2903
2718
  }
2904
2719
 
2905
- inline std::size_t node_vector_bytes_(dim_t dim) const noexcept { return dim * sizeof(scalar_t); }
2906
- inline std::size_t node_vector_bytes_(node_t node) const noexcept { return node_vector_bytes_(node.dim()); }
2907
-
2908
- node_bytes_split_t node_malloc_(dim_t dims_to_store, level_t level) noexcept {
2909
-
2910
- std::size_t vector_bytes = node_vector_bytes_(dims_to_store);
2911
- std::size_t node_bytes = node_bytes_(dims_to_store, level);
2912
- std::size_t non_vector_bytes = node_bytes - vector_bytes;
2913
-
2914
- byte_t* data = (byte_t*)point_allocator_.allocate(node_bytes);
2915
- if (!data)
2916
- return node_bytes_split_t{};
2917
- return {{data, non_vector_bytes}, {data + non_vector_bytes, vector_bytes}};
2720
+ span_bytes_t node_malloc_(level_t level) noexcept {
2721
+ std::size_t node_bytes = node_bytes_(level);
2722
+ byte_t* data = (byte_t*)tape_allocator_.allocate(node_bytes);
2723
+ return data ? span_bytes_t{data, node_bytes} : span_bytes_t{};
2918
2724
  }
2919
2725
 
2920
- node_t node_make_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
2921
- node_bytes_split_t node_bytes = node_malloc_(vector.size() * store_vector, level);
2922
- if (store_vector) {
2923
- std::memset(node_bytes.tape.data(), 0, node_bytes.tape.size());
2924
- std::memcpy(node_bytes.vector.data(), vector.data(), node_bytes.vector.size());
2925
- } else {
2926
- std::memset(node_bytes.tape.data(), 0, node_bytes.memory_usage());
2927
- }
2928
- node_t node = node_bytes;
2929
- node.label(label);
2930
- node.dim(static_cast<dim_t>(vector.size()));
2726
+ node_t node_make_(key_t key, level_t level) noexcept {
2727
+ span_bytes_t node_bytes = node_malloc_(level);
2728
+ if (!node_bytes)
2729
+ return {};
2730
+
2731
+ std::memset(node_bytes.data(), 0, node_bytes.size());
2732
+ node_t node{(byte_t*)node_bytes.data()};
2733
+ node.key(key);
2931
2734
  node.level(level);
2932
2735
  return node;
2933
2736
  }
2934
2737
 
2935
- node_t node_make_copy_(node_bytes_split_t old_bytes) noexcept {
2936
- if (old_bytes.colocated()) {
2937
- byte_t* data = (byte_t*)point_allocator_.allocate(old_bytes.memory_usage());
2938
- std::memcpy(data, old_bytes.tape.data(), old_bytes.memory_usage());
2939
- return node_t{data, reinterpret_cast<scalar_t*>(data + old_bytes.tape.size())};
2940
- } else {
2941
- node_t old_node = old_bytes;
2942
- node_bytes_split_t node_bytes = node_malloc_(old_node.vector_view().size(), old_node.level());
2943
- std::memcpy(node_bytes.tape.data(), old_bytes.tape.data(), old_bytes.tape.size());
2944
- std::memcpy(node_bytes.vector.data(), old_bytes.vector.data(), old_bytes.vector.size());
2945
- return node_bytes;
2946
- }
2738
+ node_t node_make_copy_(span_bytes_t old_bytes) noexcept {
2739
+ byte_t* data = (byte_t*)tape_allocator_.allocate(old_bytes.size());
2740
+ if (!data)
2741
+ return {};
2742
+ std::memcpy(data, old_bytes.data(), old_bytes.size());
2743
+ return node_t{data};
2947
2744
  }
2948
2745
 
2949
- void node_free_(std::size_t id) noexcept {
2950
-
2746
+ void node_free_(std::size_t idx) noexcept {
2951
2747
  if (viewed_file_)
2952
2748
  return;
2953
2749
 
2954
- node_t& node = nodes_[id];
2955
- std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_bytes_split_(node).colocated();
2956
- point_allocator_.deallocate(node.tape(), node_bytes);
2750
+ node_t& node = nodes_[idx];
2751
+ tape_allocator_.deallocate(node.tape(), node_bytes_(node).size());
2957
2752
  node = node_t{};
2958
2753
  }
2959
2754
 
2960
- inline node_t node_with_id_(std::size_t idx) const noexcept { return nodes_[idx]; }
2755
+ inline node_t node_at_(std::size_t idx) const noexcept { return nodes_[idx]; }
2961
2756
  inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
2962
2757
 
2963
2758
  inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
@@ -2970,91 +2765,102 @@ class index_gt {
2970
2765
 
2971
2766
  struct node_lock_t {
2972
2767
  visits_bitset_t& bitset;
2973
- std::size_t idx;
2974
-
2975
- inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
2768
+ std::size_t slot;
2769
+ inline ~node_lock_t() noexcept { bitset.atomic_reset(slot); }
2976
2770
  };
2977
2771
 
2978
- inline node_lock_t node_lock_(std::size_t idx) const noexcept {
2979
- while (nodes_mutexes_.atomic_set(idx))
2772
+ inline node_lock_t node_lock_(std::size_t slot) const noexcept {
2773
+ while (nodes_mutexes_.atomic_set(slot))
2980
2774
  ;
2981
- return {nodes_mutexes_, idx};
2775
+ return {nodes_mutexes_, slot};
2982
2776
  }
2983
2777
 
2984
- void connect_node_across_levels_( //
2985
- id_t node_id, vector_view_t vector, //
2986
- id_t entry_id, level_t max_level, level_t target_level, //
2987
- add_config_t const& config, context_t& context) usearch_noexcept_m {
2778
+ template <typename value_at, typename metric_at, typename prefetch_at>
2779
+ void connect_node_across_levels_( //
2780
+ value_at&& value, metric_at&& metric, prefetch_at&& prefetch, //
2781
+ std::size_t node_slot, std::size_t entry_slot, level_t max_level, level_t target_level, //
2782
+ index_update_config_t const& config, context_t& context) usearch_noexcept_m {
2988
2783
 
2989
2784
  // Go down the level, tracking only the closest match
2990
- id_t closest_id = search_for_one_(entry_id, vector, max_level, target_level, context);
2785
+ std::size_t closest_slot = search_for_one_( //
2786
+ value, metric, prefetch, //
2787
+ entry_slot, max_level, target_level, context);
2991
2788
 
2992
2789
  // From `target_level` down perform proper extensive search
2993
2790
  for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
2994
2791
  // TODO: Handle out of memory conditions
2995
- search_to_insert_(closest_id, vector, level, config.expansion, context);
2996
- closest_id = connect_new_node_(node_id, level, context);
2997
- reconnect_neighbor_nodes_(node_id, level, context);
2792
+ search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
2793
+ closest_slot = connect_new_node_(metric, node_slot, level, context);
2794
+ reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
2998
2795
  }
2999
2796
  }
3000
2797
 
3001
- id_t connect_new_node_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
2798
+ template <typename metric_at>
2799
+ std::size_t connect_new_node_( //
2800
+ metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
3002
2801
 
3003
- node_t new_node = node_with_id_(new_id);
2802
+ node_t new_node = node_at_(new_slot);
3004
2803
  top_candidates_t& top = context.top_candidates;
3005
2804
 
3006
- // Outgoing links from `new_id`:
2805
+ // Outgoing links from `new_slot`:
3007
2806
  neighbors_ref_t new_neighbors = neighbors_(new_node, level);
3008
2807
  {
3009
2808
  usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
3010
- candidates_view_t top_view = refine_(top, config_.connectivity, context);
2809
+ candidates_view_t top_view = refine_(metric, config_.connectivity, top, context);
3011
2810
 
3012
2811
  for (std::size_t idx = 0; idx != top_view.size(); idx++) {
3013
2812
  usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
3014
- usearch_assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
3015
- new_neighbors.push_back(top_view[idx].id);
2813
+ usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
2814
+ new_neighbors.push_back(top_view[idx].slot);
3016
2815
  }
3017
2816
  }
3018
2817
 
3019
2818
  return new_neighbors[0];
3020
2819
  }
3021
2820
 
3022
- void reconnect_neighbor_nodes_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
2821
+ template <typename value_at, typename metric_at>
2822
+ void reconnect_neighbor_nodes_( //
2823
+ metric_at&& metric, std::size_t new_slot, value_at&& value, level_t level,
2824
+ context_t& context) usearch_noexcept_m {
3023
2825
 
3024
- node_t new_node = node_with_id_(new_id);
2826
+ node_t new_node = node_at_(new_slot);
3025
2827
  top_candidates_t& top = context.top_candidates;
3026
2828
  neighbors_ref_t new_neighbors = neighbors_(new_node, level);
3027
2829
 
3028
2830
  // Reverse links from the neighbors:
3029
- std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
3030
- for (id_t close_id : new_neighbors) {
3031
- node_t close_node = node_with_id_(close_id);
3032
- node_lock_t close_lock = node_lock_(close_id);
2831
+ std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
2832
+ for (compressed_slot_t close_slot : new_neighbors) {
2833
+ if (close_slot == new_slot)
2834
+ continue;
2835
+ node_lock_t close_lock = node_lock_(close_slot);
2836
+ node_t close_node = node_at_(close_slot);
3033
2837
 
3034
2838
  neighbors_ref_t close_header = neighbors_(close_node, level);
3035
2839
  usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
3036
- usearch_assert_m(close_id != new_id, "Self-loops are impossible");
2840
+ usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
3037
2841
  usearch_assert_m(level <= close_node.level(), "Linking to missing level");
3038
2842
 
3039
- // If `new_id` is already present in the neighboring connections of `close_id`
2843
+ // If `new_slot` is already present in the neighboring connections of `close_slot`
3040
2844
  // then no need to modify any connections or run the heuristics.
3041
2845
  if (close_header.size() < connectivity_max) {
3042
- close_header.push_back(new_id);
2846
+ close_header.push_back(static_cast<compressed_slot_t>(new_slot));
3043
2847
  continue;
3044
2848
  }
3045
2849
 
3046
2850
  // To fit a new connection we need to drop an existing one.
3047
2851
  top.clear();
3048
2852
  usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
3049
- top.insert_reserved({context.measure(new_node, close_node), new_id});
3050
- for (id_t successor_id : close_header)
3051
- top.insert_reserved({context.measure(node_with_id_(successor_id), close_node), successor_id});
2853
+ top.insert_reserved(
2854
+ {context.measure(value, citerator_at(close_slot), metric), static_cast<compressed_slot_t>(new_slot)});
2855
+ for (compressed_slot_t successor_slot : close_header)
2856
+ top.insert_reserved(
2857
+ {context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
3052
2858
 
3053
2859
  // Export the results:
3054
2860
  close_header.clear();
3055
- candidates_view_t top_view = refine_(top, connectivity_max, context);
2861
+ candidates_view_t top_view = refine_(metric, connectivity_max, top, context);
3056
2862
  for (std::size_t idx = 0; idx != top_view.size(); idx++)
3057
- close_header.push_back(top_view[idx].id);
2863
+ close_header.push_back(top_view[idx].slot);
3058
2864
  }
3059
2865
  }
3060
2866
 
@@ -3064,31 +2870,105 @@ class index_gt {
3064
2870
  return (level_t)r;
3065
2871
  }
3066
2872
 
3067
- id_t search_for_one_( //
3068
- id_t closest_id, vector_view_t query, //
3069
- level_t begin_level, level_t end_level, //
3070
- context_t& context) const noexcept {
2873
+ struct candidates_range_t;
2874
+ class candidates_iterator_t {
2875
+ friend struct candidates_range_t;
2876
+
2877
+ index_gt const& index_;
2878
+ neighbors_ref_t neighbors_;
2879
+ visits_bitset_t& visits_;
2880
+ std::size_t current_;
2881
+
2882
+ candidates_iterator_t& skip_missing() noexcept {
2883
+ while (current_ != neighbors_.size()) {
2884
+ compressed_slot_t neighbor_slot = neighbors_[current_];
2885
+ if (visits_.test(neighbor_slot))
2886
+ current_++;
2887
+ else
2888
+ break;
2889
+ }
2890
+ return *this;
2891
+ }
3071
2892
 
3072
- distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
2893
+ public:
2894
+ using element_t = compressed_slot_t;
2895
+ using iterator_category = std::forward_iterator_tag;
2896
+ using value_type = element_t;
2897
+ using difference_type = std::ptrdiff_t;
2898
+ using pointer = misaligned_ptr_gt<element_t>;
2899
+ using reference = misaligned_ref_gt<element_t>;
2900
+
2901
+ reference operator*() const noexcept { return slot(); }
2902
+ candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_bitset_t& visits,
2903
+ std::size_t progress) noexcept
2904
+ : index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
2905
+ candidates_iterator_t operator++(int) noexcept {
2906
+ return candidates_iterator_t(index_, visits_, neighbors_, current_ + 1).skip_missing();
2907
+ }
2908
+ candidates_iterator_t& operator++() noexcept {
2909
+ ++current_;
2910
+ skip_missing();
2911
+ return *this;
2912
+ }
2913
+ bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
2914
+ bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
2915
+
2916
+ key_t key() const noexcept { return index_->node_at_(slot()).key(); }
2917
+ compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
2918
+ friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
2919
+ friend inline key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
2920
+ };
2921
+
2922
+ struct candidates_range_t {
2923
+ index_gt const& index;
2924
+ neighbors_ref_t neighbors;
2925
+ visits_bitset_t& visits;
2926
+
2927
+ candidates_iterator_t begin() const noexcept {
2928
+ return candidates_iterator_t{index, neighbors, visits, 0}.skip_missing();
2929
+ }
2930
+ candidates_iterator_t end() const noexcept { return {index, neighbors, visits, neighbors.size()}; }
2931
+ };
2932
+
2933
+ template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
2934
+ std::size_t search_for_one_( //
2935
+ value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
2936
+ std::size_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
2937
+
2938
+ visits_bitset_t& visits = context.visits;
2939
+ visits.clear();
2940
+
2941
+ // Optional prefetching
2942
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
2943
+ prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
2944
+
2945
+ distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
3073
2946
  for (level_t level = begin_level; level > end_level; --level) {
3074
2947
  bool changed;
3075
2948
  do {
3076
2949
  changed = false;
3077
- node_t closest_node = node_with_id_(closest_id);
3078
- node_lock_t closest_lock = node_lock_(closest_id);
3079
- neighbors_ref_t closest_neighbors = neighbors_non_base_(closest_node, level);
3080
- for (id_t candidate_id : closest_neighbors) {
3081
- distance_t candidate_dist = context.measure(query, node_with_id_(candidate_id));
2950
+ node_lock_t closest_lock = node_lock_(closest_slot);
2951
+ neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
2952
+
2953
+ // Optional prefetching
2954
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
2955
+ candidates_range_t missing_candidates{*this, closest_neighbors, visits};
2956
+ prefetch(missing_candidates.begin(), missing_candidates.end());
2957
+ }
2958
+
2959
+ // Actual traversal
2960
+ for (compressed_slot_t candidate_slot : closest_neighbors) {
2961
+ distance_t candidate_dist = context.measure(query, citerator_at(candidate_slot), metric);
3082
2962
  if (candidate_dist < closest_dist) {
3083
2963
  closest_dist = candidate_dist;
3084
- closest_id = candidate_id;
2964
+ closest_slot = candidate_slot;
3085
2965
  changed = true;
3086
2966
  }
3087
2967
  }
3088
2968
  context.iteration_cycles++;
3089
2969
  } while (changed);
3090
2970
  }
3091
- return closest_id;
2971
+ return closest_slot;
3092
2972
  }
3093
2973
 
3094
2974
  /**
@@ -3096,8 +2976,11 @@ class index_gt {
3096
2976
  * Locks the nodes in the process, assuming other threads are updating neighbors lists.
3097
2977
  * @return `true` if procedure succeeded, `false` if run out of memory.
3098
2978
  */
3099
- bool search_to_insert_( //
3100
- id_t start_id, vector_view_t query, level_t level, std::size_t top_limit, context_t& context) noexcept {
2979
+ template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
2980
+ bool search_to_insert_( //
2981
+ value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
2982
+ std::size_t start_slot, std::size_t new_slot, level_t level, std::size_t top_limit,
2983
+ context_t& context) noexcept {
3101
2984
 
3102
2985
  visits_bitset_t& visits = context.visits;
3103
2986
  next_candidates_t& next = context.next_candidates; // pop min, push
@@ -3107,10 +2990,14 @@ class index_gt {
3107
2990
  next.clear();
3108
2991
  top.clear();
3109
2992
 
3110
- distance_t radius = context.measure(query, node_with_id_(start_id));
3111
- next.insert_reserved({-radius, start_id});
3112
- top.insert_reserved({radius, start_id});
3113
- visits.set(start_id);
2993
+ // Optional prefetching
2994
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
2995
+ prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
2996
+
2997
+ distance_t radius = context.measure(query, citerator_at(start_slot), metric);
2998
+ next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
2999
+ top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
3000
+ visits.set(start_slot);
3114
3001
 
3115
3002
  while (!next.empty()) {
3116
3003
 
@@ -3121,24 +3008,30 @@ class index_gt {
3121
3008
  next.pop();
3122
3009
  context.iteration_cycles++;
3123
3010
 
3124
- id_t candidate_id = candidacy.id;
3125
- node_t candidate_ref = node_with_id_(candidate_id);
3126
- node_lock_t candidate_lock = node_lock_(candidate_id);
3011
+ compressed_slot_t candidate_slot = candidacy.slot;
3012
+ if (new_slot == candidate_slot)
3013
+ continue;
3014
+ node_t candidate_ref = node_at_(candidate_slot);
3015
+ node_lock_t candidate_lock = node_lock_(candidate_slot);
3127
3016
  neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
3128
3017
 
3129
- prefetch_neighbors_(candidate_neighbors, visits);
3130
- for (id_t successor_id : candidate_neighbors) {
3131
- if (visits.test(successor_id))
3132
- continue;
3018
+ // Optional prefetching
3019
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
3020
+ candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
3021
+ prefetch(missing_candidates.begin(), missing_candidates.end());
3022
+ }
3133
3023
 
3134
- visits.set(successor_id);
3135
- distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
3024
+ for (compressed_slot_t successor_slot : candidate_neighbors) {
3025
+ if (visits.set(successor_slot))
3026
+ continue;
3136
3027
 
3028
+ // node_lock_t successor_lock = node_lock_(successor_slot);
3029
+ distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
3137
3030
  if (top.size() < top_limit || successor_dist < radius) {
3138
3031
  // This can substantially grow our priority queue:
3139
- next.insert({-successor_dist, successor_id});
3032
+ next.insert({-successor_dist, successor_slot});
3140
3033
  // This will automatically evict poor matches:
3141
- top.insert({successor_dist, successor_id}, top_limit);
3034
+ top.insert({successor_dist, successor_slot}, top_limit);
3142
3035
  radius = top.top().distance;
3143
3036
  }
3144
3037
  }
@@ -3151,10 +3044,10 @@ class index_gt {
3151
3044
  * Doesn't lock any nodes, assuming read-only simultaneous access.
3152
3045
  * @return `true` if procedure succeeded, `false` if run out of memory.
3153
3046
  */
3154
- template <typename predicate_at>
3155
- bool search_to_find_in_base_( //
3156
- id_t start_id, vector_view_t query, std::size_t expansion, context_t& context,
3157
- predicate_at&& predicate) const noexcept {
3047
+ template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
3048
+ bool search_to_find_in_base_( //
3049
+ value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
3050
+ std::size_t start_slot, std::size_t expansion, context_t& context) const noexcept {
3158
3051
 
3159
3052
  visits_bitset_t& visits = context.visits;
3160
3053
  next_candidates_t& next = context.next_candidates; // pop min, push
@@ -3165,10 +3058,14 @@ class index_gt {
3165
3058
  next.clear();
3166
3059
  top.clear();
3167
3060
 
3168
- distance_t radius = context.measure(query, node_with_id_(start_id));
3169
- next.insert_reserved({-radius, start_id});
3170
- top.insert_reserved({radius, start_id});
3171
- visits.set(start_id);
3061
+ // Optional prefetching
3062
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
3063
+ prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
3064
+
3065
+ distance_t radius = context.measure(query, citerator_at(start_slot), metric);
3066
+ next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
3067
+ top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
3068
+ visits.set(start_slot);
3172
3069
 
3173
3070
  while (!next.empty()) {
3174
3071
 
@@ -3179,28 +3076,29 @@ class index_gt {
3179
3076
  next.pop();
3180
3077
  context.iteration_cycles++;
3181
3078
 
3182
- id_t candidate_id = candidate.id;
3183
- neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
3079
+ neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
3184
3080
 
3185
- prefetch_neighbors_(candidate_neighbors, visits);
3186
- for (id_t successor_id : candidate_neighbors) {
3187
- if (visits.test(successor_id))
3188
- continue;
3081
+ // Optional prefetching
3082
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
3083
+ candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
3084
+ prefetch(missing_candidates.begin(), missing_candidates.end());
3085
+ }
3189
3086
 
3190
- visits.set(successor_id);
3191
- node_t successor = node_with_id_(successor_id);
3192
- distance_t successor_dist = context.measure(query, successor);
3087
+ for (compressed_slot_t successor_slot : candidate_neighbors) {
3088
+ if (visits.set(successor_slot))
3089
+ continue;
3193
3090
 
3091
+ distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
3194
3092
  if (top.size() < top_limit || successor_dist < radius) {
3195
3093
  // This can substantially grow our priority queue:
3196
- next.insert({-successor_dist, successor_id});
3197
- if (predicate( //
3198
- match_t{member_cref_t{successor.label(), successor.vector_view(), successor_id},
3199
- successor_dist})) {
3200
- // This will automatically evict poor matches:
3201
- top.insert({successor_dist, successor_id}, top_limit);
3202
- radius = top.top().distance;
3203
- }
3094
+ next.insert({-successor_dist, successor_slot});
3095
+ if (!is_dummy<predicate_at>())
3096
+ if (!predicate(member_cref_t{node_at_(successor_slot).ckey(), successor_slot}))
3097
+ continue;
3098
+
3099
+ // This will automatically evict poor matches:
3100
+ top.insert({successor_dist, successor_slot}, top_limit);
3101
+ radius = top.top().distance;
3204
3102
  }
3205
3103
  }
3206
3104
  }
@@ -3209,33 +3107,35 @@ class index_gt {
3209
3107
  }
3210
3108
 
3211
3109
  /**
3212
- * @brief Iterates through all managed vectors, without actually touching the index.
3110
+ * @brief Iterates through all members, without actually touching the index.
3213
3111
  */
3214
- template <typename predicate_at>
3215
- void search_exact_( //
3216
- vector_view_t query, std::size_t count, context_t& context, //
3217
- predicate_at&& predicate) const noexcept {
3112
+ template <typename value_at, typename metric_at, typename predicate_at>
3113
+ void search_exact_( //
3114
+ value_at&& query, metric_at&& metric, predicate_at&& predicate, //
3115
+ std::size_t count, context_t& context) const noexcept {
3218
3116
 
3219
3117
  top_candidates_t& top = context.top_candidates;
3220
3118
  top.clear();
3221
3119
  top.reserve(count);
3222
3120
  for (std::size_t i = 0; i != size(); ++i) {
3223
- id_t id = static_cast<id_t>(i);
3224
- node_t node = node_with_id_(i);
3225
- distance_t distance = context.measure(query, node);
3226
- if (predicate(match_t{member_cref_t{node.label(), node.vector_view(), id}, distance}))
3227
- top.insert(candidate_t{distance, id}, count);
3121
+ if (!is_dummy<predicate_at>())
3122
+ if (!predicate(at(i)))
3123
+ continue;
3124
+
3125
+ distance_t distance = context.measure(query, citerator_at(i), metric);
3126
+ top.insert(candidate_t{distance, static_cast<compressed_slot_t>(i)}, count);
3228
3127
  }
3229
3128
  }
3230
3129
 
3231
- void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
3232
-
3233
3130
  /**
3234
3131
  * @brief This algorithm from the original paper implements a heuristic,
3235
3132
  * that massively reduces the number of connections a point has,
3236
3133
  * to keep only the neighbors, that are from each other.
3237
3134
  */
3238
- candidates_view_t refine_(top_candidates_t& top, std::size_t needed, context_t& context) const noexcept {
3135
+ template <typename metric_at>
3136
+ candidates_view_t refine_( //
3137
+ metric_at&& metric, //
3138
+ std::size_t needed, top_candidates_t& top, context_t& context) const noexcept {
3239
3139
 
3240
3140
  top.sort_ascending();
3241
3141
  candidate_t* top_data = top.data();
@@ -3247,14 +3147,14 @@ class index_gt {
3247
3147
  std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
3248
3148
  while (submitted_count < needed && consumed_count < top_count) {
3249
3149
  candidate_t candidate = top_data[consumed_count];
3250
- node_t candidate_ref = node_with_id_(candidate.id);
3251
- distance_t candidate_dist = candidate.distance;
3252
3150
  bool good = true;
3253
3151
  for (std::size_t idx = 0; idx < submitted_count; idx++) {
3254
3152
  candidate_t submitted = top_data[idx];
3255
- node_t submitted_node = node_with_id_(submitted.id);
3256
- distance_t inter_result_dist = context.measure(submitted_node, candidate_ref);
3257
- if (inter_result_dist < candidate_dist) {
3153
+ distance_t inter_result_dist = context.measure( //
3154
+ citerator_at(candidate.slot), //
3155
+ citerator_at(submitted.slot), //
3156
+ metric);
3157
+ if (inter_result_dist < candidate.distance) {
3258
3158
  good = false;
3259
3159
  break;
3260
3160
  }
@@ -3272,44 +3172,218 @@ class index_gt {
3272
3172
  }
3273
3173
  };
3274
3174
 
3175
+ struct join_result_t {
3176
+ error_t error{};
3177
+ std::size_t intersection_size{};
3178
+ std::size_t engagements{};
3179
+ std::size_t visited_members{};
3180
+ std::size_t computed_distances{};
3181
+
3182
+ explicit operator bool() const noexcept { return !error; }
3183
+ join_result_t failed(error_t message) noexcept {
3184
+ error = std::move(message);
3185
+ return std::move(*this);
3186
+ }
3187
+ };
3188
+
3275
3189
  /**
3276
- * @brief Extracts metadata from pre-constructed index on disk, without loading it
3277
- * or mapping the whole binary file.
3190
+ * @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
3191
+ * to perform fast one-to-one matching between two large collections
3192
+ * of vectors, using approximate nearest neighbors search.
3193
+ *
3194
+ * @param[inout] man_to_woman Container to map ::first keys to ::second.
3195
+ * @param[inout] woman_to_man Container to map ::second keys to ::first.
3196
+ * @param[in] executor Thread-pool to execute the job in parallel.
3197
+ * @param[in] progress Callback to report the execution progress.
3278
3198
  */
3279
- inline file_head_result_t index_metadata(char const* file_path) noexcept {
3280
-
3281
- file_head_result_t result;
3282
- file_header_t state_buffer{};
3283
- std::FILE* file = std::fopen(file_path, "rb");
3284
- if (!file)
3285
- return result.failed(std::strerror(errno));
3286
-
3287
- // Read the header
3288
- std::size_t read = std::fread(&state_buffer[0], sizeof(file_header_t), 1, file);
3289
- std::fclose(file);
3290
- if (!read)
3291
- return result.failed(std::strerror(errno));
3292
-
3293
- // Parse and validate the MIME type
3294
- file_head_t state{state_buffer};
3295
- if (std::strncmp(state.magic, default_magic(), std::strlen(default_magic())) != 0)
3296
- return result.failed("Wrong MIME type!");
3297
-
3298
- result.version_major = state.version_major;
3299
- result.version_minor = state.version_minor;
3300
- result.version_patch = state.version_patch;
3301
- result.metric = state.metric;
3302
- result.connectivity = state.connectivity;
3303
- result.max_level = state.max_level;
3304
- result.vector_alignment = state.vector_alignment;
3305
- result.bytes_per_label = state.bytes_per_label;
3306
- result.bytes_per_id = state.bytes_per_id;
3307
- result.scalar_kind = state.scalar_kind;
3308
- result.size = state.size;
3309
- result.entry_idx = state.entry_idx;
3310
- result.bytes_for_graphs = state.bytes_for_graphs;
3311
- result.bytes_for_vectors = state.bytes_for_vectors;
3312
- result.bytes_checksum = state.bytes_checksum;
3199
+ template < //
3200
+
3201
+ typename men_at, //
3202
+ typename women_at, //
3203
+ typename men_values_at, //
3204
+ typename women_values_at, //
3205
+ typename men_metric_at, //
3206
+ typename women_metric_at, //
3207
+
3208
+ typename man_to_woman_at = dummy_label_to_label_mapping_t, //
3209
+ typename woman_to_man_at = dummy_label_to_label_mapping_t, //
3210
+ typename executor_at = dummy_executor_t, //
3211
+ typename progress_at = dummy_progress_t //
3212
+ >
3213
+ static join_result_t join( //
3214
+ men_at const& men, //
3215
+ women_at const& women, //
3216
+ men_values_at const& men_values, //
3217
+ women_values_at const& women_values, //
3218
+ men_metric_at&& men_metric, //
3219
+ women_metric_at&& women_metric, //
3220
+
3221
+ index_join_config_t config = {}, //
3222
+ man_to_woman_at&& man_to_woman = man_to_woman_at{}, //
3223
+ woman_to_man_at&& woman_to_man = woman_to_man_at{}, //
3224
+ executor_at&& executor = executor_at{}, //
3225
+ progress_at&& progress = progress_at{}) noexcept {
3226
+
3227
+ if (women.size() < men.size())
3228
+ return unum::usearch::join( //
3229
+ women, men, //
3230
+ women_values, men_values, //
3231
+ std::forward<women_metric_at>(women_metric), std::forward<men_metric_at>(men_metric), //
3232
+
3233
+ config, //
3234
+ std::forward<woman_to_man_at>(woman_to_man), //
3235
+ std::forward<man_to_woman_at>(man_to_woman), //
3236
+ std::forward<executor_at>(executor), //
3237
+ std::forward<progress_at>(progress));
3238
+
3239
+ join_result_t result;
3240
+
3241
+ // Sanity checks and argument validation:
3242
+ if (&men == &women)
3243
+ return result.failed("Can't join with itself, consider copying");
3244
+
3245
+ if (config.max_proposals == 0)
3246
+ config.max_proposals = std::log(men.size()) + executor.size();
3247
+
3248
+ using proposals_count_t = std::uint16_t;
3249
+ config.max_proposals = (std::min)(men.size(), config.max_proposals);
3250
+
3251
+ using distance_t = typename men_at::distance_t;
3252
+ using dynamic_allocator_traits_t = typename men_at::dynamic_allocator_traits_t;
3253
+ using man_key_t = typename men_at::key_t;
3254
+ using woman_key_t = typename women_at::key_t;
3255
+
3256
+ // Use the `compressed_slot_t` type of the larger collection
3257
+ using compressed_slot_t = typename women_at::compressed_slot_t;
3258
+ using compressed_slot_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<compressed_slot_t>;
3259
+ using proposals_count_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<proposals_count_t>;
3260
+
3261
+ // Create an atomic queue, as a ring structure, from/to which
3262
+ // free men will be added/pulled.
3263
+ std::mutex free_men_mutex{};
3264
+ ring_gt<compressed_slot_t, compressed_slot_allocator_t> free_men;
3265
+ free_men.reserve(men.size());
3266
+ for (std::size_t i = 0; i != men.size(); ++i)
3267
+ free_men.push(static_cast<compressed_slot_t>(i));
3268
+
3269
+ // We are gonna need some temporary memory.
3270
+ buffer_gt<proposals_count_t, proposals_count_allocator_t> proposal_counts(men.size());
3271
+ buffer_gt<compressed_slot_t, compressed_slot_allocator_t> man_to_woman_slots(men.size());
3272
+ buffer_gt<compressed_slot_t, compressed_slot_allocator_t> woman_to_man_slots(women.size());
3273
+ if (!proposal_counts || !man_to_woman_slots || !woman_to_man_slots)
3274
+ return result.failed("Can't temporary mappings");
3275
+
3276
+ compressed_slot_t missing_slot;
3277
+ std::memset((void*)&missing_slot, 0xFF, sizeof(compressed_slot_t));
3278
+ std::memset((void*)man_to_woman_slots.data(), 0xFF, sizeof(compressed_slot_t) * men.size());
3279
+ std::memset((void*)woman_to_man_slots.data(), 0xFF, sizeof(compressed_slot_t) * women.size());
3280
+ std::memset(proposal_counts.data(), 0, sizeof(proposals_count_t) * men.size());
3281
+
3282
+ // Define locks, to limit concurrent accesses to `man_to_woman_slots` and `woman_to_man_slots`.
3283
+ visits_bitset_t men_locks(men.size()), women_locks(women.size());
3284
+ if (!men_locks || !women_locks)
3285
+ return result.failed("Can't allocate locks");
3286
+
3287
+ std::atomic<std::size_t> rounds{0};
3288
+ std::atomic<std::size_t> engagements{0};
3289
+ std::atomic<std::size_t> computed_distances{0};
3290
+ std::atomic<std::size_t> visited_members{0};
3291
+
3292
+ // Concurrently process all the men
3293
+ executor.execute_bulk([&](std::size_t thread_idx) {
3294
+ index_search_config_t search_config;
3295
+ search_config.expansion = config.expansion;
3296
+ search_config.exact = config.exact;
3297
+ search_config.thread = thread_idx;
3298
+ compressed_slot_t free_man_slot;
3299
+
3300
+ // While there exist a free man who still has a woman to propose to.
3301
+ while (true) {
3302
+ std::size_t passed_rounds = 0;
3303
+ std::size_t total_rounds = 0;
3304
+ {
3305
+ std::unique_lock<std::mutex> pop_lock(free_men_mutex);
3306
+ if (!free_men.try_pop(free_man_slot))
3307
+ // Primary exit path, we have exhausted the list of candidates
3308
+ break;
3309
+ passed_rounds = ++rounds;
3310
+ total_rounds = passed_rounds + free_men.size();
3311
+ }
3312
+ progress(passed_rounds, total_rounds);
3313
+ while (men_locks.atomic_set(free_man_slot))
3314
+ ;
3315
+
3316
+ proposals_count_t& free_man_proposals = proposal_counts[free_man_slot];
3317
+ if (free_man_proposals >= config.max_proposals)
3318
+ continue;
3319
+
3320
+ // Find the closest woman, to whom this man hasn't proposed yet.
3321
+ ++free_man_proposals;
3322
+ auto candidates = women.search(men_values[free_man_slot], free_man_proposals, women_metric, search_config);
3323
+ visited_members += candidates.visited_members;
3324
+ computed_distances += candidates.computed_distances;
3325
+ if (!candidates) {
3326
+ // TODO:
3327
+ }
3328
+
3329
+ auto match = candidates.back();
3330
+ auto woman = match.member;
3331
+ while (women_locks.atomic_set(woman.slot))
3332
+ ;
3333
+
3334
+ compressed_slot_t husband_slot = woman_to_man_slots[woman.slot];
3335
+ bool woman_is_free = husband_slot == missing_slot;
3336
+ if (woman_is_free) {
3337
+ // Engagement
3338
+ man_to_woman_slots[free_man_slot] = woman.slot;
3339
+ woman_to_man_slots[woman.slot] = free_man_slot;
3340
+ engagements++;
3341
+ } else {
3342
+ distance_t distance_from_husband = women_metric(women_values[woman.slot], men_values[husband_slot]);
3343
+ distance_t distance_from_candidate = match.distance;
3344
+ if (distance_from_husband > distance_from_candidate) {
3345
+ // Break-up
3346
+ while (men_locks.atomic_set(husband_slot))
3347
+ ;
3348
+ man_to_woman_slots[husband_slot] = missing_slot;
3349
+ men_locks.atomic_reset(husband_slot);
3350
+
3351
+ // New Engagement
3352
+ man_to_woman_slots[free_man_slot] = woman.slot;
3353
+ woman_to_man_slots[woman.slot] = free_man_slot;
3354
+ engagements++;
3355
+
3356
+ std::unique_lock<std::mutex> push_lock(free_men_mutex);
3357
+ free_men.push(husband_slot);
3358
+ } else {
3359
+ std::unique_lock<std::mutex> push_lock(free_men_mutex);
3360
+ free_men.push(free_man_slot);
3361
+ }
3362
+ }
3363
+
3364
+ men_locks.atomic_reset(free_man_slot);
3365
+ women_locks.atomic_reset(woman.slot);
3366
+ }
3367
+ });
3368
+
3369
+ // Export the "slots" into keys:
3370
+ std::size_t intersection_size = 0;
3371
+ for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
3372
+ compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
3373
+ if (woman_slot != missing_slot) {
3374
+ man_key_t man = men.at(man_slot).key;
3375
+ woman_key_t woman = women.at(woman_slot).key;
3376
+ man_to_woman[man] = woman;
3377
+ woman_to_man[woman] = man;
3378
+ intersection_size++;
3379
+ }
3380
+ }
3381
+
3382
+ // Export stats
3383
+ result.engagements = engagements;
3384
+ result.intersection_size = intersection_size;
3385
+ result.computed_distances = computed_distances;
3386
+ result.visited_members = visited_members;
3313
3387
  return result;
3314
3388
  }
3315
3389