usearch 0.22.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/.github/workflows/prerelease.yml +3 -3
  2. package/.releaserc +10 -2
  3. package/.vscode/launch.json +10 -10
  4. package/.vscode/settings.json +9 -1
  5. package/.vscode/tasks.json +4 -4
  6. package/CMakeLists.txt +6 -1
  7. package/Cargo.toml +2 -2
  8. package/README.md +35 -41
  9. package/build.rs +2 -2
  10. package/c/CMakeLists.txt +48 -0
  11. package/c/README.md +34 -0
  12. package/c/lib.cpp +83 -72
  13. package/c/test.c +20 -20
  14. package/c/usearch.h +132 -32
  15. package/conanfile.py +1 -1
  16. package/cpp/CMakeLists.txt +3 -1
  17. package/cpp/README.md +16 -16
  18. package/cpp/bench.cpp +50 -126
  19. package/cpp/test.cpp +164 -80
  20. package/docs/benchmarks.md +26 -5
  21. package/docs/c/reference.rst +3 -3
  22. package/docs/compilation.md +31 -49
  23. package/docs/cpp/index.rst +1 -1
  24. package/docs/cpp/reference.rst +4 -4
  25. package/golang/lib.go +275 -141
  26. package/golang/lib_test.go +105 -0
  27. package/include/usearch/index.hpp +1681 -1656
  28. package/include/usearch/index_dense.hpp +1400 -0
  29. package/include/usearch/index_plugins.hpp +1263 -0
  30. package/java/README.md +1 -1
  31. package/java/cloud/unum/usearch/Index.java +10 -10
  32. package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
  33. package/java/test/IndexTest.java +1 -1
  34. package/javascript/README.md +6 -6
  35. package/javascript/lib.cpp +94 -55
  36. package/javascript/test.js +25 -11
  37. package/javascript/usearch.d.ts +87 -0
  38. package/javascript/usearch.js +2 -118
  39. package/objc/USearchObjective.mm +57 -31
  40. package/objc/include/USearchObjective.h +32 -22
  41. package/package.json +5 -3
  42. package/python/README.md +23 -14
  43. package/python/lib.cpp +468 -444
  44. package/python/scripts/bench.ipynb +3 -2
  45. package/python/scripts/bench.py +4 -4
  46. package/python/scripts/index_faiss.py +5 -5
  47. package/python/scripts/join.py +4 -4
  48. package/python/scripts/test.py +152 -61
  49. package/python/scripts/test_jit.py +10 -12
  50. package/python/scripts/test_sparse.py +32 -0
  51. package/python/usearch/client.py +26 -28
  52. package/python/usearch/eval.py +73 -23
  53. package/python/usearch/index.py +520 -228
  54. package/python/usearch/io.py +57 -25
  55. package/python/usearch/numba.py +4 -4
  56. package/python/usearch/server.py +10 -10
  57. package/rust/lib.cpp +22 -27
  58. package/rust/lib.hpp +11 -12
  59. package/rust/lib.rs +3 -3
  60. package/setup.py +5 -2
  61. package/simsimd/.github/workflows/prerelease.yml +73 -0
  62. package/simsimd/.github/workflows/release.yml +110 -0
  63. package/simsimd/.github/workflows/update_version.sh +3 -0
  64. package/simsimd/.releaserc +77 -0
  65. package/simsimd/.vscode/launch.json +27 -1
  66. package/simsimd/.vscode/settings.json +22 -1
  67. package/simsimd/.vscode/tasks.json +6 -0
  68. package/simsimd/CMakeLists.txt +3 -2
  69. package/simsimd/README.md +82 -46
  70. package/simsimd/bench.cxx +63 -24
  71. package/simsimd/include/simsimd/simsimd.h +88 -66
  72. package/simsimd/include/simsimd/simsimd_chem.h +126 -0
  73. package/simsimd/package.json +8 -0
  74. package/simsimd/pyproject.toml +42 -0
  75. package/simsimd/python/python.c +81 -0
  76. package/simsimd/python/run.sh +1 -0
  77. package/simsimd/python/test.c +109 -0
  78. package/simsimd/python/test.py +23 -0
  79. package/simsimd/setup.py +44 -0
  80. package/swift/Index+Sugar.swift +75 -23
  81. package/swift/README.md +26 -2
  82. package/swift/Test.swift +19 -7
  83. package/swift/USearch.swift +2 -2
  84. package/tsconfig.json +8 -0
  85. package/wasm/CMakeLists.txt +1 -0
  86. package/wasmer.toml +1 -1
  87. package/wolfram/CMakeLists.txt +1 -0
  88. package/wolfram/lib.cpp +11 -13
  89. package/golang/usearch.h +0 -9
  90. package/golang/usearch_par_test.go +0 -40
  91. package/golang/usearch_test.go +0 -82
  92. package/include/usearch/index_punned_dense.hpp +0 -1001
  93. package/include/usearch/index_punned_helpers.hpp +0 -676
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @file index.hpp
3
- * @author Ashot Vardanian
3
+ * @author Ash Vardanian
4
4
  * @brief Single-header Vector Search.
5
5
  * @date 2023-04-26
6
6
  *
@@ -14,6 +14,7 @@
14
14
  #define USEARCH_VERSION_PATCH 0
15
15
 
16
16
  // Inferring C++ version
17
+ // https://stackoverflow.com/a/61552074
17
18
  #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
18
19
  #define USEARCH_DEFINED_CPP17
19
20
  #endif
@@ -42,10 +43,13 @@
42
43
  #endif
43
44
 
44
45
  // Inferring hardware bitness: 32 vs 64
45
- #if __WORDSIZE == 64
46
+ // https://stackoverflow.com/a/5273354
47
+ #if INTPTR_MAX == INT64_MAX
46
48
  #define USEARCH_64BIT_ENV
47
- #else
49
+ #elif INTPTR_MAX == INT32_MAX
48
50
  #define USEARCH_32BIT_ENV
51
+ #else
52
+ #error Unknown pointer size or missing size macros!
49
53
  #endif
50
54
 
51
55
  #if !defined(USEARCH_USE_OPENMP)
@@ -119,51 +123,7 @@
119
123
  namespace unum {
120
124
  namespace usearch {
121
125
 
122
- using f64_t = double;
123
- using f32_t = float;
124
126
  using byte_t = char;
125
- enum b1x8_t : unsigned char {};
126
-
127
- enum class metric_kind_t : std::uint8_t {
128
- unknown_k = 0,
129
- // Classics:
130
- ip_k = 'i',
131
- cos_k = 'c',
132
- l2sq_k = 'e',
133
-
134
- // Custom:
135
- pearson_k = 'p',
136
- haversine_k = 'h',
137
-
138
- // Sets:
139
- jaccard_k = 'j',
140
- hamming_k = 'b',
141
- tanimoto_k = 't',
142
- sorensen_k = 's',
143
- };
144
-
145
- enum class scalar_kind_t : std::uint8_t {
146
- unknown_k = 0,
147
- f64_k,
148
- f32_k,
149
- f16_k,
150
- f8_k,
151
- b1x8_k,
152
- };
153
-
154
- template <typename scalar_at> scalar_kind_t common_scalar_kind() noexcept {
155
- if (std::is_same<scalar_at, f32_t>())
156
- return scalar_kind_t::f32_k;
157
- if (std::is_same<scalar_at, f64_t>())
158
- return scalar_kind_t::f64_k;
159
- if (std::is_same<scalar_at, b1x8_t>())
160
- return scalar_kind_t::b1x8_k;
161
- return scalar_kind_t::unknown_k;
162
- }
163
-
164
- template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
165
-
166
- template <typename at> at square(at value) noexcept { return value * value; }
167
127
 
168
128
  template <std::size_t multiple_ak> std::size_t divide_round_up(std::size_t num) noexcept {
169
129
  return (num + multiple_ak - 1) / multiple_ak;
@@ -187,6 +147,7 @@ inline std::size_t ceil2(std::size_t v) noexcept {
187
147
  return v;
188
148
  }
189
149
 
150
+ /// @brief Simply dereferencing misaligned pointers can be dangerous.
190
151
  template <typename at> void misaligned_store(void* ptr, at v) noexcept {
191
152
  static_assert(!std::is_reference<at>::value, "Can't store a reference");
192
153
  std::memcpy(ptr, &v, sizeof(at));
@@ -207,54 +168,80 @@ template <typename at, typename other_at = at> at exchange(at& obj, other_at&& n
207
168
  return old_value;
208
169
  }
209
170
 
171
+ /// @brief The `std::destroy_at` alternative for C++11.
172
+ template <typename at, typename sfinae_at = at>
173
+ typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
174
+ template <typename at, typename sfinae_at = at>
175
+ typename std::enable_if<!std::is_pod<sfinae_at>::value>::type destroy_at(at* obj) {
176
+ obj->~sfinae_at();
177
+ }
178
+
179
+ /// @brief The `std::construct_at` alternative for C++11.
180
+ template <typename at, typename sfinae_at = at>
181
+ typename std::enable_if<std::is_pod<sfinae_at>::value>::type construct_at(at*) {}
182
+ template <typename at, typename sfinae_at = at>
183
+ typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* obj) {
184
+ new (obj) at();
185
+ }
186
+
187
+ /**
188
+ * @brief A reference to a misaligned memory location with a specific type.
189
+ * It is needed to avoid Undefined Behavior when dereferencing addresses
190
+ * indivisible by `sizeof(at)`.
191
+ */
210
192
  template <typename at> class misaligned_ref_gt {
193
+ using element_t = at;
194
+ using mutable_t = typename std::remove_const<element_t>::type;
211
195
  byte_t* ptr_;
212
196
 
213
197
  public:
214
198
  misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
215
- operator at() const noexcept { return misaligned_load<at>(ptr_); }
216
- misaligned_ref_gt& operator=(at const& v) noexcept {
217
- misaligned_store<at>(ptr_, v);
199
+ operator mutable_t() const noexcept { return misaligned_load<mutable_t>(ptr_); }
200
+ misaligned_ref_gt& operator=(mutable_t const& v) noexcept {
201
+ misaligned_store<mutable_t>(ptr_, v);
218
202
  return *this;
219
203
  }
220
204
 
221
205
  void reset(byte_t* ptr) noexcept { ptr_ = ptr; }
206
+ byte_t* ptr() const noexcept { return ptr_; }
222
207
  };
223
208
 
209
+ /**
210
+ * @brief A pointer to a misaligned memory location with a specific type.
211
+ * It is needed to avoid Undefined Behavior when dereferencing addresses
212
+ * indivisible by `sizeof(at)`.
213
+ */
224
214
  template <typename at> class misaligned_ptr_gt {
215
+ using element_t = at;
216
+ using mutable_t = typename std::remove_const<element_t>::type;
225
217
  byte_t* ptr_;
226
218
 
227
219
  public:
228
220
  using iterator_category = std::random_access_iterator_tag;
229
- using value_type = at;
221
+ using value_type = element_t;
230
222
  using difference_type = std::ptrdiff_t;
231
- using pointer = misaligned_ptr_gt<at>;
232
- using reference = misaligned_ref_gt<at>;
223
+ using pointer = misaligned_ptr_gt<element_t>;
224
+ using reference = misaligned_ref_gt<element_t>;
233
225
 
234
226
  reference operator*() const noexcept { return {ptr_}; }
227
+ reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
228
+ value_type operator[](std::size_t i) const noexcept {
229
+ return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
230
+ }
235
231
 
236
232
  misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
237
- misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(at)); }
238
- misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(at)); }
239
- misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(at)); }
240
- misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(at)); }
233
+ misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(element_t)); }
234
+ misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(element_t)); }
235
+ misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(element_t)); }
236
+ misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(element_t)); }
237
+
238
+ // clang-format off
239
+ misaligned_ptr_gt& operator++() noexcept { ptr_ += sizeof(element_t); return *this; }
240
+ misaligned_ptr_gt& operator--() noexcept { ptr_ -= sizeof(element_t); return *this; }
241
+ misaligned_ptr_gt& operator+=(difference_type d) noexcept { ptr_ += d * sizeof(element_t); return *this; }
242
+ misaligned_ptr_gt& operator-=(difference_type d) noexcept { ptr_ -= d * sizeof(element_t); return *this; }
243
+ // clang-format on
241
244
 
242
- misaligned_ptr_gt& operator++() noexcept {
243
- ptr_ += sizeof(at);
244
- return *this;
245
- }
246
- misaligned_ptr_gt& operator--() noexcept {
247
- ptr_ -= sizeof(at);
248
- return *this;
249
- }
250
- misaligned_ptr_gt& operator+=(difference_type d) noexcept {
251
- ptr_ += d * sizeof(at);
252
- return *this;
253
- }
254
- misaligned_ptr_gt& operator-=(difference_type d) noexcept {
255
- ptr_ -= d * sizeof(at);
256
- return *this;
257
- }
258
245
  bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
259
246
  bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
260
247
  };
@@ -278,307 +265,56 @@ template <typename scalar_at> class span_gt {
278
265
  };
279
266
 
280
267
  /**
281
- * @brief Inner (Dot) Product distance.
282
- */
283
- template <typename scalar_at = float, typename result_at = scalar_at> struct ip_gt {
284
- using scalar_t = scalar_at;
285
- using view_t = span_gt<scalar_t const>;
286
- using result_t = result_at;
287
- using result_type = result_t;
288
-
289
- inline metric_kind_t kind() const noexcept { return metric_kind_t::ip_k; }
290
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
291
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
292
-
293
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
294
- result_type ab{};
295
- #if USEARCH_USE_OPENMP
296
- #pragma omp simd reduction(+ : ab)
297
- #elif defined(USEARCH_DEFINED_CLANG)
298
- #pragma clang loop vectorize(enable)
299
- #elif defined(USEARCH_DEFINED_GCC)
300
- #pragma GCC ivdep
301
- #endif
302
- for (std::size_t i = 0; i != dim; ++i)
303
- ab += result_t(a[i]) * result_t(b[i]);
304
- return 1 - ab;
305
- }
306
- };
307
-
308
- /**
309
- * @brief Cosine (Angular) distance.
310
- * Identical to the Inner Product of normalized vectors.
311
- * Unless you are running on an tiny embedded platform, this metric
312
- * is recommended over `::ip_gt` for low-precision scalars.
313
- */
314
- template <typename scalar_at = float, typename result_at = scalar_at> struct cos_gt {
315
- using scalar_t = scalar_at;
316
- using view_t = span_gt<scalar_t const>;
317
- using result_t = result_at;
318
- using result_type = result_t;
319
-
320
- inline metric_kind_t kind() const noexcept { return metric_kind_t::cos_k; }
321
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
322
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
323
-
324
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
325
- result_t ab{}, a2{}, b2{};
326
- #if USEARCH_USE_OPENMP
327
- #pragma omp simd reduction(+ : ab, a2, b2)
328
- #elif defined(USEARCH_DEFINED_CLANG)
329
- #pragma clang loop vectorize(enable)
330
- #elif defined(USEARCH_DEFINED_GCC)
331
- #pragma GCC ivdep
332
- #endif
333
- for (std::size_t i = 0; i != dim; ++i)
334
- ab += result_t(a[i]) * result_t(b[i]), //
335
- a2 += square<result_t>(a[i]), //
336
- b2 += square<result_t>(b[i]);
337
- return (ab != 0) ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 1;
338
- }
339
- };
340
-
341
- /**
342
- * @brief Squared Euclidean (L2) distance.
343
- * Square root is avoided at the end, as it won't affect the ordering.
268
+ * @brief Similar to `std::vector`, but doesn't support dynamic resizing.
269
+ * On the bright side, this can't throw exceptions.
344
270
  */
345
- template <typename scalar_at = float, typename result_at = scalar_at> struct l2sq_gt {
346
- using scalar_t = scalar_at;
347
- using view_t = span_gt<scalar_t const>;
348
- using result_t = result_at;
349
- using result_type = result_t;
350
-
351
- inline metric_kind_t kind() const noexcept { return metric_kind_t::l2sq_k; }
352
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
353
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
354
-
355
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
356
- result_t ab_deltas_sq{};
357
- #if USEARCH_USE_OPENMP
358
- #pragma omp simd reduction(+ : ab_deltas_sq)
359
- #elif defined(USEARCH_DEFINED_CLANG)
360
- #pragma clang loop vectorize(enable)
361
- #elif defined(USEARCH_DEFINED_GCC)
362
- #pragma GCC ivdep
363
- #endif
364
- for (std::size_t i = 0; i != dim; ++i)
365
- ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
366
- return ab_deltas_sq;
367
- }
368
- };
369
-
370
- /**
371
- * @brief Hamming distance computes the number of differing bits in
372
- * two arrays of integers. An example would be a textual document,
373
- * tokenized and hashed into a fixed-capacity bitset.
374
- */
375
- template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct hamming_gt {
376
- using scalar_t = scalar_at;
377
- using view_t = span_gt<scalar_t const>;
378
- using result_t = result_at;
379
- using result_type = result_t;
380
- static_assert( //
381
- std::is_unsigned<scalar_t>::value ||
382
- (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
383
- "Hamming distance requires unsigned integral words");
384
-
385
- inline metric_kind_t kind() const noexcept { return metric_kind_t::hamming_k; }
386
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
387
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
388
-
389
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
390
- constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
391
- result_t matches{};
392
- #if USEARCH_USE_OPENMP
393
- #pragma omp simd reduction(+ : matches)
394
- #elif defined(USEARCH_DEFINED_CLANG)
395
- #pragma clang loop vectorize(enable)
396
- #elif defined(USEARCH_DEFINED_GCC)
397
- #pragma GCC ivdep
398
- #endif
399
- for (std::size_t i = 0; i != words; ++i)
400
- matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
401
- return matches;
402
- }
403
- };
271
+ template <typename scalar_at, typename allocator_at> class buffer_gt {
272
+ scalar_at* data_;
273
+ std::size_t size_;
404
274
 
405
- /**
406
- * @brief Tanimoto distance is the intersection over bitwise union.
407
- * Often used in chemistry and biology to compare molecular fingerprints.
408
- */
409
- template <typename scalar_at = std::uint64_t, typename result_at = float> struct tanimoto_gt {
410
- using scalar_t = scalar_at;
411
- using view_t = span_gt<scalar_t const>;
412
- using result_t = result_at;
413
- using result_type = result_t;
414
- static_assert( //
415
- std::is_unsigned<scalar_t>::value ||
416
- (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
417
- "Tanimoto distance requires unsigned integral words");
418
- static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
419
-
420
- inline metric_kind_t kind() const noexcept { return metric_kind_t::tanimoto_k; }
421
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
422
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
423
-
424
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
425
- constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
426
- result_t and_count{};
427
- result_t or_count{};
428
- #if USEARCH_USE_OPENMP
429
- #pragma omp simd reduction(+ : and_count, or_count)
430
- #elif defined(USEARCH_DEFINED_CLANG)
431
- #pragma clang loop vectorize(enable)
432
- #elif defined(USEARCH_DEFINED_GCC)
433
- #pragma GCC ivdep
434
- #endif
435
- for (std::size_t i = 0; i != words; ++i)
436
- and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
437
- or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
438
- return 1 - result_t(and_count) / or_count;
275
+ public:
276
+ buffer_gt() noexcept : data_(nullptr), size_(0u) {}
277
+ buffer_gt(std::size_t size) noexcept : data_(allocator_at{}.allocate(size)), size_(data_ ? size : 0u) {
278
+ if (!std::is_trivially_default_constructible<scalar_at>::value)
279
+ for (std::size_t i = 0; i != size_; ++i)
280
+ construct_at(data_ + i);
281
+ }
282
+ ~buffer_gt() noexcept {
283
+ if (!std::is_trivially_destructible<scalar_at>::value)
284
+ for (std::size_t i = 0; i != size_; ++i)
285
+ destroy_at(data_ + i);
286
+ allocator_at{}.deallocate(data_, size_);
287
+ data_ = nullptr;
288
+ size_ = 0;
439
289
  }
440
- };
441
-
442
- /**
443
- * @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
444
- * Often used in chemistry and biology to compare molecular fingerprints.
445
- */
446
- template <typename scalar_at = std::uint64_t, typename result_at = float> struct sorensen_gt {
447
- using scalar_t = scalar_at;
448
- using view_t = span_gt<scalar_t const>;
449
- using result_t = result_at;
450
- using result_type = result_t;
451
- static_assert( //
452
- std::is_unsigned<scalar_t>::value ||
453
- (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
454
- "Sorensen-Dice distance requires unsigned integral words");
455
- static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
456
-
457
- inline metric_kind_t kind() const noexcept { return metric_kind_t::sorensen_k; }
458
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
459
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
460
-
461
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
462
- constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
463
- result_t and_count{};
464
- result_t any_count{};
465
- #if USEARCH_USE_OPENMP
466
- #pragma omp simd reduction(+ : and_count, any_count)
467
- #elif defined(USEARCH_DEFINED_CLANG)
468
- #pragma clang loop vectorize(enable)
469
- #elif defined(USEARCH_DEFINED_GCC)
470
- #pragma GCC ivdep
471
- #endif
472
- for (std::size_t i = 0; i != words; ++i)
473
- and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
474
- any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
475
- return 1 - 2 * result_t(and_count) / any_count;
290
+ scalar_at* data() const noexcept { return data_; }
291
+ std::size_t size() const noexcept { return size_; }
292
+ scalar_at* begin() const noexcept { return data_; }
293
+ scalar_at* end() const noexcept { return data_ + size_; }
294
+ operator scalar_at*() const noexcept { return data(); }
295
+ scalar_at& operator[](std::size_t i) noexcept { return data_[i]; }
296
+ scalar_at const& operator[](std::size_t i) const noexcept { return data_[i]; }
297
+ explicit operator bool() const noexcept { return data_; }
298
+ scalar_at* release() noexcept {
299
+ size_ = 0;
300
+ return exchange(data_, nullptr);
476
301
  }
477
- };
478
302
 
479
- /**
480
- * @brief Counts the number of matching elements in two unique sorted sets.
481
- * Can be used to compute the similarity between two textual documents
482
- * using the IDs of tokens present in them.
483
- * Similar to `tanimoto_gt` for dense representations.
484
- */
485
- template <typename scalar_at = std::int32_t, typename result_at = float> struct jaccard_gt {
486
- using scalar_t = scalar_at;
487
- using view_t = span_gt<scalar_t const>;
488
- using result_t = result_at;
489
- using result_type = result_t;
490
- static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
491
-
492
- inline metric_kind_t kind() const noexcept { return metric_kind_t::jaccard_k; }
493
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
494
- inline result_t operator()(view_t a, view_t b) const noexcept {
495
- return operator()(a.data(), b.data(), a.size(), b.size());
496
- }
497
-
498
- inline result_t operator()( //
499
- scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
500
- result_t intersection{};
501
- std::size_t i{};
502
- std::size_t j{};
503
- while (i != a_length && j != b_length) {
504
- intersection += a[i] == b[j];
505
- i += a[i] < b[j];
506
- j += a[i] >= b[j];
507
- }
508
- return 1 - intersection / (a_length + b_length - intersection);
509
- }
510
- };
303
+ buffer_gt(buffer_gt const&) = delete;
304
+ buffer_gt& operator=(buffer_gt const&) = delete;
511
305
 
512
- /**
513
- * @brief Counts the number of matching elements in two unique sorted sets.
514
- * Can be used to compute the similarity between two textual documents
515
- * using the IDs of tokens present in them.
516
- */
517
- template <typename scalar_at = float, typename result_at = float> struct pearson_correlation_gt {
518
- using scalar_t = scalar_at;
519
- using view_t = span_gt<scalar_t const>;
520
- using result_t = result_at;
521
- using result_type = result_t;
522
-
523
- inline metric_kind_t kind() const noexcept { return metric_kind_t::pearson_k; }
524
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
525
- inline result_t operator()(view_t a, view_t b) const noexcept { return operator()(a.data(), b.data(), a.size()); }
526
-
527
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
528
- result_t a_sum{}, b_sum{}, ab_sum{};
529
- result_t a_sq_sum{}, b_sq_sum{};
530
- #if USEARCH_USE_OPENMP
531
- #pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
532
- #elif defined(USEARCH_DEFINED_CLANG)
533
- #pragma clang loop vectorize(enable)
534
- #elif defined(USEARCH_DEFINED_GCC)
535
- #pragma GCC ivdep
536
- #endif
537
- for (std::size_t i = 0; i != dim; ++i) {
538
- a_sum += result_t(a[i]);
539
- b_sum += result_t(b[i]);
540
- ab_sum += result_t(a[i]) * result_t(b[i]);
541
- a_sq_sum += result_t(a[i]) * result_t(a[i]);
542
- b_sq_sum += result_t(b[i]) * result_t(b[i]);
543
- }
544
- result_t denom = std::sqrt((dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum));
545
- result_t corr = (dim * ab_sum - a_sum * b_sum) / denom;
546
- return -corr;
306
+ buffer_gt(buffer_gt&& other) noexcept : data_(exchange(other.data_, nullptr)), size_(exchange(other.size_, 0)) {}
307
+ buffer_gt& operator=(buffer_gt&& other) noexcept {
308
+ std::swap(data_, other.data_);
309
+ std::swap(size_, other.size_);
310
+ return *this;
547
311
  }
548
312
  };
549
313
 
550
314
  /**
551
- * @brief Haversine distance for the shortest distance between two nodes on
552
- * the surface of a 3D sphere, defined with latitude and longitude.
315
+ * @brief A lightweight error class for handling error messages,
316
+ * which are expected to be allocated in static memory.
553
317
  */
554
- template <typename scalar_at = float, typename result_at = scalar_at> struct haversine_gt {
555
- using scalar_t = scalar_at;
556
- using view_t = span_gt<scalar_t const>;
557
- using result_t = result_at;
558
- using result_type = result_t;
559
- static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
560
-
561
- inline metric_kind_t kind() const noexcept { return metric_kind_t::haversine_k; }
562
- inline scalar_kind_t scalar_kind() const noexcept { return common_scalar_kind<scalar_t>(); }
563
-
564
- inline result_t operator()(scalar_t const* a, scalar_t const* b) const noexcept {
565
- result_t lat_a = a[0], lon_a = a[1];
566
- result_t lat_b = b[0], lon_b = b[1];
567
-
568
- result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a);
569
- result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a);
570
-
571
- result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
572
- result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
573
-
574
- result_t x = //
575
- square(std::sin(lat_delta / 2.f)) +
576
- std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta / 2.f));
577
-
578
- return std::atan2(std::sqrt(x), std::sqrt(1.f - x));
579
- }
580
- };
581
-
582
318
  class error_t {
583
319
  char const* message_{};
584
320
 
@@ -596,11 +332,14 @@ class error_t {
596
332
  explicit operator bool() const noexcept { return message_ != nullptr; }
597
333
  char const* what() const noexcept { return message_; }
598
334
 
599
- #if defined(__cpp_exceptions) && (1 == __cpp_exceptions)
335
+ #if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
600
336
  ~error_t() noexcept(false) {
601
- if (message_)
602
- if (!std::uncaught_exception())
603
- raise();
337
+ #if defined(USEARCH_DEFINED_CPP17)
338
+ if (message_ && std::uncaught_exceptions() == 0)
339
+ #else
340
+ if (message_ && std::uncaught_exception() == 0)
341
+ #endif
342
+ raise();
604
343
  }
605
344
  void raise() noexcept(false) {
606
345
  if (message_)
@@ -615,6 +354,13 @@ class error_t {
615
354
  #endif
616
355
  };
617
356
 
357
+ /**
358
+ * @brief Similar to `std::expected` in C++23, wraps a statement evaluation result,
359
+ * or an error. It's used to avoid raising exception, and gracefully propagate
360
+ * the error.
361
+ *
362
+ * @tparam result_at The type of the expected result.
363
+ */
618
364
  template <typename result_at> struct expected_gt {
619
365
  result_at result;
620
366
  error_t error;
@@ -637,49 +383,41 @@ template <typename result_at> struct expected_gt {
637
383
 
638
384
  /**
639
385
  * @brief Light-weight bitset implementation to track visited nodes during graph traversal.
640
- * Extends basic functionality with atomic operations.
386
+ * Extends basic functionality with @b atomic operations.
641
387
  */
642
- template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
388
+ template <typename allocator_at = std::allocator<byte_t>> class visits_bitset_gt {
643
389
  using allocator_t = allocator_at;
644
390
  using byte_t = typename allocator_t::value_type;
645
391
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
646
392
 
647
- using slot_t = unsigned long;
393
+ using compressed_slot_t = unsigned long;
648
394
 
649
- static constexpr std::size_t bits_per_slot() { return sizeof(slot_t) * CHAR_BIT; }
650
- static constexpr slot_t bits_mask() { return sizeof(slot_t) * CHAR_BIT - 1; }
395
+ static constexpr std::size_t bits_per_slot() { return sizeof(compressed_slot_t) * CHAR_BIT; }
396
+ static constexpr compressed_slot_t bits_mask() { return sizeof(compressed_slot_t) * CHAR_BIT - 1; }
397
+ static constexpr std::size_t slots(std::size_t bits) { return divide_round_up<bits_per_slot()>(bits); }
651
398
 
652
- slot_t* slots_{};
399
+ compressed_slot_t* slots_{};
653
400
  /// @brief Number of slots.
654
401
  std::size_t count_{};
655
402
 
656
403
  public:
657
404
  visits_bitset_gt() noexcept {}
658
405
  ~visits_bitset_gt() noexcept { reset(); }
659
- void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(slot_t)); }
406
+
407
+ explicit operator bool() const noexcept { return slots_; }
408
+ void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(compressed_slot_t)); }
660
409
 
661
410
  void reset() noexcept {
662
411
  if (slots_)
663
- allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(slot_t));
412
+ allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(compressed_slot_t));
664
413
  slots_ = nullptr;
665
414
  count_ = 0;
666
415
  }
667
416
 
668
- bool resize(std::size_t capacity) noexcept {
669
-
670
- std::size_t count = divide_round_up<bits_per_slot()>(capacity);
671
- if (count <= count_)
672
- return true;
673
-
674
- slot_t* slots = (slot_t*)allocator_t{}.allocate(count * sizeof(slot_t));
675
- if (!slots)
676
- return false;
677
-
678
- reset();
679
- count_ = count;
680
- slots_ = slots;
417
+ visits_bitset_gt(std::size_t capacity) noexcept
418
+ : slots_((compressed_slot_t*)allocator_t{}.allocate(slots(capacity) * sizeof(compressed_slot_t))),
419
+ count_(slots_ ? slots(capacity) : 0u) {
681
420
  clear();
682
- return true;
683
421
  }
684
422
 
685
423
  visits_bitset_gt(visits_bitset_gt&& other) noexcept {
@@ -697,33 +435,54 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
697
435
  visits_bitset_gt& operator=(visits_bitset_gt const&) = delete;
698
436
 
699
437
  inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
700
- inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
438
+ inline bool set(std::size_t i) noexcept {
439
+ compressed_slot_t& slot = slots_[i / bits_per_slot()];
440
+ compressed_slot_t mask{1ul << (i & bits_mask())};
441
+ bool value = slot & mask;
442
+ slot |= mask;
443
+ return value;
444
+ }
701
445
 
702
446
  #if defined(USEARCH_DEFINED_WINDOWS)
703
447
 
704
448
  inline bool atomic_set(std::size_t i) noexcept {
705
- slot_t mask{1ul << (i & bits_mask())};
449
+ compressed_slot_t mask{1ul << (i & bits_mask())};
706
450
  return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
707
451
  }
708
452
 
709
453
  inline void atomic_reset(std::size_t i) noexcept {
710
- slot_t mask{1ul << (i & bits_mask())};
454
+ compressed_slot_t mask{1ul << (i & bits_mask())};
711
455
  InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
712
456
  }
713
457
 
714
458
  #else
715
459
 
716
460
  inline bool atomic_set(std::size_t i) noexcept {
717
- slot_t mask{1ul << (i & bits_mask())};
461
+ compressed_slot_t mask{1ul << (i & bits_mask())};
718
462
  return __atomic_fetch_or(&slots_[i / bits_per_slot()], mask, __ATOMIC_ACQUIRE) & mask;
719
463
  }
720
464
 
721
465
  inline void atomic_reset(std::size_t i) noexcept {
722
- slot_t mask{1ul << (i & bits_mask())};
466
+ compressed_slot_t mask{1ul << (i & bits_mask())};
723
467
  __atomic_fetch_and(&slots_[i / bits_per_slot()], ~mask, __ATOMIC_RELEASE);
724
468
  }
725
469
 
726
470
  #endif
471
+
472
+ class lock_t {
473
+ visits_bitset_gt& bitset_;
474
+ std::size_t bit_offset_;
475
+
476
+ public:
477
+ inline ~lock_t() noexcept { bitset_.atomic_reset(bit_offset_); }
478
+ inline lock_t(visits_bitset_gt& bitset, std::size_t bit_offset) noexcept
479
+ : bitset_(bitset), bit_offset_(bit_offset) {
480
+ while (bitset_.atomic_set(bit_offset_))
481
+ ;
482
+ }
483
+ };
484
+
485
+ inline lock_t lock(std::size_t i) noexcept { return {*this, i}; }
727
486
  };
728
487
 
729
488
  using visits_bitset_t = visits_bitset_gt<>;
@@ -1006,10 +765,10 @@ class usearch_pack_m uint40_t {
1006
765
 
1007
766
  public:
1008
767
  inline uint40_t() noexcept { broadcast(0); }
1009
- inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets, (char*)&n, 4); }
1010
- inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n, 5); }
1011
- #if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
1012
- inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n, 5); }
768
+ inline uint40_t(std::uint32_t n) noexcept { std::memcpy(&octets[1], &n, 4); }
769
+
770
+ #ifdef USEARCH_64BIT_ENV
771
+ inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
1013
772
  #endif
1014
773
 
1015
774
  uint40_t(uint40_t&&) = default;
@@ -1017,47 +776,26 @@ class usearch_pack_m uint40_t {
1017
776
  uint40_t& operator=(uint40_t&&) = default;
1018
777
  uint40_t& operator=(uint40_t const&) = default;
1019
778
 
1020
- inline uint40_t& operator+=(std::uint32_t n) noexcept {
1021
- std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
1022
- octets[4] += static_cast<unsigned char>((tail + n) < tail);
1023
- tail += n;
1024
- return *this;
1025
- }
1026
-
1027
- inline uint40_t& operator+=(std::size_t n) noexcept {
1028
- unsigned char* n_octets = reinterpret_cast<unsigned char*>(&n);
1029
- std::uint32_t& n_tail = *reinterpret_cast<std::uint32_t*>(n_octets);
1030
- std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
1031
- octets[4] += static_cast<unsigned char>((tail + n_tail) < tail);
1032
- tail += n_tail;
1033
- octets[4] += n_octets[4];
1034
- return *this;
1035
- }
1036
-
1037
- inline uint40_t operator+(std::size_t n) noexcept {
1038
- uint40_t other(*this);
1039
- other += n;
1040
- return other;
779
+ #if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
780
+ inline uint40_t(std::size_t n) noexcept {
781
+ #ifdef USEARCH_64BIT_ENV
782
+ std::memcpy(octets, &n, 5);
783
+ #else
784
+ std::memcpy(octets, &n, 4);
785
+ #endif
1041
786
  }
787
+ #endif
1042
788
 
1043
789
  inline operator std::size_t() const noexcept {
1044
790
  std::size_t result = 0;
1045
791
  #ifdef USEARCH_64BIT_ENV
1046
- std::memcpy((char*)&result + 3, octets, 5);
792
+ std::memcpy(&result, octets, 5);
1047
793
  #else
1048
- std::memcpy((char*)&result, octets + 1, 4);
794
+ std::memcpy(&result, octets + 1, 4);
1049
795
  #endif
1050
796
  return result;
1051
797
  }
1052
798
 
1053
- inline uint40_t& operator++() noexcept { return *this += 1u; }
1054
-
1055
- inline uint40_t operator++(int) noexcept {
1056
- uint40_t old = *this;
1057
- *this += 1u;
1058
- return old;
1059
- }
1060
-
1061
799
  inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
1062
800
  inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
1063
801
  };
@@ -1068,6 +806,9 @@ class usearch_pack_m uint40_t {
1068
806
 
1069
807
  static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
1070
808
 
809
+ /**
810
+ * @brief Basic single-threaded @b ring class, used for all kinds of task queues.
811
+ */
1071
812
  template <typename element_at, typename allocator_at = std::allocator<element_at>> //
1072
813
  class ring_gt {
1073
814
  public:
@@ -1202,12 +943,6 @@ constexpr std::size_t default_expansion_search() { return 64; }
1202
943
 
1203
944
  constexpr std::size_t default_allocator_entry_bytes() { return 64; }
1204
945
 
1205
- /**
1206
- * @brief The "magic" sequence helps infer the type of the file.
1207
- * USearch indexes start with the "usearch" string.
1208
- */
1209
- constexpr char const* default_magic() { return "usearch"; }
1210
-
1211
946
  /**
1212
947
  * @brief Configuration settings for the index construction.
1213
948
  * Includes the main `::connectivity` parameter (`M` in the paper)
@@ -1219,14 +954,16 @@ struct index_config_t {
1219
954
  /// > It is called `M` in the paper.
1220
955
  std::size_t connectivity = default_connectivity();
1221
956
 
1222
- /// @brief Parameter controlling the physical layout of vectors
1223
- /// in memory. When using multi-byte scalar types, like `float`,
1224
- /// dereferencing at mis-aligned locations may cause UB. So you
1225
- /// should set it to at least the size of expected floats.
1226
- /// Moreover, when using SIMD-accelerated metrics, you may prefer
1227
- /// to align to SIMD register size for higher performance with
1228
- /// less split-loads.
1229
- std::size_t vector_alignment = 1;
957
+ /// @brief Number of neighbors per graph node in base level graph.
958
+ /// Defaults to double of the other levels, so 64 in FAISS and 32 in hnswlib.
959
+ /// > It is called `M0` in the paper.
960
+ std::size_t connectivity_base = default_connectivity() * 2;
961
+
962
+ inline index_config_t() = default;
963
+ inline index_config_t(std::size_t c) noexcept
964
+ : connectivity(c ? c : default_connectivity()), connectivity_base(c ? c * 2 : default_connectivity() * 2) {}
965
+ inline index_config_t(std::size_t c, std::size_t cb) noexcept
966
+ : connectivity(c), connectivity_base((std::max)(c, cb)) {}
1230
967
  };
1231
968
 
1232
969
  struct index_limits_t {
@@ -1234,13 +971,13 @@ struct index_limits_t {
1234
971
  std::size_t threads_add = std::thread::hardware_concurrency();
1235
972
  std::size_t threads_search = std::thread::hardware_concurrency();
1236
973
 
1237
- index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
1238
- index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
1239
- std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
1240
- std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
974
+ inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
975
+ inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
976
+ inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
977
+ inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
1241
978
  };
1242
979
 
1243
- struct add_config_t {
980
+ struct index_update_config_t {
1244
981
  /// @brief Hyper-parameter controlling the quality of indexing.
1245
982
  /// Defaults to 40 in FAISS and 200 in hnswlib.
1246
983
  /// > It is called `efConstruction` in the paper.
@@ -1248,12 +985,9 @@ struct add_config_t {
1248
985
 
1249
986
  /// @brief Optional thread identifier for multi-threaded construction.
1250
987
  std::size_t thread = 0;
1251
-
1252
- /// @brief Don't copy the ::vector, if it's persisted elsewhere.
1253
- bool store_vector = true;
1254
988
  };
1255
989
 
1256
- struct search_config_t {
990
+ struct index_search_config_t {
1257
991
  /// @brief Hyper-parameter controlling the quality of search.
1258
992
  /// Defaults to 16 in FAISS and 10 in hnswlib.
1259
993
  /// > It is called `ef` in the paper.
@@ -1266,133 +1000,21 @@ struct search_config_t {
1266
1000
  bool exact = false;
1267
1001
  };
1268
1002
 
1269
- struct copy_config_t {
1270
- bool copy_vectors = true;
1271
- };
1003
+ struct index_copy_config_t {};
1272
1004
 
1273
- struct join_config_t {
1005
+ struct index_join_config_t {
1274
1006
  /// @brief Controls maximum number of proposals per man during stable marriage.
1275
1007
  std::size_t max_proposals = 0;
1008
+
1276
1009
  /// @brief Hyper-parameter controlling the quality of search.
1277
1010
  /// Defaults to 16 in FAISS and 10 in hnswlib.
1278
1011
  /// > It is called `ef` in the paper.
1279
1012
  std::size_t expansion = default_expansion_search();
1013
+
1280
1014
  /// @brief Brute-forces exhaustive search over all entries in the index.
1281
1015
  bool exact = false;
1282
1016
  };
1283
1017
 
1284
- using file_header_t = byte_t[64];
1285
-
1286
- /**
1287
- * @brief Serialized binary representations of the USearch index start with metadata.
1288
- * Metadata is parsed into a `file_head_t`, containing the USearch package version,
1289
- * and the properties of the index.
1290
- *
1291
- * It uses: 13 bytes for file versioning, 22 bytes for structural information = 35 bytes.
1292
- * The following 24 bytes contain binary size of the graph, of the vectors, and the checksum,
1293
- * leaving 5 bytes at the end vacant.
1294
- */
1295
- struct file_head_t {
1296
-
1297
- // Versioning: 7 + 2 * 3 = 13 bytes
1298
- using magic_t = char[7];
1299
- using version_major_t = std::uint16_t;
1300
- using version_minor_t = std::uint16_t;
1301
- using version_patch_t = std::uint16_t;
1302
-
1303
- // Structural: 1 * 6 + 8 * 2 = 22 bytes
1304
- using connectivity_t = std::uint8_t;
1305
- using max_level_t = std::uint8_t;
1306
- using vector_alignment_t = std::uint8_t;
1307
- using bytes_per_label_t = std::uint8_t;
1308
- using bytes_per_id_t = std::uint8_t;
1309
- using size_t = std::uint64_t;
1310
- using entry_idx_t = std::uint64_t;
1311
-
1312
- // Versioning:
1313
- char const* magic;
1314
- misaligned_ref_gt<version_major_t> version_major;
1315
- misaligned_ref_gt<version_minor_t> version_minor;
1316
- misaligned_ref_gt<version_patch_t> version_patch;
1317
-
1318
- // Structural:
1319
- misaligned_ref_gt<metric_kind_t> metric;
1320
- misaligned_ref_gt<connectivity_t> connectivity;
1321
- misaligned_ref_gt<max_level_t> max_level;
1322
- misaligned_ref_gt<vector_alignment_t> vector_alignment;
1323
- misaligned_ref_gt<bytes_per_label_t> bytes_per_label;
1324
- misaligned_ref_gt<bytes_per_id_t> bytes_per_id;
1325
- misaligned_ref_gt<scalar_kind_t> scalar_kind;
1326
- misaligned_ref_gt<size_t> size;
1327
- misaligned_ref_gt<entry_idx_t> entry_idx;
1328
-
1329
- // Additional:
1330
- misaligned_ref_gt<size_t> bytes_for_graphs;
1331
- misaligned_ref_gt<size_t> bytes_for_vectors;
1332
- misaligned_ref_gt<size_t> bytes_checksum;
1333
-
1334
- file_head_t(byte_t* ptr) noexcept
1335
- : magic((char const*)exchange(ptr, ptr + sizeof(magic_t))),
1336
- version_major(exchange(ptr, ptr + sizeof(version_major_t))),
1337
- version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
1338
- version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
1339
- metric(exchange(ptr, ptr + sizeof(metric_kind_t))), connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
1340
- max_level(exchange(ptr, ptr + sizeof(max_level_t))),
1341
- vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
1342
- bytes_per_label(exchange(ptr, ptr + sizeof(bytes_per_label_t))),
1343
- bytes_per_id(exchange(ptr, ptr + sizeof(bytes_per_id_t))),
1344
- scalar_kind(exchange(ptr, ptr + sizeof(scalar_kind_t))), size(exchange(ptr, ptr + sizeof(size_t))),
1345
- entry_idx(exchange(ptr, ptr + sizeof(entry_idx_t))), bytes_for_graphs(exchange(ptr, ptr + sizeof(size_t))),
1346
- bytes_for_vectors(exchange(ptr, ptr + sizeof(size_t))), bytes_checksum(exchange(ptr, ptr + sizeof(size_t))) {}
1347
- };
1348
-
1349
- struct file_head_result_t {
1350
-
1351
- using magic_t = file_head_t::magic_t;
1352
- using version_major_t = file_head_t::version_major_t;
1353
- using version_minor_t = file_head_t::version_minor_t;
1354
- using version_patch_t = file_head_t::version_patch_t;
1355
-
1356
- using connectivity_t = file_head_t::connectivity_t;
1357
- using max_level_t = file_head_t::max_level_t;
1358
- using vector_alignment_t = file_head_t::vector_alignment_t;
1359
- using bytes_per_label_t = file_head_t::bytes_per_label_t;
1360
- using bytes_per_id_t = file_head_t::bytes_per_id_t;
1361
- using size_t = file_head_t::size_t;
1362
- using entry_idx_t = file_head_t::entry_idx_t;
1363
-
1364
- // Versioning:
1365
- version_major_t version_major;
1366
- version_minor_t version_minor;
1367
- version_patch_t version_patch;
1368
-
1369
- // Structural:
1370
- metric_kind_t metric;
1371
- connectivity_t connectivity;
1372
- max_level_t max_level;
1373
- vector_alignment_t vector_alignment;
1374
- bytes_per_label_t bytes_per_label;
1375
- bytes_per_id_t bytes_per_id;
1376
- scalar_kind_t scalar_kind;
1377
- size_t size;
1378
- entry_idx_t entry_idx;
1379
-
1380
- // Additional:
1381
- size_t bytes_for_graphs;
1382
- size_t bytes_for_vectors;
1383
- size_t bytes_checksum;
1384
-
1385
- error_t error;
1386
-
1387
- explicit operator bool() const noexcept { return !error; }
1388
- file_head_result_t failed(error_t message) noexcept {
1389
- error = std::move(message);
1390
- return std::move(*this);
1391
- }
1392
- };
1393
-
1394
- static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 bytes");
1395
-
1396
1018
  /// @brief C++17 and newer version deprecate the `std::result_of`
1397
1019
  template <typename metric_at, typename... args_at>
1398
1020
  using return_type_gt =
@@ -1402,32 +1024,65 @@ using return_type_gt =
1402
1024
  typename std::result_of<metric_at(args_at...)>::type;
1403
1025
  #endif
1404
1026
 
1405
- /// @brief OS-specific to wrap open file-descriptors/handles.
1406
- #if defined(USEARCH_DEFINED_WINDOWS)
1407
- struct viewed_file_t {
1408
- HANDLE file_handle{};
1409
- HANDLE mapping_handle{};
1410
- void* ptr{};
1411
- size_t length{};
1412
- explicit operator bool() const noexcept { return mapping_handle != nullptr; }
1413
- };
1414
- #else
1415
- struct viewed_file_t {
1416
- int file_descriptor{};
1417
- void* ptr{};
1418
- size_t length{};
1419
- explicit operator bool() const noexcept { return file_descriptor != 0; }
1027
+ /**
1028
+ * @brief An example of what a USearch-compatible ad-hoc filter would look like.
1029
+ *
1030
+ * A similar function object can be passed to search queries to further filter entries
1031
+ * on their auxiliary properties, such as some categorical labels stored in an external DBMS.
1032
+ */
1033
+ struct dummy_predicate_t {
1034
+ template <typename member_at> constexpr bool operator()(member_at&&) const noexcept { return true; }
1420
1035
  };
1421
- #endif
1422
1036
 
1423
- struct dummy_predicate_t {
1424
- template <typename match_at> constexpr bool operator()(match_at&&) const noexcept { return true; }
1037
+ /**
1038
+ * @brief An example of what a USearch-compatible ad-hoc operation on in-flight entries.
1039
+ *
1040
+ * This kind of callbacks is used when the engine is being updated and you want to patch
1041
+ * the entries, while their are still under locks - limiting concurrent access and providing
1042
+ * consistency.
1043
+ */
1044
+ struct dummy_callback_t {
1045
+ template <typename member_at> void operator()(member_at&&) const noexcept {}
1425
1046
  };
1426
1047
 
1048
+ /**
1049
+ * @brief An example of what a USearch-compatible progress-bar should look like.
1050
+ *
1051
+ * This is particularly helpful when handling long-running tasks, like serialization,
1052
+ * saving, and loading from disk, or index-level joins.
1053
+ */
1427
1054
  struct dummy_progress_t {
1428
1055
  inline void operator()(std::size_t /*progress*/, std::size_t /*total*/) const noexcept {}
1429
1056
  };
1430
1057
 
1058
+ /**
1059
+ * @brief An example of what a USearch-compatible values prefetching mechanism should look like.
1060
+ *
1061
+ * USearch is designed to handle very large datasets, that may not fir into RAM. Fetching from
1062
+ * external memory is very expensive, so we've added a pre-fetching mechanism, that accepts
1063
+ * multiple objects at once, to cache in RAM ahead of the computation.
1064
+ * The received iterators support both `get_slot` and `get_key` operations.
1065
+ * An example usage may look like this:
1066
+ *
1067
+ * template <typename member_citerator_like_at>
1068
+ * inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {
1069
+ * for (; begin != end; ++begin)
1070
+ * io_uring_prefetch(offset_in_file(get_key(begin)));
1071
+ * }
1072
+ */
1073
+ struct dummy_prefetch_t {
1074
+ template <typename member_citerator_like_at>
1075
+ inline void operator()(member_citerator_like_at, member_citerator_like_at) const noexcept {}
1076
+ };
1077
+
1078
+ /**
1079
+ * @brief An example of what a USearch-compatible executor (thread-pool) should look like.
1080
+ *
1081
+ * It's expected to have `execute_bulk(callback)` API to schedule one task per thread;
1082
+ * an identical `execute_bulk(count, callback)` overload that also accepts the number
1083
+ * of tasks, and somehow schedules them between threads; as well as `size()` to
1084
+ * determine the number of available threads.
1085
+ */
1431
1086
  struct dummy_executor_t {
1432
1087
  dummy_executor_t() noexcept {}
1433
1088
  std::size_t size() const noexcept { return 1; }
@@ -1444,13 +1099,34 @@ struct dummy_executor_t {
1444
1099
  }
1445
1100
  };
1446
1101
 
1102
+ /**
1103
+ * @brief An example of what a USearch-compatible label-to-label mapping should look like.
1104
+ *
1105
+ * This is particularly helpful for "Semantic Joins", where we map entries of one collection
1106
+ * to entries of another. In assymetric setups, where A -> B is needed, but B -> A is not,
1107
+ * this can be passed to minimize memory usage.
1108
+ */
1447
1109
  struct dummy_label_to_label_mapping_t {
1448
1110
  struct member_ref_t {
1449
- template <typename label_at> member_ref_t& operator=(label_at&&) noexcept { return *this; }
1111
+ template <typename key_at> member_ref_t& operator=(key_at&&) noexcept { return *this; }
1450
1112
  };
1451
- template <typename label_at> member_ref_t operator[](label_at&&) const noexcept { return {}; }
1113
+ template <typename key_at> member_ref_t operator[](key_at&&) const noexcept { return {}; }
1452
1114
  };
1453
1115
 
1116
+ /**
1117
+ * @brief Checks if the provided object has a dummy type, emulating an interface,
1118
+ * but performaing no real computation.
1119
+ */
1120
+ template <typename object_at> static constexpr bool is_dummy() {
1121
+ using object_t = typename std::remove_all_extents<object_at>::type;
1122
+ return std::is_same<object_t, dummy_predicate_t>::value || //
1123
+ std::is_same<object_t, dummy_callback_t>::value || //
1124
+ std::is_same<object_t, dummy_progress_t>::value || //
1125
+ std::is_same<object_t, dummy_prefetch_t>::value || //
1126
+ std::is_same<object_t, dummy_executor_t>::value || //
1127
+ std::is_same<object_t, dummy_label_to_label_mapping_t>::value;
1128
+ }
1129
+
1454
1130
  template <typename, typename at> struct has_reset_gt {
1455
1131
  static_assert(std::integral_constant<at, false>::value, "Second template parameter needs to be of function type.");
1456
1132
  };
@@ -1474,116 +1150,388 @@ struct has_reset_gt<check_at, return_at(args_at...)> {
1474
1150
  */
1475
1151
  template <typename at> constexpr bool has_reset() { return has_reset_gt<at, void()>::value; }
1476
1152
 
1153
+ struct serialization_result_t {
1154
+ error_t error;
1155
+
1156
+ explicit operator bool() const noexcept { return !error; }
1157
+ serialization_result_t failed(error_t message) noexcept {
1158
+ error = std::move(message);
1159
+ return std::move(*this);
1160
+ }
1161
+ };
1162
+
1477
1163
  /**
1478
- * @brief Approximate Nearest Neighbors Search index using the
1479
- * Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
1480
- *
1481
- * @section Features
1482
- * - Thread-safe for concurrent construction.
1483
- * - Doesn't allocate new threads, and reuses the ones its called from.
1484
- * - Search for vectors of different dimensionality, if ::`metric_at` supports that.
1485
- *
1486
- * @tparam metric_at
1487
- * A function responsible for computing the distance (dis-similarity) between two vectors.
1488
- * Can have following signatures:
1489
- * - `distance_t (*) (span_gt<scalar_t const>, span_gt<scalar_t const>)`
1490
- * - `distance_t (*) (scalar_t const *, scalar_t const *)`
1491
- *
1492
- * @tparam label_at
1493
- * The type of unique labels to assign to vectors.
1494
- *
1495
- * @tparam id_at
1496
- * The smallest unsigned integer type to address indexed elements.
1497
- * Can be a built-in `uint32_t`, `uint64_t`, or our custom `uint40_t`.
1498
- *
1499
- * @tparam dynamic_allocator_at
1500
- * Dynamic memory allocator for temporary buffers, visits indicators, and
1501
- * priority queues, needed during construction and traversals of graphs.
1502
- * The allocated buffers may be uninitialized.
1503
- *
1504
- * @tparam tape_allocator_at
1505
- * Potentially different memory allocator for primary allocations of nodes and vectors.
1506
- * It would never `deallocate` separate entries, and would only free all the space at once.
1507
- * The allocated buffers may be uninitialized.
1508
- *
1509
- * @section Usage
1510
- *
1511
- * @subsection Exceptions
1512
- *
1513
- * None of the methods throw exceptions in the "Release" compilation mode.
1514
- * It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
1515
- * safe to copy.
1516
- *
1517
- * @section Implementation Details
1518
- *
1519
- * Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
1520
- * Every added vector forms a node in one or more levels of the graph.
1521
- * Every node is present in the base level. Every following level contains a smaller
1522
- * fraction of nodes. During search, the operation starts with the smaller levels
1523
- * and zooms-in on every following iteration of larger graph traversals.
1524
- *
1525
- * Just one memory allocation is performed regardless of the number of levels.
1526
- * The adjacency lists across all levels are concatenated into that single buffer.
1527
- * That buffer starts with a "head", that stores the metadata, such as the
1528
- * tallest "level" of the graph that it belongs to, the external "label", and the
1529
- * number of "dimensions" in the vector.
1530
- *
1531
- * @subsection Colocated Nodes
1532
- *
1533
- * Every node contains some graph-related data, as well as the original vector.
1534
- * Assuming the vectors can get large, we allow separating the index from the vectors.
1535
- * They can either be stored in a single continuous block of memory, or in two separate
1536
- * locations. Vector storage can also be detached from `index_gt` and managed by the user.
1537
- * To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
1538
- * overloads.
1539
- *
1540
- * @subsection Missing Functionality
1541
- *
1542
- * To simplify the implementation, the `index_gt` lacks endpoints to remove existing
1543
- * vectors. That, however, is solved by `index_punned_dense_gt`, which also adds automatic casting.
1164
+ * @brief Smart-pointer wrapping the LibC @b `FILE` for binary file @b outputs.
1544
1165
  *
1166
+ * This class raises no exceptions and corresponds errors through `serialization_result_t`.
1167
+ * The class automatically closes the file when the object is destroyed.
1545
1168
  */
1546
- template <typename metric_at = ip_gt<float>, //
1547
- typename label_at = std::int64_t, //
1548
- typename id_at = std::uint32_t, //
1549
- typename dynamic_allocator_at = std::allocator<char>, //
1550
- typename tape_allocator_at = dynamic_allocator_at> //
1551
- class index_gt {
1552
- public:
1553
- using metric_t = metric_at;
1554
- using label_t = label_at;
1555
- using id_t = id_at;
1556
- using dynamic_allocator_t = dynamic_allocator_at;
1557
- using tape_allocator_t = tape_allocator_at;
1558
- static_assert(sizeof(label_t) >= sizeof(id_t), "Having tiny labels doesn't make sense.");
1169
+ class output_file_t {
1170
+ char const* path_ = nullptr;
1171
+ std::FILE* file_ = nullptr;
1559
1172
 
1560
- using scalar_t = typename metric_t::scalar_t;
1561
- using vector_view_t = span_gt<scalar_t const>;
1562
- using distance_t = return_type_gt<metric_t, vector_view_t, vector_view_t>;
1173
+ public:
1174
+ output_file_t(char const* path) noexcept : path_(path) {}
1175
+ ~output_file_t() noexcept { close(); }
1176
+ output_file_t(output_file_t&& other) noexcept
1177
+ : path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
1178
+ output_file_t& operator=(output_file_t&& other) noexcept {
1179
+ std::swap(path_, other.path_);
1180
+ std::swap(file_, other.file_);
1181
+ return *this;
1182
+ }
1183
+ serialization_result_t open_if_not() noexcept {
1184
+ serialization_result_t result;
1185
+ if (!file_)
1186
+ file_ = std::fopen(path_, "wb");
1187
+ if (!file_)
1188
+ return result.failed(std::strerror(errno));
1189
+ return result;
1190
+ }
1191
+ serialization_result_t write(void* begin, std::size_t length) noexcept {
1192
+ serialization_result_t result;
1193
+ std::size_t written = std::fwrite(begin, length, 1, file_);
1194
+ if (!written)
1195
+ result.failed(std::strerror(errno));
1196
+ return result;
1197
+ }
1198
+ void close() noexcept {
1199
+ if (file_)
1200
+ std::fclose(exchange(file_, nullptr));
1201
+ }
1202
+ };
1563
1203
 
1564
- struct member_cref_t {
1565
- label_t label;
1566
- vector_view_t vector;
1567
- id_t id;
1568
- };
1204
+ /**
1205
+ * @brief Smart-pointer wrapping the LibC @b `FILE` for binary files @b inputs.
1206
+ *
1207
+ * This class raises no exceptions and corresponds errors through `serialization_result_t`.
1208
+ * The class automatically closes the file when the object is destroyed.
1209
+ */
1210
+ class input_file_t {
1211
+ char const* path_ = nullptr;
1212
+ std::FILE* file_ = nullptr;
1569
1213
 
1570
- struct member_ref_t {
1571
- misaligned_ref_gt<label_t> label;
1572
- vector_view_t vector;
1573
- id_t id;
1214
+ public:
1215
+ input_file_t(char const* path) noexcept : path_(path) {}
1216
+ ~input_file_t() noexcept { close(); }
1217
+ input_file_t(input_file_t&& other) noexcept
1218
+ : path_(exchange(other.path_, nullptr)), file_(exchange(other.file_, nullptr)) {}
1219
+ input_file_t& operator=(input_file_t&& other) noexcept {
1220
+ std::swap(path_, other.path_);
1221
+ std::swap(file_, other.file_);
1222
+ return *this;
1223
+ }
1574
1224
 
1575
- inline operator member_cref_t() const noexcept { return {label, vector, id}; }
1576
- };
1225
+ serialization_result_t open_if_not() noexcept {
1226
+ serialization_result_t result;
1227
+ if (!file_)
1228
+ file_ = std::fopen(path_, "rb");
1229
+ if (!file_)
1230
+ return result.failed(std::strerror(errno));
1231
+ return result;
1232
+ }
1233
+ serialization_result_t read(void* begin, std::size_t length) noexcept {
1234
+ serialization_result_t result;
1235
+ std::size_t read = std::fread(begin, length, 1, file_);
1236
+ if (!read)
1237
+ result.failed(std::feof(file_) ? "End of file reached!" : std::strerror(errno));
1238
+ return result;
1239
+ }
1240
+ void close() noexcept {
1241
+ if (file_)
1242
+ std::fclose(exchange(file_, nullptr));
1243
+ }
1244
+
1245
+ explicit operator bool() const noexcept { return file_; }
1246
+ bool seek_to(std::size_t progress) noexcept { return std::fseek(file_, progress, SEEK_SET) == 0; }
1247
+ bool seek_to_end() noexcept { return std::fseek(file_, 0L, SEEK_END) == 0; }
1248
+ bool infer_progress(std::size_t& progress) noexcept {
1249
+ long int result = std::ftell(file_);
1250
+ if (result == -1L)
1251
+ return false;
1252
+ progress = static_cast<std::size_t>(result);
1253
+ return true;
1254
+ }
1255
+ };
1256
+
1257
+ /**
1258
+ * @brief Represents a memory-mapped file or a pre-allocated anonymous memory region.
1259
+ *
1260
+ * This class provides a convenient way to memory-map a file and access its contents as a block of
1261
+ * memory. The class handles platform-specific memory-mapping operations on Windows, Linux, and MacOS.
1262
+ * The class automatically closes the file when the object is destroyed.
1263
+ */
1264
+ class memory_mapped_file_t {
1265
+ char const* path_{}; /**< The path to the file to be memory-mapped. */
1266
+ void* ptr_{}; /**< A pointer to the memory-mapping. */
1267
+ size_t length_{}; /**< The length of the memory-mapped file in bytes. */
1268
+
1269
+ #if defined(USEARCH_DEFINED_WINDOWS)
1270
+ HANDLE file_handle_{}; /**< The file handle on Windows. */
1271
+ HANDLE mapping_handle_{}; /**< The mapping handle on Windows. */
1272
+ #else
1273
+ int file_descriptor_{}; /**< The file descriptor on Linux and MacOS. */
1274
+ #endif
1275
+
1276
+ public:
1277
+ explicit operator bool() const noexcept { return ptr_ != nullptr; }
1278
+ byte_t const* data() const noexcept { return reinterpret_cast<byte_t const*>(ptr_); }
1279
+ std::size_t size() const noexcept { return static_cast<std::size_t>(length_); }
1280
+
1281
+ memory_mapped_file_t() noexcept {}
1282
+ memory_mapped_file_t(char const* path) noexcept : path_(path) {}
1283
+ ~memory_mapped_file_t() noexcept { close(); }
1284
+ memory_mapped_file_t(memory_mapped_file_t&& other) noexcept
1285
+ : path_(exchange(other.path_, nullptr)), ptr_(exchange(other.ptr_, nullptr)),
1286
+ length_(exchange(other.length_, 0)),
1287
+ #if defined(USEARCH_DEFINED_WINDOWS)
1288
+ file_handle_(exchange(other.file_handle_, nullptr)), mapping_handle_(exchange(other.mapping_handle_, nullptr))
1289
+ #else
1290
+ file_descriptor_(exchange(other.file_descriptor_, 0))
1291
+ #endif
1292
+ {
1293
+ }
1294
+
1295
+ memory_mapped_file_t(byte_t* data, std::size_t length) noexcept : ptr_(data), length_(length) {}
1296
+
1297
+ memory_mapped_file_t& operator=(memory_mapped_file_t&& other) noexcept {
1298
+ std::swap(path_, other.path_);
1299
+ std::swap(ptr_, other.ptr_);
1300
+ std::swap(length_, other.length_);
1301
+ #if defined(USEARCH_DEFINED_WINDOWS)
1302
+ std::swap(file_handle_, other.file_handle_);
1303
+ std::swap(mapping_handle_, other.mapping_handle_);
1304
+ #else
1305
+ std::swap(file_descriptor_, other.file_descriptor_);
1306
+ #endif
1307
+ return *this;
1308
+ }
1309
+
1310
+ serialization_result_t open_if_not() noexcept {
1311
+ serialization_result_t result;
1312
+ if (!path_ || ptr_)
1313
+ return result;
1314
+
1315
+ #if defined(USEARCH_DEFINED_WINDOWS)
1316
+
1317
+ HANDLE file_handle =
1318
+ CreateFile(path_, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
1319
+ if (file_handle == INVALID_HANDLE_VALUE)
1320
+ return result.failed("Opening file failed!");
1321
+
1322
+ std::size_t file_length = GetFileSize(file_handle, 0);
1323
+ HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
1324
+ if (mapping_handle == 0) {
1325
+ CloseHandle(file_handle);
1326
+ return result.failed("Mapping file failed!");
1327
+ }
1328
+
1329
+ byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
1330
+ if (file == 0) {
1331
+ CloseHandle(mapping_handle);
1332
+ CloseHandle(file_handle);
1333
+ return result.failed("View the map failed!");
1334
+ }
1335
+ file_handle_ = file_handle;
1336
+ mapping_handle_ = mapping_handle;
1337
+ ptr_ = file;
1338
+ length_ = file_length;
1339
+ #else
1340
+
1341
+ #if defined(USEARCH_DEFINED_LINUX)
1342
+ int descriptor = open(path_, O_RDONLY | O_NOATIME);
1343
+ #else
1344
+ int descriptor = open(path_, O_RDONLY);
1345
+ #endif
1346
+
1347
+ // Estimate the file size
1348
+ struct stat file_stat;
1349
+ int fstat_status = fstat(descriptor, &file_stat);
1350
+ if (fstat_status < 0) {
1351
+ ::close(descriptor);
1352
+ return result.failed(std::strerror(errno));
1353
+ }
1354
+
1355
+ // Map the entire file
1356
+ byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_SHARED, descriptor, 0);
1357
+ if (file == MAP_FAILED) {
1358
+ ::close(descriptor);
1359
+ return result.failed(std::strerror(errno));
1360
+ }
1361
+ file_descriptor_ = descriptor;
1362
+ ptr_ = file;
1363
+ length_ = file_stat.st_size;
1364
+ #endif // Platform specific code
1365
+ return result;
1366
+ }
1367
+
1368
+ void close() noexcept {
1369
+ if (!path_) {
1370
+ ptr_ = nullptr;
1371
+ length_ = 0;
1372
+ return;
1373
+ }
1374
+ #if defined(USEARCH_DEFINED_WINDOWS)
1375
+ UnmapViewOfFile(ptr_);
1376
+ CloseHandle(mapping_handle_);
1377
+ CloseHandle(file_handle_);
1378
+ mapping_handle_ = nullptr;
1379
+ file_handle_ = nullptr;
1380
+ #else
1381
+ munmap(ptr_, length_);
1382
+ ::close(file_descriptor_);
1383
+ file_descriptor_ = 0;
1384
+ #endif
1385
+ ptr_ = nullptr;
1386
+ length_ = 0;
1387
+ }
1388
+ };
1389
+
1390
+ struct index_serialized_header_t {
1391
+ std::uint64_t size = 0;
1392
+ std::uint64_t connectivity = 0;
1393
+ std::uint64_t connectivity_base = 0;
1394
+ std::uint64_t max_level = 0;
1395
+ std::uint64_t entry_slot = 0;
1396
+ };
1397
+
1398
+ using default_key_t = std::uint64_t;
1399
+ using default_slot_t = std::uint32_t;
1400
+ using default_distance_t = float;
1401
+
1402
+ template <typename key_at = default_key_t> struct member_gt {
1403
+ key_at key;
1404
+ std::size_t slot;
1405
+ };
1406
+
1407
+ template <typename key_at> inline std::size_t get_slot(member_gt<key_at> const& m) noexcept { return m.slot; }
1408
+ template <typename key_at> inline key_at get_key(member_gt<key_at> const& m) noexcept { return m.key; }
1409
+
1410
+ template <typename key_at = default_key_t> struct member_cref_gt {
1411
+ misaligned_ref_gt<key_at const> key;
1412
+ std::size_t slot;
1413
+ };
1414
+
1415
+ template <typename key_at> inline std::size_t get_slot(member_cref_gt<key_at> const& m) noexcept { return m.slot; }
1416
+ template <typename key_at> inline key_at get_key(member_cref_gt<key_at> const& m) noexcept { return m.key; }
1417
+
1418
+ template <typename key_at = default_key_t> struct member_ref_gt {
1419
+ misaligned_ref_gt<key_at> key;
1420
+ std::size_t slot;
1421
+
1422
+ inline operator member_cref_gt<key_at>() const noexcept { return {key.ptr(), slot}; }
1423
+ };
1424
+
1425
+ template <typename key_at> inline std::size_t get_slot(member_ref_gt<key_at> const& m) noexcept { return m.slot; }
1426
+ template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m) noexcept { return m.key; }
1427
+
1428
+ /**
1429
+ * @brief Approximate Nearest Neighbors Search @b index-structure using the
1430
+ * Hierarchical Navigable Small World @b (HNSW) graphs algorithm.
1431
+ * If classical containers store @b Key->Value mappings, this one can
1432
+ * be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
1433
+ *
1434
+ * Unlike most implementations, this one is generic anc can be used for any search,
1435
+ * not just within equi-dimensional vectors. Examples range from texts to similar Chess
1436
+ * positions.
1437
+ *
1438
+ * @tparam key_at
1439
+ * The type of primary objects stored in the index.
1440
+ * The values, to which those map, are not managed by the same index structure.
1441
+ *
1442
+ * @tparam compressed_slot_at
1443
+ * The smallest unsigned integer type to address indexed elements.
1444
+ * It is used internally to maximize space-efficiency and is generally
1445
+ * up-casted to @b `std::size_t` in public interfaces.
1446
+ * Can be a built-in @b `uint32_t`, `uint64_t`, or our custom @b `uint40_t`.
1447
+ * Which makes the most sense for 4B+ entry indexes.
1448
+ *
1449
+ * @tparam dynamic_allocator_at
1450
+ * Dynamic memory allocator for temporary buffers, visits indicators, and
1451
+ * priority queues, needed during construction and traversals of graphs.
1452
+ * The allocated buffers may be uninitialized.
1453
+ *
1454
+ * @tparam tape_allocator_at
1455
+ * Potentially different memory allocator for primary allocations of nodes and vectors.
1456
+ * It would never `deallocate` separate entries, and would only free all the space at once.
1457
+ * The allocated buffers may be uninitialized.
1458
+ *
1459
+ * @section Features
1460
+ *
1461
+ * - Thread-safe for concurrent construction, search, and updates.
1462
+ * - Doesn't allocate new threads, and reuses the ones its called from.
1463
+ * - Allows storing value externally, managing just the similarity index.
1464
+ * - Joins.
1465
+
1466
+ * @section Usage
1467
+ *
1468
+ * @subsection Exceptions
1469
+ *
1470
+ * None of the methods throw exceptions in the "Release" compilation mode.
1471
+ * It may only `throw` if your memory ::dynamic_allocator_at or ::metric_at isn't
1472
+ * safe to copy.
1473
+ *
1474
+ * @subsection Serialization
1475
+ *
1476
+ * When serialized, doesn't include any additional metadata.
1477
+ * It is just the multi-level proximity-graph. You may want to store metadata about
1478
+ * the used metric and key types somewhere else.
1479
+ *
1480
+ * @section Implementation Details
1481
+ *
1482
+ * Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
1483
+ * Every added vector forms a node in one or more levels of the graph.
1484
+ * Every node is present in the base level. Every following level contains a smaller
1485
+ * fraction of nodes. During search, the operation starts with the smaller levels
1486
+ * and zooms-in on every following iteration of larger graph traversals.
1487
+ *
1488
+ * Just one memory allocation is performed regardless of the number of levels.
1489
+ * The adjacency lists across all levels are concatenated into that single buffer.
1490
+ * That buffer starts with a "head", that stores the metadata, such as the
1491
+ * tallest "level" of the graph that it belongs to, the external "key", and the
1492
+ * number of "dimensions" in the vector.
1493
+ *
1494
+ * @section Metrics, Predicates and Callbacks
1495
+ *
1496
+ *
1497
+ * @section Smart References and Iterators
1498
+ *
1499
+ * - `member_citerator_t` and `member_iterator_t` have only slots, no indirections.
1500
+ *
1501
+ * - `member_cref_t` and `member_ref_t` contains the `slot` and a reference
1502
+ * to the key. So it passes through 1 level of visited_members in `nodes_`.
1503
+ * Retrieving the key via `get_key` will cause fetching yet another cache line.
1504
+ *
1505
+ * - `member_gt` contains an already prefetched copy of the key.
1506
+ *
1507
+ */
1508
+ template <typename distance_at = default_distance_t, //
1509
+ typename key_at = default_key_t, //
1510
+ typename compressed_slot_at = default_slot_t, //
1511
+ typename dynamic_allocator_at = std::allocator<byte_t>, //
1512
+ typename tape_allocator_at = dynamic_allocator_at> //
1513
+ class index_gt {
1514
+ public:
1515
+ using distance_t = distance_at;
1516
+ using key_t = key_at;
1517
+ using compressed_slot_t = compressed_slot_at;
1518
+ using dynamic_allocator_t = dynamic_allocator_at;
1519
+ using tape_allocator_t = tape_allocator_at;
1520
+ static_assert(sizeof(key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
1521
+
1522
+ using member_cref_t = member_cref_gt<key_t>;
1523
+ using member_ref_t = member_ref_gt<key_t>;
1577
1524
 
1578
1525
  template <typename ref_at, typename index_at> class member_iterator_gt {
1579
1526
  using ref_t = ref_at;
1580
1527
  using index_t = index_at;
1581
- index_t* index_{};
1582
- std::size_t offset_{};
1583
1528
 
1584
1529
  friend class index_gt;
1585
1530
  member_iterator_gt() noexcept {}
1586
- member_iterator_gt(index_t* index, std::size_t offset) noexcept : index_(index), offset_(offset) {}
1531
+ member_iterator_gt(index_t* index, std::size_t slot) noexcept : index_(index), slot_(slot) {}
1532
+
1533
+ index_t* index_{};
1534
+ std::size_t slot_{};
1587
1535
 
1588
1536
  public:
1589
1537
  using iterator_category = std::random_access_iterator_tag;
@@ -1592,45 +1540,32 @@ class index_gt {
1592
1540
  using pointer = void;
1593
1541
  using reference = ref_t;
1594
1542
 
1595
- reference operator*() const noexcept {
1596
- node_t node = index_->node_with_id_(offset_);
1597
- return {node.label(), node.vector_view(), static_cast<id_t>(offset_)};
1598
- }
1599
-
1600
- member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, offset_ + 1); }
1601
- member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, offset_ - 1); }
1602
- member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, offset_ + d); }
1603
- member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, offset_ - d); }
1604
-
1605
- member_iterator_gt& operator++() noexcept {
1606
- offset_ += 1;
1607
- return *this;
1608
- }
1609
- member_iterator_gt& operator--() noexcept {
1610
- offset_ -= 1;
1611
- return *this;
1612
- }
1613
- member_iterator_gt& operator+=(difference_type d) noexcept {
1614
- offset_ += d;
1615
- return *this;
1616
- }
1617
- member_iterator_gt& operator-=(difference_type d) noexcept {
1618
- offset_ -= d;
1619
- return *this;
1620
- }
1621
- bool operator==(member_iterator_gt const& other) const noexcept {
1622
- return index_ == other.index_ && offset_ == other.offset_;
1623
- }
1624
- bool operator!=(member_iterator_gt const& other) const noexcept {
1625
- return index_ != other.index_ || offset_ != other.offset_;
1626
- }
1543
+ reference operator*() const noexcept { return {index_->node_at_(slot_).key(), slot_}; }
1544
+ key_t key() const noexcept { return index_->node_at_(slot_).key(); }
1545
+
1546
+ friend inline std::size_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
1547
+ friend inline key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
1548
+
1549
+ member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
1550
+ member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
1551
+ member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
1552
+ member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
1553
+
1554
+ // clang-format off
1555
+ member_iterator_gt& operator++() noexcept { slot_ += 1; return *this; }
1556
+ member_iterator_gt& operator--() noexcept { slot_ -= 1; return *this; }
1557
+ member_iterator_gt& operator+=(difference_type d) noexcept { slot_ += d; return *this; }
1558
+ member_iterator_gt& operator-=(difference_type d) noexcept { slot_ -= d; return *this; }
1559
+ bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
1560
+ bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
1561
+ // clang-format on
1627
1562
  };
1628
1563
 
1629
1564
  using member_iterator_t = member_iterator_gt<member_ref_t, index_gt>;
1630
1565
  using member_citerator_t = member_iterator_gt<member_cref_t, index_gt const>;
1631
1566
 
1632
1567
  // STL compatibility:
1633
- using value_type = std::pair<label_t, distance_t>;
1568
+ using value_type = key_t;
1634
1569
  using allocator_type = dynamic_allocator_t;
1635
1570
  using size_type = std::size_t;
1636
1571
  using difference_type = std::ptrdiff_t;
@@ -1643,108 +1578,113 @@ class index_gt {
1643
1578
  using reverse_iterator = std::reverse_iterator<member_iterator_t>;
1644
1579
  using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
1645
1580
 
1646
- private:
1647
- using neighbors_count_t = std::uint32_t;
1648
- using dim_t = std::uint32_t;
1649
- using level_t = std::int32_t;
1650
-
1651
- using allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
1581
+ using dynamic_allocator_traits_t = std::allocator_traits<dynamic_allocator_t>;
1652
1582
  using byte_t = typename dynamic_allocator_t::value_type;
1653
- static_assert(sizeof(byte_t) == 1, //
1654
- "Primary allocator must allocate separate addressable bytes");
1583
+ static_assert( //
1584
+ sizeof(byte_t) == 1, //
1585
+ "Primary allocator must allocate separate addressable bytes");
1655
1586
 
1656
1587
  using tape_allocator_traits_t = std::allocator_traits<tape_allocator_t>;
1657
- static_assert(sizeof(typename tape_allocator_traits_t::value_type) == 1, //
1658
- "Tape allocator must allocate separate addressable bytes");
1588
+ static_assert( //
1589
+ sizeof(typename tape_allocator_traits_t::value_type) == 1, //
1590
+ "Tape allocator must allocate separate addressable bytes");
1659
1591
 
1592
+ private:
1660
1593
  /**
1661
- * @brief How much larger (number of neighbors per node) will
1662
- * the base level be compared to other levels.
1594
+ * @brief Integer for the number of node neighbors at a specific level of the
1595
+ * multi-level graph. It's selected to be `std::uint32_t` to improve the
1596
+ * alignment in most common cases.
1663
1597
  */
1664
- static constexpr std::size_t base_level_multiple_() { return 2; }
1598
+ using neighbors_count_t = std::uint32_t;
1599
+ using level_t = std::int16_t;
1665
1600
 
1666
1601
  /**
1667
1602
  * @brief How many bytes of memory are needed to form the "head" of the node.
1668
1603
  */
1669
- static constexpr std::size_t node_head_bytes_() { return sizeof(label_t) + sizeof(dim_t) + sizeof(level_t); }
1604
+ static constexpr std::size_t node_head_bytes_() { return sizeof(key_t) + sizeof(level_t); }
1670
1605
 
1671
1606
  using visits_bitset_t = visits_bitset_gt<dynamic_allocator_t>;
1672
1607
 
1673
1608
  struct precomputed_constants_t {
1674
1609
  double inverse_log_connectivity{};
1675
- std::size_t connectivity_max_base{};
1676
1610
  std::size_t neighbors_bytes{};
1677
1611
  std::size_t neighbors_base_bytes{};
1678
1612
  };
1613
+ /// @brief A space-efficient internal data-structure used in graph traversal queues.
1679
1614
  struct candidate_t {
1680
1615
  distance_t distance;
1681
- id_t id;
1682
- };
1683
- struct compare_by_distance_t {
1684
- inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
1616
+ compressed_slot_t slot;
1617
+ inline bool operator<(candidate_t other) const noexcept { return distance < other.distance; }
1685
1618
  };
1686
1619
 
1687
1620
  using candidates_view_t = span_gt<candidate_t const>;
1688
- using candidates_allocator_t = typename allocator_traits_t::template rebind_alloc<candidate_t>;
1689
- using top_candidates_t = sorted_buffer_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
1690
- using next_candidates_t = max_heap_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
1621
+ using candidates_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<candidate_t>;
1622
+ using top_candidates_t = sorted_buffer_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
1623
+ using next_candidates_t = max_heap_gt<candidate_t, std::less<candidate_t>, candidates_allocator_t>;
1691
1624
 
1692
1625
  /**
1693
- * @brief A loosely-structured handle for every node. One such node is created for every vector.
1626
+ * @brief A loosely-structured handle for every node. One such node is created for every member.
1694
1627
  * To minimize memory usage and maximize the number of entries per cache-line, it only
1695
- * stores to pointers. To reinterpret those into a structured `node_t` more
1696
- * information is needed from the parent `index_gt`.
1628
+ * stores to pointers. The internal tape starts with a `key_t` @b key, then
1629
+ * a `level_t` for the number of graph @b levels in which this member appears,
1630
+ * then the { `neighbors_count_t`, `compressed_slot_t`, `compressed_slot_t` ... } sequences
1631
+ * for @b each-level.
1697
1632
  */
1698
1633
  class node_t {
1699
1634
  byte_t* tape_{};
1700
- scalar_t* vector_{};
1701
1635
 
1702
1636
  public:
1703
- explicit node_t(byte_t* tape, scalar_t* vector) noexcept : tape_(tape), vector_(vector) {}
1637
+ explicit node_t(byte_t* tape) noexcept : tape_(tape) {}
1704
1638
  byte_t* tape() const noexcept { return tape_; }
1705
- scalar_t* vector() const noexcept { return vector_; }
1706
1639
  byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
1640
+ explicit operator bool() const noexcept { return tape_; }
1707
1641
 
1708
1642
  node_t() = default;
1709
1643
  node_t(node_t const&) = default;
1710
1644
  node_t& operator=(node_t const&) = default;
1711
1645
 
1712
- misaligned_ref_gt<label_t> label() const noexcept { return {tape_}; }
1713
- misaligned_ref_gt<dim_t> dim() const noexcept { return {tape_ + sizeof(label_t)}; }
1714
- misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(label_t) + sizeof(dim_t)}; }
1646
+ misaligned_ref_gt<key_t const> ckey() const noexcept { return {tape_}; }
1647
+ misaligned_ref_gt<key_t> key() const noexcept { return {tape_}; }
1648
+ misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(key_t)}; }
1715
1649
 
1716
- void label(label_t v) noexcept { return misaligned_store<label_t>(tape_, v); }
1717
- void dim(dim_t v) noexcept { return misaligned_store<dim_t>(tape_ + sizeof(label_t), v); }
1718
- void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
1719
-
1720
- operator vector_view_t() const noexcept { return {vector(), dim()}; }
1721
- vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
1722
- explicit operator bool() const noexcept { return tape_; }
1650
+ void key(key_t v) noexcept { return misaligned_store<key_t>(tape_, v); }
1651
+ void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(key_t), v); }
1723
1652
  };
1724
1653
 
1654
+ static_assert(std::is_trivially_copy_constructible<node_t>::value, "Nodes must be light!");
1655
+ static_assert(std::is_trivially_destructible<node_t>::value, "Nodes must be light!");
1656
+
1725
1657
  /**
1726
1658
  * @brief A slice of the node's tape, containing a the list of neighbors
1727
1659
  * for a node in a single graph level. It's pre-allocated to fit
1728
- * as many neighbors IDs, as may be needed at the target level,
1660
+ * as many neighbors "slots", as may be needed at the target level,
1729
1661
  * and starts with a single integer `neighbors_count_t` counter.
1730
1662
  */
1731
1663
  class neighbors_ref_t {
1732
1664
  byte_t* tape_;
1733
1665
 
1734
- static constexpr std::size_t shift(std::size_t i = 0) { return sizeof(neighbors_count_t) + sizeof(id_t) * i; }
1666
+ static constexpr std::size_t shift(std::size_t i = 0) {
1667
+ return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
1668
+ }
1735
1669
 
1736
1670
  public:
1737
1671
  neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
1738
- misaligned_ptr_gt<id_t> begin() noexcept { return tape_ + shift(); }
1739
- misaligned_ptr_gt<id_t> end() noexcept { return begin() + size(); }
1740
- misaligned_ptr_gt<id_t const> begin() const noexcept { return tape_ + shift(); }
1741
- misaligned_ptr_gt<id_t const> end() const noexcept { return begin() + size(); }
1742
- id_t operator[](std::size_t i) const noexcept { return misaligned_load<id_t>(tape_ + shift(i)); }
1672
+ misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
1673
+ misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
1674
+ misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
1675
+ misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
1676
+ compressed_slot_t operator[](std::size_t i) const noexcept {
1677
+ return misaligned_load<compressed_slot_t>(tape_ + shift(i));
1678
+ }
1743
1679
  std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
1744
- void clear() noexcept { misaligned_store<neighbors_count_t>(tape_, 0); }
1745
- void push_back(id_t id) noexcept {
1680
+ void clear() noexcept {
1681
+ neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1682
+ std::memset(tape_, 0, shift(n));
1683
+ // misaligned_store<neighbors_count_t>(tape_, 0);
1684
+ }
1685
+ void push_back(compressed_slot_t slot) noexcept {
1746
1686
  neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1747
- misaligned_store<id_t>(tape_ + shift(n), id);
1687
+ misaligned_store<compressed_slot_t>(tape_ + shift(n), slot);
1748
1688
  misaligned_store<neighbors_count_t>(tape_, n + 1);
1749
1689
  }
1750
1690
  };
@@ -1759,44 +1699,72 @@ class index_gt {
1759
1699
  next_candidates_t next_candidates{};
1760
1700
  visits_bitset_t visits{};
1761
1701
  std::default_random_engine level_generator{};
1762
- metric_t metric{};
1763
1702
  std::size_t iteration_cycles{};
1764
- std::size_t measurements_count{};
1703
+ std::size_t computed_distances_count{};
1704
+
1705
+ template <typename value_at, typename metric_at, typename entry_at> //
1706
+ inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
1707
+ static_assert( //
1708
+ std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
1709
+ "Unexpected type");
1710
+
1711
+ computed_distances_count++;
1712
+ return metric(first, second);
1713
+ }
1714
+
1715
+ template <typename metric_at, typename entry_at> //
1716
+ inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
1717
+ static_assert( //
1718
+ std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
1719
+ "Unexpected type");
1765
1720
 
1766
- inline distance_t measure(vector_view_t a, vector_view_t b) noexcept {
1767
- measurements_count++;
1768
- return metric(a, b);
1721
+ computed_distances_count++;
1722
+ return metric(first, second);
1769
1723
  }
1770
1724
  };
1771
1725
 
1772
1726
  index_config_t config_{};
1773
1727
  index_limits_t limits_{};
1774
- metric_t metric_{};
1728
+
1775
1729
  mutable dynamic_allocator_t dynamic_allocator_{};
1776
1730
  tape_allocator_t tape_allocator_{};
1731
+
1777
1732
  precomputed_constants_t pre_{};
1778
- viewed_file_t viewed_file_{};
1733
+ memory_mapped_file_t viewed_file_{};
1779
1734
 
1780
- usearch_align_m mutable std::atomic<std::size_t> capacity_{};
1781
- usearch_align_m mutable std::atomic<std::size_t> size_{};
1735
+ /// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
1736
+ usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
1782
1737
 
1783
- /// @brief Controls access to `max_level_` and `entry_id_`.
1738
+ /// @brief Number of "slots" already storing non-null nodes.
1739
+ usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
1740
+
1741
+ /// @brief Controls access to `max_level_` and `entry_slot_`.
1784
1742
  /// If any thread is updating those values, no other threads can `add()` or `search()`.
1785
1743
  std::mutex global_mutex_{};
1744
+
1745
+ /// @brief The level of the top-most graph in the index. Grows as the logarithm of size, starts from zero.
1786
1746
  level_t max_level_{};
1787
- id_t entry_id_{};
1788
1747
 
1789
- using nodes_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
1790
- node_t* nodes_{};
1748
+ /// @brief The slot in which the only node of the top-level graph is stored.
1749
+ std::size_t entry_slot_{};
1750
+
1751
+ using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
1752
+
1753
+ /// @brief C-style array of `node_t` smart-pointers.
1754
+ buffer_gt<node_t, nodes_allocator_t> nodes_{};
1755
+
1756
+ /// @brief Mutex, that limits concurrent access to `nodes_`.
1791
1757
  mutable visits_bitset_t nodes_mutexes_{};
1792
1758
 
1793
- using contexts_allocator_t = typename allocator_traits_t::template rebind_alloc<context_t>;
1794
- context_t* contexts_{};
1759
+ using contexts_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<context_t>;
1760
+
1761
+ /// @brief Array of thread-specific buffers for temporary data.
1762
+ mutable buffer_gt<context_t, contexts_allocator_t> contexts_{};
1795
1763
 
1796
1764
  public:
1797
1765
  std::size_t connectivity() const noexcept { return config_.connectivity; }
1798
- std::size_t capacity() const noexcept { return capacity_; }
1799
- std::size_t size() const noexcept { return size_; }
1766
+ std::size_t capacity() const noexcept { return nodes_capacity_; }
1767
+ std::size_t size() const noexcept { return nodes_count_; }
1800
1768
  std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
1801
1769
  index_config_t const& config() const noexcept { return config_; }
1802
1770
  index_limits_t const& limits() const noexcept { return limits_; }
@@ -1806,16 +1774,17 @@ class index_gt {
1806
1774
  * @section Exceptions
1807
1775
  * Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
1808
1776
  */
1809
- explicit index_gt(index_config_t config = {}, metric_t metric = {}, dynamic_allocator_t allocator = {},
1810
- tape_allocator_t tape_allocator = {}) noexcept
1811
- : config_(config), limits_(0, 0), metric_(metric), dynamic_allocator_(std::move(allocator)),
1812
- tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), size_(0u), max_level_(-1),
1813
- entry_id_(0u), nodes_(nullptr), nodes_mutexes_(), contexts_(nullptr) {}
1777
+ explicit index_gt( //
1778
+ index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
1779
+ tape_allocator_t tape_allocator = {}) noexcept
1780
+ : config_(config), limits_(0, 0), dynamic_allocator_(std::move(dynamic_allocator)),
1781
+ tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), nodes_count_(0u), max_level_(-1),
1782
+ entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
1814
1783
 
1815
1784
  /**
1816
1785
  * @brief Clones the structure with the same hyper-parameters, but without contents.
1817
1786
  */
1818
- index_gt fork() noexcept { return index_gt{config_, metric_, dynamic_allocator_}; }
1787
+ index_gt fork() noexcept { return index_gt{config_, dynamic_allocator_, tape_allocator_}; }
1819
1788
 
1820
1789
  ~index_gt() noexcept { reset(); }
1821
1790
 
@@ -1837,21 +1806,21 @@ class index_gt {
1837
1806
  }
1838
1807
  };
1839
1808
 
1840
- copy_result_t copy(copy_config_t /*config*/ = {}) const noexcept {
1809
+ copy_result_t copy(index_copy_config_t /*config*/ = {}) const noexcept {
1841
1810
  copy_result_t result;
1842
1811
  index_gt& other = result.index;
1843
- other = index_gt(config_, metric_, dynamic_allocator_, tape_allocator_);
1812
+ other = index_gt(config_, dynamic_allocator_, tape_allocator_);
1844
1813
  if (!other.reserve(limits_))
1845
1814
  return result.failed("Failed to reserve the contexts");
1846
1815
 
1847
1816
  // Now all is left - is to allocate new `node_t` instances and populate
1848
1817
  // the `other.nodes_` array into it.
1849
- for (std::size_t i = 0; i != size_; ++i)
1850
- other.nodes_[i] = other.node_make_copy_(node_bytes_split_(nodes_[i]));
1818
+ for (std::size_t i = 0; i != nodes_count_; ++i)
1819
+ other.nodes_[i] = other.node_make_copy_(node_bytes_(nodes_[i]));
1851
1820
 
1852
- other.size_ = size_.load();
1821
+ other.nodes_count_ = nodes_count_.load();
1853
1822
  other.max_level_ = max_level_;
1854
- other.entry_id_ = entry_id_;
1823
+ other.entry_slot_ = entry_slot_;
1855
1824
  return result;
1856
1825
  }
1857
1826
 
@@ -1862,13 +1831,10 @@ class index_gt {
1862
1831
  member_iterator_t begin() noexcept { return {this, 0}; }
1863
1832
  member_iterator_t end() noexcept { return {this, size()}; }
1864
1833
 
1865
- member_ref_t at(std::size_t i) noexcept {
1866
- return {nodes_[i].label(), nodes_[i].vector_view(), static_cast<id_t>(i)};
1867
- }
1868
-
1869
- member_cref_t at(std::size_t i) const noexcept {
1870
- return {nodes_[i].label(), nodes_[i].vector_view(), static_cast<id_t>(i)};
1871
- }
1834
+ member_ref_t at(std::size_t slot) noexcept { return {nodes_[slot].key(), slot}; }
1835
+ member_cref_t at(std::size_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
1836
+ member_iterator_t iterator_at(std::size_t slot) noexcept { return {this, slot}; }
1837
+ member_citerator_t citerator_at(std::size_t slot) const noexcept { return {this, slot}; }
1872
1838
 
1873
1839
  dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
1874
1840
  tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
@@ -1876,40 +1842,40 @@ class index_gt {
1876
1842
  #pragma region Adjusting Configuration
1877
1843
 
1878
1844
  /**
1879
- * @brief Erases all the vectors from the index.
1880
- * Will change `size()` to zero, but will keep the same `capacity()`.
1881
- * Will keep the number of available threads/contexts the same as it was.
1845
+ * @brief Erases all the vectors from the index.
1846
+ *
1847
+ * Will change `size()` to zero, but will keep the same `capacity()`.
1848
+ * Will keep the number of available threads/contexts the same as it was.
1882
1849
  */
1883
1850
  void clear() noexcept {
1884
1851
  if (!has_reset<tape_allocator_t>()) {
1885
- std::size_t n = size_;
1852
+ std::size_t n = nodes_count_;
1886
1853
  for (std::size_t i = 0; i != n; ++i)
1887
1854
  node_free_(i);
1888
1855
  } else
1889
1856
  tape_allocator_.deallocate(nullptr, 0);
1890
- size_ = 0;
1857
+ nodes_count_ = 0;
1891
1858
  max_level_ = -1;
1892
- entry_id_ = 0u;
1859
+ entry_slot_ = 0u;
1893
1860
  }
1894
1861
 
1895
1862
  /**
1896
- * @brief Erases all the vectors from the index, also deallocating the registry.
1897
- * Will change both `size()` and `capacity()` to zero.
1898
- * Will deallocate all threads/contexts.
1863
+ * @brief Erases all members from index, closing files, and returning RAM to OS.
1864
+ *
1865
+ * Will change both `size()` and `capacity()` to zero.
1866
+ * Will deallocate all threads/contexts.
1867
+ * If the index is memory-mapped - releases the mapping and the descriptor.
1899
1868
  */
1900
1869
  void reset() noexcept {
1901
1870
  clear();
1902
1871
 
1903
- if (nodes_)
1904
- nodes_allocator_t{}.deallocate(exchange(nodes_, nullptr), limits_.members);
1905
- if (contexts_) {
1906
- for (std::size_t i = 0; i != limits_.threads(); ++i)
1907
- contexts_[i].~context_t();
1908
- contexts_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
1909
- }
1872
+ nodes_ = {};
1873
+ contexts_ = {};
1874
+ nodes_mutexes_ = {};
1910
1875
  limits_ = index_limits_t{0, 0};
1911
- capacity_ = 0;
1912
- reset_view_();
1876
+ nodes_capacity_ = 0;
1877
+ viewed_file_ = memory_mapped_file_t{};
1878
+ tape_allocator_ = {};
1913
1879
  }
1914
1880
 
1915
1881
  /**
@@ -1918,24 +1884,23 @@ class index_gt {
1918
1884
  void swap(index_gt& other) noexcept {
1919
1885
  std::swap(config_, other.config_);
1920
1886
  std::swap(limits_, other.limits_);
1921
- std::swap(metric_, other.metric_);
1922
1887
  std::swap(dynamic_allocator_, other.dynamic_allocator_);
1923
1888
  std::swap(tape_allocator_, other.tape_allocator_);
1924
1889
  std::swap(pre_, other.pre_);
1925
1890
  std::swap(viewed_file_, other.viewed_file_);
1926
1891
  std::swap(max_level_, other.max_level_);
1927
- std::swap(entry_id_, other.entry_id_);
1892
+ std::swap(entry_slot_, other.entry_slot_);
1928
1893
  std::swap(nodes_, other.nodes_);
1929
1894
  std::swap(nodes_mutexes_, other.nodes_mutexes_);
1930
1895
  std::swap(contexts_, other.contexts_);
1931
1896
 
1932
1897
  // Non-atomic parts.
1933
- std::size_t capacity_copy = capacity_;
1934
- std::size_t size_copy = size_;
1935
- capacity_ = other.capacity_.load();
1936
- size_ = other.size_.load();
1937
- other.capacity_ = capacity_copy;
1938
- other.size_ = size_copy;
1898
+ std::size_t capacity_copy = nodes_capacity_;
1899
+ std::size_t count_copy = nodes_count_;
1900
+ nodes_capacity_ = other.nodes_capacity_.load();
1901
+ nodes_count_ = other.nodes_count_.load();
1902
+ other.nodes_capacity_ = capacity_copy;
1903
+ other.nodes_count_ = count_copy;
1939
1904
  }
1940
1905
 
1941
1906
  /**
@@ -1949,72 +1914,30 @@ class index_gt {
1949
1914
  && limits.members <= limits_.members)
1950
1915
  return true;
1951
1916
 
1952
- if (!nodes_mutexes_.resize(limits.members))
1953
- return false;
1954
-
1955
- nodes_allocator_t node_allocator;
1956
- node_t* new_nodes = node_allocator.allocate(limits.members);
1957
- if (!new_nodes)
1917
+ visits_bitset_t new_mutexes(limits.members);
1918
+ buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
1919
+ buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
1920
+ if (!new_nodes || !new_contexts || !new_mutexes)
1958
1921
  return false;
1959
1922
 
1960
- std::size_t limits_threads = limits.threads();
1961
- contexts_allocator_t context_allocator;
1962
- context_t* new_contexts = context_allocator.allocate(limits_threads);
1963
- if (!new_contexts) {
1964
- node_allocator.deallocate(new_nodes, limits.members);
1965
- return false;
1966
- }
1967
- for (std::size_t i = 0; i != limits_threads; ++i) {
1968
- context_t& context = new_contexts[i];
1969
- new (&context) context_t();
1970
- context.metric = metric_;
1971
- if (!context.visits.resize(limits.members)) {
1972
- for (std::size_t j = 0; j != i; ++j)
1973
- context.visits.reset();
1974
- node_allocator.deallocate(new_nodes, limits.members);
1975
- context_allocator.deallocate(new_contexts, limits_threads);
1923
+ for (context_t& context : new_contexts) {
1924
+ context.visits = visits_bitset_t(limits.members);
1925
+ if (!context.visits)
1976
1926
  return false;
1977
- }
1978
- }
1979
-
1980
- // We have passed all the require memory allocations.
1981
- // The remaining code can't fail. Let's just reuse some of our existing buffers.
1982
- for (std::size_t i = 0; i != limits_.threads(); ++i) {
1983
- context_t& old_context = contexts_[i];
1984
- context_t& context = new_contexts[i];
1985
- std::swap(old_context.top_candidates, context.top_candidates);
1986
- std::swap(old_context.next_candidates, context.next_candidates);
1987
- std::swap(old_context.iteration_cycles, context.iteration_cycles);
1988
- std::swap(old_context.measurements_count, context.measurements_count);
1989
- old_context.visits.reset();
1990
1927
  }
1991
1928
 
1992
1929
  // Move the nodes info, and deallocate previous buffers.
1993
1930
  if (nodes_)
1994
- std::memcpy(new_nodes, nodes_, sizeof(node_t) * size()), node_allocator.deallocate(nodes_, limits_.members);
1995
- if (contexts_)
1996
- context_allocator.deallocate(contexts_, limits_.threads());
1931
+ std::memcpy(new_nodes.data(), nodes_.data(), sizeof(node_t) * size());
1997
1932
 
1998
1933
  limits_ = limits;
1999
- capacity_ = limits.members;
2000
- nodes_ = new_nodes;
2001
- contexts_ = new_contexts;
1934
+ nodes_capacity_ = limits.members;
1935
+ nodes_ = std::move(new_nodes);
1936
+ contexts_ = std::move(new_contexts);
1937
+ nodes_mutexes_ = std::move(new_mutexes);
2002
1938
  return true;
2003
1939
  }
2004
1940
 
2005
- /**
2006
- * @brief Optimizes configuration options to fit the maximum number
2007
- * of neighbors in CPU-cache-aligned buffers.
2008
- */
2009
- static index_config_t optimize(index_config_t config) noexcept {
2010
- precomputed_constants_t pre = precompute_(config);
2011
- std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
2012
- std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
2013
- std::size_t added_connections = (rounded_size - bytes_per_node_base) / sizeof(id_t);
2014
- config.connectivity = config.connectivity + added_connections / base_level_multiple_();
2015
- return config;
2016
- }
2017
-
2018
1941
  #pragma endregion
2019
1942
 
2020
1943
  #pragma region Construction and Search
@@ -2022,9 +1945,9 @@ class index_gt {
2022
1945
  struct add_result_t {
2023
1946
  error_t error{};
2024
1947
  std::size_t new_size{};
2025
- std::size_t cycles{};
2026
- std::size_t measurements{};
2027
- id_t id{};
1948
+ std::size_t visited_members{};
1949
+ std::size_t computed_distances{};
1950
+ std::size_t slot{};
2028
1951
 
2029
1952
  explicit operator bool() const noexcept { return !error; }
2030
1953
  add_result_t failed(error_t message) noexcept {
@@ -2033,29 +1956,31 @@ class index_gt {
2033
1956
  }
2034
1957
  };
2035
1958
 
1959
+ /// @brief Describes a matched search result, augmenting `member_cref_t`
1960
+ /// contents with `distance` to the query object.
2036
1961
  struct match_t {
2037
1962
  member_cref_t member;
2038
1963
  distance_t distance;
2039
1964
  };
2040
1965
 
2041
1966
  class search_result_t {
2042
- index_gt const& index_;
2043
- top_candidates_t& top_;
1967
+ node_t const* nodes_{};
1968
+ top_candidates_t const* top_{};
2044
1969
 
2045
1970
  friend class index_gt;
2046
- inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
1971
+ inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
1972
+ : nodes_(index.nodes_), top_(&top) {}
2047
1973
 
2048
1974
  public:
1975
+ /** @brief Number of search results found. */
2049
1976
  std::size_t count{};
2050
- std::size_t cycles{};
2051
- std::size_t measurements{};
1977
+ /** @brief Number of graph nodes traversed. */
1978
+ std::size_t visited_members{};
1979
+ /** @brief Number of times the distances were computed. */
1980
+ std::size_t computed_distances{};
2052
1981
  error_t error{};
2053
1982
 
2054
- // todo: this is a hack to get a handle to a failed search_result_t without changing its member types
2055
- // given that search_result_t can represent a failed result (which implies no valid top_candidates_t &top_),
2056
- // the top_ probably needs to be a pointer.
2057
- inline search_result_t(index_gt const& index) noexcept
2058
- : index_(index), top_(index.contexts_[0].top_candidates) {}
1983
+ inline search_result_t() noexcept {}
2059
1984
  inline search_result_t(search_result_t&&) = default;
2060
1985
  inline search_result_t& operator=(search_result_t&&) = default;
2061
1986
 
@@ -2071,46 +1996,88 @@ class index_gt {
2071
1996
  inline match_t operator[](std::size_t i) const noexcept { return at(i); }
2072
1997
  inline match_t front() const noexcept { return at(0); }
2073
1998
  inline match_t back() const noexcept { return at(count - 1); }
2074
- inline bool contains(label_t label) const noexcept {
1999
+ inline bool contains(key_t key) const noexcept {
2075
2000
  for (std::size_t i = 0; i != count; ++i)
2076
- if (at(i).member.label == label)
2001
+ if (at(i).member.key == key)
2077
2002
  return true;
2078
2003
  return false;
2079
2004
  }
2080
2005
  inline match_t at(std::size_t i) const noexcept {
2081
- candidate_t const* top_ordered = top_.data();
2006
+ candidate_t const* top_ordered = top_->data();
2082
2007
  candidate_t candidate = top_ordered[i];
2083
- node_t node = index_.node_with_id_(candidate.id);
2084
- return {member_cref_t{node.label(), node.vector_view(), candidate.id}, candidate.distance};
2008
+ node_t node = nodes_[candidate.slot];
2009
+ return {member_cref_t{node.ckey(), candidate.slot}, candidate.distance};
2085
2010
  }
2086
- inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
2011
+ inline std::size_t merge_into( //
2012
+ key_t* keys, distance_t* distances, //
2013
+ std::size_t old_count, std::size_t max_count) const noexcept {
2014
+
2015
+ std::size_t merged_count = old_count;
2087
2016
  for (std::size_t i = 0; i != count; ++i) {
2088
2017
  match_t result = operator[](i);
2089
- labels[i] = result.member.label;
2018
+ distance_t* merged_end = distances + merged_count;
2019
+ std::size_t offset = std::lower_bound(distances, merged_end, result.distance) - distances;
2020
+ if (offset == max_count)
2021
+ continue;
2022
+
2023
+ std::size_t count_worse = merged_count - offset - (max_count == merged_count);
2024
+ std::memmove(keys + offset + 1, keys + offset, count_worse * sizeof(key_t));
2025
+ std::memmove(distances + offset + 1, distances + offset, count_worse * sizeof(distance_t));
2026
+ keys[offset] = result.member.key;
2027
+ distances[offset] = result.distance;
2028
+ merged_count += merged_count != max_count;
2029
+ }
2030
+ return merged_count;
2031
+ }
2032
+ inline std::size_t dump_to(key_t* keys, distance_t* distances) const noexcept {
2033
+ for (std::size_t i = 0; i != count; ++i) {
2034
+ match_t result = operator[](i);
2035
+ keys[i] = result.member.key;
2090
2036
  distances[i] = result.distance;
2091
2037
  }
2092
2038
  return count;
2093
2039
  }
2094
- inline std::size_t dump_to(label_t* labels) const noexcept {
2040
+ inline std::size_t dump_to(key_t* keys) const noexcept {
2095
2041
  for (std::size_t i = 0; i != count; ++i) {
2096
2042
  match_t result = operator[](i);
2097
- labels[i] = result.member.label;
2043
+ keys[i] = result.member.key;
2098
2044
  }
2099
2045
  return count;
2100
2046
  }
2101
2047
  };
2102
2048
 
2103
2049
  /**
2104
- * @brief Inserts a new vector into the index. Thread-safe.
2050
+ * @brief Inserts a new entry into the index. Thread-safe. Supports @b heterogeneous lookups.
2051
+ * Expects needed capacity to be reserved ahead of time: `size() < capacity()`.
2052
+ *
2053
+ * @tparam metric_at
2054
+ * A function responsible for computing the distance @b (dis-similarity) between two objects.
2055
+ * It should be callable into distinctly different scenarios:
2056
+ * - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
2057
+ * - `distance_t operator() (entry_at, entry_at)` - between existing entries.
2058
+ * Where any possible `entry_at` has both two interfaces: `std::size_t slot()`, `key_t key()`.
2105
2059
  *
2106
- * @param[in] label External identifier/name/descriptor for the vector.
2107
- * @param[in] vector Contiguous range of scalars forming a vector view.
2060
+ * @param[in] key External identifier/name/descriptor for the new entry.
2061
+ * @param[in] value Content that will be compared against other entries to index.
2062
+ * @param[in] metric Callable object measuring distance between ::value and present objects.
2108
2063
  * @param[in] config Configuration options for this specific operation.
2064
+ * @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
2109
2065
  */
2110
- add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) usearch_noexcept_m {
2066
+ template < //
2067
+ typename value_at, //
2068
+ typename metric_at, //
2069
+ typename callback_at = dummy_callback_t, //
2070
+ typename prefetch_at = dummy_prefetch_t //
2071
+ >
2072
+ add_result_t add( //
2073
+ key_t key, value_at&& value, metric_at&& metric, //
2074
+ index_update_config_t config = {}, //
2075
+ callback_at&& callback = callback_at{}, //
2076
+ prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
2111
2077
 
2112
- usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
2113
2078
  add_result_t result;
2079
+ if (is_immutable())
2080
+ return result.failed("Can't add to an immutable index");
2114
2081
 
2115
2082
  // Make sure we have enough local memory to perform this request
2116
2083
  context_t& context = contexts_[config.thread];
@@ -2121,7 +2088,8 @@ class index_gt {
2121
2088
 
2122
2089
  // The top list needs one more slot than the connectivity of the base level
2123
2090
  // for the heuristic, that tries to squeeze one more element into saturated list.
2124
- std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config.expansion);
2091
+ std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
2092
+ std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
2125
2093
  if (!top.reserve(top_limit))
2126
2094
  return result.failed("Out of memory!");
2127
2095
  if (!next.reserve(config.expansion))
@@ -2129,60 +2097,98 @@ class index_gt {
2129
2097
 
2130
2098
  // Determining how much memory to allocate for the node depends on the target level
2131
2099
  std::unique_lock<std::mutex> new_level_lock(global_mutex_);
2132
- level_t max_level_copy = max_level_; // Copy under lock
2133
- id_t entry_id_copy = entry_id_; // Copy under lock
2100
+ level_t max_level_copy = max_level_; // Copy under lock
2101
+ std::size_t entry_idx_copy = entry_slot_; // Copy under lock
2134
2102
  level_t target_level = choose_random_level_(context.level_generator);
2135
- if (target_level <= max_level_copy)
2136
- new_level_lock.unlock();
2103
+
2104
+ // Make sure we are not overflowing
2105
+ std::size_t capacity = nodes_capacity_.load();
2106
+ std::size_t new_slot = nodes_count_.fetch_add(1);
2107
+ if (new_slot >= capacity) {
2108
+ nodes_count_.fetch_sub(1);
2109
+ return result.failed("Reserve capacity ahead of insertions!");
2110
+ }
2137
2111
 
2138
2112
  // Allocate the neighbors
2139
- node_t node = node_make_(label, vector, target_level, config.store_vector);
2140
- if (!node)
2113
+ node_t node = node_make_(key, target_level);
2114
+ if (!node) {
2115
+ nodes_count_.fetch_sub(1);
2141
2116
  return result.failed("Out of memory!");
2142
- std::size_t old_size = size_.fetch_add(1);
2143
- id_t new_id = static_cast<id_t>(old_size);
2144
- nodes_[old_size] = node;
2145
- result.new_size = old_size + 1;
2146
- result.id = new_id;
2147
- node_lock_t new_lock = node_lock_(old_size);
2117
+ }
2118
+ if (target_level <= max_level_copy)
2119
+ new_level_lock.unlock();
2120
+
2121
+ nodes_[new_slot] = node;
2122
+ result.new_size = new_slot + 1;
2123
+ result.slot = new_slot;
2124
+ callback(at(new_slot));
2125
+ node_lock_t new_lock = node_lock_(new_slot);
2148
2126
 
2149
2127
  // Do nothing for the first element
2150
- if (!new_id) {
2151
- entry_id_ = new_id;
2128
+ if (!new_slot) {
2129
+ entry_slot_ = new_slot;
2152
2130
  max_level_ = target_level;
2153
2131
  return result;
2154
2132
  }
2155
2133
 
2156
2134
  // Pull stats
2157
- result.measurements = context.measurements_count;
2158
- result.cycles = context.iteration_cycles;
2135
+ result.computed_distances = context.computed_distances_count;
2136
+ result.visited_members = context.iteration_cycles;
2159
2137
 
2160
- connect_node_across_levels_(new_id, vector, entry_id_copy, max_level_copy, target_level, config, context);
2138
+ connect_node_across_levels_( //
2139
+ value, metric, prefetch, //
2140
+ new_slot, entry_idx_copy, max_level_copy, target_level, //
2141
+ config, context);
2161
2142
 
2162
2143
  // Normalize stats
2163
- result.measurements = context.measurements_count - result.measurements;
2164
- result.cycles = context.iteration_cycles - result.cycles;
2144
+ result.computed_distances = context.computed_distances_count - result.computed_distances;
2145
+ result.visited_members = context.iteration_cycles - result.visited_members;
2165
2146
 
2166
2147
  // Updating the entry point if needed
2167
2148
  if (target_level > max_level_copy) {
2168
- entry_id_ = new_id;
2149
+ entry_slot_ = new_slot;
2169
2150
  max_level_ = target_level;
2170
2151
  }
2171
2152
  return result;
2172
2153
  }
2173
2154
 
2174
2155
  /**
2175
- * @brief Update an existing entry, replacing a vector and a label. Thread-safe.
2156
+ * @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
2176
2157
  *
2177
- * @param[in] old_id Existing internal identifier for a node to be replaced.
2178
- * @param[in] label External identifier/name/descriptor for the vector.
2179
- * @param[in] vector Contiguous range of scalars forming a vector view.
2158
+ * @tparam metric_at
2159
+ * A function responsible for computing the distance @b (dis-similarity) between two objects.
2160
+ * It should be callable into distinctly different scenarios:
2161
+ * - `distance_t operator() (value_at, entry_at)` - from new object to existing entries.
2162
+ * - `distance_t operator() (entry_at, entry_at)` - between existing entries.
2163
+ * For any possible `entry_at` following interfaces will work:
2164
+ * - `std::size_t get_slot(entry_at const &)`
2165
+ * - `key_t get_key(entry_at const &)`
2166
+ *
2167
+ * @param[in] iterator Iterator pointing to an existing entry to be replaced.
2168
+ * @param[in] key External identifier/name/descriptor for the entry.
2169
+ * @param[in] value Content that will be compared against other entries in the index.
2170
+ * @param[in] metric Callable object measuring distance between ::value and present objects.
2180
2171
  * @param[in] config Configuration options for this specific operation.
2172
+ * @param[in] callback On-success callback, executed while the `member_ref_t` is still under lock.
2181
2173
  */
2182
- add_result_t update(id_t old_id, label_t label, vector_view_t vector, add_config_t config = {}) usearch_noexcept_m {
2174
+ template < //
2175
+ typename value_at, //
2176
+ typename metric_at, //
2177
+ typename callback_at = dummy_callback_t, //
2178
+ typename prefetch_at = dummy_prefetch_t //
2179
+ >
2180
+ add_result_t update( //
2181
+ member_iterator_t iterator, //
2182
+ key_t key, //
2183
+ value_at&& value, //
2184
+ metric_at&& metric, //
2185
+ index_update_config_t config = {}, //
2186
+ callback_at&& callback = callback_at{}, //
2187
+ prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
2183
2188
 
2184
2189
  usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
2185
2190
  add_result_t result;
2191
+ std::size_t old_slot = iterator.slot_;
2186
2192
 
2187
2193
  // Make sure we have enough local memory to perform this request
2188
2194
  context_t& context = contexts_[config.thread];
@@ -2193,57 +2199,77 @@ class index_gt {
2193
2199
 
2194
2200
  // The top list needs one more slot than the connectivity of the base level
2195
2201
  // for the heuristic, that tries to squeeze one more element into saturated list.
2196
- std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config.expansion);
2202
+ std::size_t connectivity_max = (std::max)(config_.connectivity_base, config_.connectivity);
2203
+ std::size_t top_limit = (std::max)(connectivity_max + 1, config.expansion);
2197
2204
  if (!top.reserve(top_limit))
2198
2205
  return result.failed("Out of memory!");
2199
2206
  if (!next.reserve(config.expansion))
2200
2207
  return result.failed("Out of memory!");
2201
2208
 
2202
- node_lock_t new_lock = node_lock_(old_id);
2203
- node_t node = node_with_id_(old_id);
2209
+ node_lock_t new_lock = node_lock_(old_slot);
2210
+ node_t node = node_at_(old_slot);
2211
+
2212
+ level_t node_level = node.level();
2213
+ span_bytes_t node_bytes = node_bytes_(node);
2214
+ std::memset(node_bytes.data(), 0, node_bytes.size());
2215
+ node.level(node_level);
2204
2216
 
2205
2217
  // Pull stats
2206
- result.measurements = context.measurements_count;
2207
- result.cycles = context.iteration_cycles;
2218
+ result.computed_distances = context.computed_distances_count;
2219
+ result.visited_members = context.iteration_cycles;
2208
2220
 
2209
- connect_node_across_levels_(old_id, vector, entry_id_, max_level_, node.level(), config, context);
2210
- node.label(label);
2221
+ connect_node_across_levels_( //
2222
+ value, metric, prefetch, //
2223
+ old_slot, entry_slot_, max_level_, node_level, //
2224
+ config, context);
2225
+ node.key(key);
2211
2226
 
2212
2227
  // Normalize stats
2213
- result.measurements = context.measurements_count - result.measurements;
2214
- result.cycles = context.iteration_cycles - result.cycles;
2228
+ result.computed_distances = context.computed_distances_count - result.computed_distances;
2229
+ result.visited_members = context.iteration_cycles - result.visited_members;
2230
+ result.slot = old_slot;
2215
2231
 
2232
+ callback(at(old_slot));
2216
2233
  return result;
2217
2234
  }
2218
2235
 
2219
2236
  /**
2220
2237
  * @brief Searches for the closest elements to the given ::query. Thread-safe.
2221
2238
  *
2222
- * @param[in] query Contiguous range of scalars forming a vector view.
2239
+ * @param[in] query Content that will be compared against other entries in the index.
2223
2240
  * @param[in] wanted The upper bound for the number of results to return.
2224
2241
  * @param[in] config Configuration options for this specific operation.
2225
2242
  * @param[in] predicate Optional filtering predicate for `member_cref_t`.
2226
2243
  * @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
2227
2244
  */
2228
- template <typename predicate_at = dummy_predicate_t>
2229
- search_result_t search( //
2230
- vector_view_t query, std::size_t wanted, search_config_t config = {},
2231
- predicate_at&& predicate = dummy_predicate_t{}) const noexcept {
2245
+ template < //
2246
+ typename value_at, //
2247
+ typename metric_at, //
2248
+ typename predicate_at = dummy_predicate_t, //
2249
+ typename prefetch_at = dummy_prefetch_t //
2250
+ >
2251
+ search_result_t search( //
2252
+ value_at&& query, //
2253
+ std::size_t wanted, //
2254
+ metric_at&& metric, //
2255
+ index_search_config_t config = {}, //
2256
+ predicate_at&& predicate = predicate_at{}, //
2257
+ prefetch_at&& prefetch = prefetch_at{}) const noexcept {
2232
2258
 
2233
2259
  context_t& context = contexts_[config.thread];
2234
2260
  top_candidates_t& top = context.top_candidates;
2235
2261
  search_result_t result{*this, top};
2236
- if (!size_)
2262
+ if (!nodes_count_)
2237
2263
  return result;
2238
2264
 
2239
2265
  // Go down the level, tracking only the closest match
2240
- result.measurements = context.measurements_count;
2241
- result.cycles = context.iteration_cycles;
2266
+ result.computed_distances = context.computed_distances_count;
2267
+ result.visited_members = context.iteration_cycles;
2242
2268
 
2243
2269
  if (config.exact) {
2244
2270
  if (!top.reserve(wanted))
2245
2271
  return result.failed("Out of memory!");
2246
- search_exact_(query, wanted, context, std::forward<predicate_at>(predicate));
2272
+ search_exact_(query, metric, predicate, wanted, context);
2247
2273
  } else {
2248
2274
  next_candidates_t& next = context.next_candidates;
2249
2275
  std::size_t expansion = (std::max)(config.expansion, wanted);
@@ -2252,9 +2278,10 @@ class index_gt {
2252
2278
  if (!top.reserve(expansion))
2253
2279
  return result.failed("Out of memory!");
2254
2280
 
2255
- id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
2281
+ std::size_t closest_slot = search_for_one_(query, metric, prefetch, entry_slot_, max_level_, 0, context);
2282
+
2256
2283
  // For bottom layer we need a more optimized procedure
2257
- if (!search_to_find_in_base_(closest_id, query, expansion, context, std::forward<predicate_at>(predicate)))
2284
+ if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
2258
2285
  return result.failed("Out of memory!");
2259
2286
  }
2260
2287
 
@@ -2262,8 +2289,8 @@ class index_gt {
2262
2289
  top.shrink(wanted);
2263
2290
 
2264
2291
  // Normalize stats
2265
- result.measurements = context.measurements_count - result.measurements;
2266
- result.cycles = context.iteration_cycles - result.cycles;
2292
+ result.computed_distances = context.computed_distances_count - result.computed_distances;
2293
+ result.visited_members = context.iteration_cycles - result.visited_members;
2267
2294
  result.count = top.size();
2268
2295
  return result;
2269
2296
  }
@@ -2283,13 +2310,13 @@ class index_gt {
2283
2310
  stats_t result{};
2284
2311
  result.nodes = size();
2285
2312
  for (std::size_t i = 0; i != result.nodes; ++i) {
2286
- node_t node = node_with_id_(i);
2287
- std::size_t max_edges = node.level() * config_.connectivity + base_level_multiple_() * config_.connectivity;
2313
+ node_t node = node_at_(i);
2314
+ std::size_t max_edges = node.level() * config_.connectivity + config_.connectivity_base;
2288
2315
  std::size_t edges = 0;
2289
2316
  for (level_t level = 0; level <= node.level(); ++level)
2290
2317
  edges += neighbors_(node, level).size();
2291
2318
 
2292
- result.allocated_bytes += node_bytes_(node);
2319
+ result.allocated_bytes += node_bytes_(node).size();
2293
2320
  result.edges += edges;
2294
2321
  result.max_edges += max_edges;
2295
2322
  }
@@ -2302,15 +2329,15 @@ class index_gt {
2302
2329
 
2303
2330
  std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
2304
2331
  for (std::size_t i = 0; i != result.nodes; ++i) {
2305
- node_t node = node_with_id_(i);
2332
+ node_t node = node_at_(i);
2306
2333
  if (static_cast<std::size_t>(node.level()) < level)
2307
2334
  continue;
2308
2335
 
2309
2336
  result.edges += neighbors_(node, level).size();
2310
- result.allocated_bytes += node_head_bytes_() + node_vector_bytes_(node) + neighbors_bytes;
2337
+ result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
2311
2338
  }
2312
2339
 
2313
- std::size_t max_edges_per_node = level ? config_.connectivity : base_level_multiple_() * config_.connectivity;
2340
+ std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
2314
2341
  result.max_edges = result.nodes * max_edges_per_node;
2315
2342
  return result;
2316
2343
  }
@@ -2318,6 +2345,8 @@ class index_gt {
2318
2345
  /**
2319
2346
  * @brief A relatively accurate lower bound on the amount of memory consumed by the system.
2320
2347
  * In practice it's error will be below 10%.
2348
+ *
2349
+ * @see `stream_length` for the length of the binary serialized representation.
2321
2350
  */
2322
2351
  std::size_t memory_usage(std::size_t allocator_entry_bytes = default_allocator_entry_bytes()) const noexcept {
2323
2352
  std::size_t total = 0;
@@ -2335,367 +2364,297 @@ class index_gt {
2335
2364
  return total;
2336
2365
  }
2337
2366
 
2338
- std::size_t memory_usage_per_node(dim_t dim, level_t level) const noexcept { return node_bytes_(dim, level); }
2339
-
2340
- void change_metric(metric_t const& m) noexcept {
2341
- metric_ = m;
2342
- for (std::size_t i = 0; i != limits_.threads(); ++i)
2343
- contexts_[i].metric = metric_;
2344
- }
2367
+ std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
2345
2368
 
2346
2369
  #pragma endregion
2347
2370
 
2348
2371
  #pragma region Serialization
2349
2372
 
2350
- struct serialization_result_t {
2351
- error_t error;
2352
-
2353
- explicit operator bool() const noexcept { return !error; }
2354
- serialization_result_t failed(error_t message) noexcept {
2355
- error = std::move(message);
2356
- return std::move(*this);
2357
- }
2358
- };
2359
-
2360
2373
  /**
2361
- * @brief Saves serialized binary index representation to disk,
2362
- * co-locating vectors and neighbors lists.
2363
- * Available on Linux, MacOS, Windows.
2374
+ * @brief Saves serialized binary index representation to a file, generally on disk.
2364
2375
  */
2365
2376
  template <typename progress_at = dummy_progress_t>
2366
- serialization_result_t save(char const* file_path, progress_at&& progress = {}) const noexcept {
2377
+ serialization_result_t save(output_file_t file, progress_at&& progress = {}) const noexcept {
2378
+
2379
+ serialization_result_t result = file.open_if_not();
2380
+ if (result)
2381
+ stream(
2382
+ [&](void* buffer, std::size_t length) {
2383
+ result = file.write(buffer, length);
2384
+ return !!result;
2385
+ },
2386
+ std::forward<progress_at>(progress));
2387
+ return result;
2388
+ }
2389
+
2390
+ /**
2391
+ * @brief Saves serialized binary index representation to a stream.
2392
+ */
2393
+ template <typename output_callback_at, typename progress_at = dummy_progress_t>
2394
+ serialization_result_t stream(output_callback_at&& callback, progress_at&& progress = {}) const noexcept {
2367
2395
 
2368
- // Make sure we have right to write to that file
2369
2396
  serialization_result_t result;
2370
- std::FILE* file = std::fopen(file_path, "wb");
2371
- if (!file)
2372
- return result.failed(std::strerror(errno));
2373
2397
 
2374
- // Prepare the header with metadata
2375
- file_header_t state_buffer{};
2376
- std::memset(state_buffer, 0, sizeof(file_header_t));
2377
- file_head_t state{state_buffer};
2378
- std::memcpy(state_buffer, default_magic(), std::strlen(default_magic()));
2379
-
2380
- // Mark compatibility
2381
- state.version_major = USEARCH_VERSION_MAJOR;
2382
- state.version_minor = USEARCH_VERSION_MINOR;
2383
- state.version_patch = USEARCH_VERSION_PATCH;
2384
- state.metric = metric_.kind();
2385
-
2386
- // Describe state
2387
- state.connectivity = config_.connectivity;
2388
- state.max_level = max_level_;
2389
- state.vector_alignment = config_.vector_alignment;
2390
- state.bytes_per_label = sizeof(label_t);
2391
- state.bytes_per_id = sizeof(id_t);
2392
- state.scalar_kind = metric_.scalar_kind();
2393
- state.size = size_;
2394
- state.entry_idx = entry_id_;
2395
-
2396
- // Augment with metadata
2397
- std::size_t graphs_bytes = 0;
2398
- std::size_t vectors_bytes = 0;
2399
- for (std::size_t i = 0; i != state.size; ++i) {
2400
- node_t node = node_with_id_(i);
2401
- std::size_t node_bytes = node_bytes_(node);
2402
- std::size_t node_vector_bytes = node_vector_bytes_(node);
2403
- graphs_bytes += node_bytes - node_vector_bytes;
2404
- vectors_bytes += node_vector_bytes;
2398
+ // Export some basic metadata
2399
+ index_serialized_header_t header;
2400
+ header.size = nodes_count_;
2401
+ header.connectivity = config_.connectivity;
2402
+ header.connectivity_base = config_.connectivity_base;
2403
+ header.max_level = max_level_;
2404
+ header.entry_slot = entry_slot_;
2405
+ if (!callback(&header, sizeof(header)))
2406
+ return result.failed("Failed to serialize into stream");
2407
+
2408
+ // Export the number of levels per node
2409
+ // That is both enough to estimate the overall memory consumption,
2410
+ // and to be able to estimate the offsets of every entry in the file.
2411
+ for (std::size_t i = 0; i != header.size; ++i) {
2412
+ node_t node = node_at_(i);
2413
+ level_t level = node.level();
2414
+ if (!callback(&level, sizeof(level)))
2415
+ return result.failed("Failed to serialize into stream");
2405
2416
  }
2406
- state.bytes_for_graphs = graphs_bytes;
2407
- state.bytes_for_vectors = vectors_bytes;
2408
- state.bytes_checksum = 0;
2409
-
2410
- // Perform serialization
2411
- auto write_chunk = [&](void* begin, std::size_t length) {
2412
- std::size_t written = std::fwrite(begin, length, 1, file);
2413
- if (!written) {
2414
- std::fclose(file);
2415
- result.failed(std::strerror(errno));
2416
- }
2417
- };
2418
2417
 
2419
- // Write the header
2420
- write_chunk(&state_buffer[0], sizeof(file_header_t));
2421
- if (result.error)
2422
- return result;
2423
-
2424
- // Serialize nodes one by one
2425
- for (std::size_t i = 0; i != state.size; ++i) {
2426
- node_t node = node_with_id_(i);
2427
- std::size_t node_bytes = node_bytes_(node);
2428
- std::size_t node_vector_bytes = node_vector_bytes_(node);
2429
- // Dump neighbors and vectors, as vectors may be in a disjoint location
2430
- write_chunk(node.tape(), node_bytes - node_vector_bytes);
2431
- if (result.error)
2432
- return result;
2433
- write_chunk(node.vector(), node_vector_bytes);
2434
- if (result.error)
2435
- return result;
2436
- progress(i, state.size);
2418
+ // After that dump the nodes themselves
2419
+ for (std::size_t i = 0; i != header.size; ++i) {
2420
+ span_bytes_t node_bytes = node_bytes_(node_at_(i));
2421
+ if (!callback(node_bytes.data(), node_bytes.size()))
2422
+ return result.failed("Failed to serialize into stream");
2423
+ progress(i, header.size);
2437
2424
  }
2438
2425
 
2439
- std::fclose(file);
2440
2426
  return {};
2441
2427
  }
2442
2428
 
2443
2429
  /**
2444
- * @brief Loads the serialized binary index representation from disk,
2445
- * copying both vectors and neighbors lists into RAM.
2446
- * Available on Linux, MacOS, Windows.
2430
+ * @brief Estimate the binary length (in bytes) of the serialized index.
2431
+ */
2432
+ std::size_t stream_length() const noexcept {
2433
+ std::size_t neighbors_length = 0;
2434
+ for (std::size_t i = 0; i != size(); ++i)
2435
+ neighbors_length += node_bytes_(node_at_(i).level());
2436
+ return sizeof(index_serialized_header_t) + neighbors_length;
2437
+ }
2438
+
2439
+ /**
2440
+ * @brief Loads the serialized binary index representation from disk to RAM.
2441
+ * Adjusts the configuration properties of the constructed index to
2442
+ * match the settings in the file.
2447
2443
  */
2448
2444
  template <typename progress_at = dummy_progress_t>
2449
- serialization_result_t load(char const* file_path, progress_at&& progress = {}) noexcept {
2445
+ serialization_result_t load(input_file_t file, progress_at&& progress = {}) noexcept {
2450
2446
 
2451
2447
  // Remove previously stored objects
2452
2448
  reset();
2453
2449
 
2454
- // Start parsing the files
2455
- serialization_result_t result;
2456
- file_header_t state_buffer{};
2457
- std::FILE* file = std::fopen(file_path, "rb");
2458
- if (!file)
2459
- return result.failed(std::strerror(errno));
2460
-
2461
- auto read_chunk = [&](void* begin, std::size_t length) {
2462
- std::size_t read = std::fread(begin, length, 1, file);
2463
- if (!read) {
2464
- std::fclose(file);
2465
- result.failed(std::strerror(errno));
2466
- }
2467
- };
2468
-
2469
- // Read the header
2470
- {
2471
- read_chunk(&state_buffer[0], sizeof(file_header_t));
2472
- if (result.error)
2473
- return result;
2450
+ serialization_result_t result = file.open_if_not();
2451
+ if (!result)
2452
+ return result;
2474
2453
 
2475
- file_head_t state{state_buffer};
2476
- if (state.bytes_per_label != sizeof(label_t)) {
2477
- std::fclose(file);
2478
- return result.failed("Incompatible label type!");
2479
- }
2480
- if (state.bytes_per_id != sizeof(id_t)) {
2481
- std::fclose(file);
2482
- return result.failed("Incompatible ID type!");
2483
- }
2454
+ // Pull basic metadata
2455
+ index_serialized_header_t header;
2456
+ result = file.read(&header, sizeof(header));
2457
+ if (!result)
2458
+ return result;
2484
2459
 
2485
- config_.connectivity = state.connectivity;
2486
- config_.vector_alignment = state.vector_alignment;
2487
- pre_ = precompute_(config_);
2460
+ // Allocate some dynamic memory to read all the levels
2461
+ using levels_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<level_t>;
2462
+ buffer_gt<level_t, levels_allocator_t> levels(header.size);
2463
+ if (!levels)
2464
+ return result.failed("Out of memory");
2465
+ result = file.read(levels, header.size * sizeof(level_t));
2466
+ if (!result)
2467
+ return result;
2488
2468
 
2489
- index_limits_t limits;
2490
- limits.members = state.size;
2491
- if (!reserve(limits)) {
2492
- std::fclose(file);
2493
- return result.failed("Out of memory");
2494
- }
2495
- size_ = state.size;
2496
- max_level_ = static_cast<level_t>(state.max_level);
2497
- entry_id_ = static_cast<id_t>(state.entry_idx);
2469
+ // Submit metadata
2470
+ config_.connectivity = header.connectivity;
2471
+ config_.connectivity_base = header.connectivity_base;
2472
+ pre_ = precompute_(config_);
2473
+ index_limits_t limits;
2474
+ limits.members = header.size;
2475
+ if (!reserve(limits)) {
2476
+ reset();
2477
+ return result.failed("Out of memory");
2498
2478
  }
2499
-
2500
- // Load nodes one by one
2501
- std::size_t const size = size_;
2502
- for (std::size_t i = 0; i != size; ++i) {
2503
- label_t label;
2504
- dim_t dim;
2505
- level_t level;
2506
- read_chunk(&label, sizeof(label));
2507
- if (result.error)
2508
- return result;
2509
- read_chunk(&dim, sizeof(dim));
2510
- if (result.error)
2479
+ nodes_count_ = header.size;
2480
+ max_level_ = static_cast<level_t>(header.max_level);
2481
+ entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
2482
+
2483
+ // Load the nodes
2484
+ for (std::size_t i = 0; i != header.size; ++i) {
2485
+ span_bytes_t node_bytes = node_malloc_(levels[i]);
2486
+ result = file.read(node_bytes.data(), node_bytes.size());
2487
+ if (!result) {
2488
+ reset();
2511
2489
  return result;
2512
- read_chunk(&level, sizeof(level));
2513
- if (result.error)
2514
- return result;
2515
-
2516
- std::size_t node_bytes = node_bytes_(dim, level);
2517
- node_t node = node_malloc_(dim, level);
2518
- node.label(label);
2519
- node.dim(dim);
2520
- node.level(level);
2521
- read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
2522
- if (result.error)
2523
- return result;
2524
- nodes_[i] = node;
2525
- progress(i, size);
2490
+ }
2491
+ nodes_[i] = node_t{node_bytes.data()};
2492
+ progress(i, header.size);
2526
2493
  }
2527
-
2528
- std::fclose(file);
2529
- reset_view_();
2530
2494
  return {};
2531
2495
  }
2532
2496
 
2533
2497
  /**
2534
2498
  * @brief Memory-maps the serialized binary index representation from disk,
2535
- * @b without copying the vectors and neighbors lists into RAM.
2536
- * Available on Linux, MacOS, but @b not on Windows.
2499
+ * @b without copying data into RAM, and fetching it on-demand.
2537
2500
  */
2538
2501
  template <typename progress_at = dummy_progress_t>
2539
- serialization_result_t view(char const* file_path, progress_at&& progress = {}) noexcept {
2502
+ serialization_result_t view(memory_mapped_file_t file, std::size_t offset = 0,
2503
+ progress_at&& progress = {}) noexcept {
2540
2504
 
2541
2505
  // Remove previously stored objects
2542
2506
  reset();
2543
2507
 
2544
- // Start parsing the files
2545
- serialization_result_t result;
2546
-
2547
- #if defined(USEARCH_DEFINED_WINDOWS)
2548
-
2549
- HANDLE file_handle =
2550
- CreateFile(file_path, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
2551
- if (file_handle == INVALID_HANDLE_VALUE)
2552
- return result.failed("Opening file failed!");
2508
+ serialization_result_t result = file.open_if_not();
2509
+ if (!result)
2510
+ return result;
2553
2511
 
2554
- size_t file_length = GetFileSize(file_handle, 0);
2555
- HANDLE mapping_handle = CreateFileMapping(file_handle, 0, PAGE_READONLY, 0, 0, 0);
2556
- if (mapping_handle == 0) {
2557
- CloseHandle(file_handle);
2558
- return result.failed("Mapping file failed!");
2512
+ // Pull basic metadata
2513
+ index_serialized_header_t header;
2514
+ if (file.size() - offset < sizeof(header))
2515
+ return result.failed("File is corrupted and lacks a header");
2516
+ std::memcpy(&header, file.data() + offset, sizeof(header));
2517
+
2518
+ // Precompute offsets of every node, but before that we need to update the configs
2519
+ // This could have been done with `std::exclusive_scan`, but it's only available from C++17.
2520
+ using offsets_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
2521
+ buffer_gt<std::size_t, offsets_allocator_t> offsets(header.size);
2522
+ if (!offsets)
2523
+ return result.failed("Out of memory");
2524
+
2525
+ config_.connectivity = header.connectivity;
2526
+ config_.connectivity_base = header.connectivity_base;
2527
+ pre_ = precompute_(config_);
2528
+ misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
2529
+ offsets[0] = offset + sizeof(header) + sizeof(level_t) * header.size;
2530
+ for (std::size_t i = 1; i < header.size; ++i)
2531
+ offsets[i] = offsets[i - 1] + node_bytes_(levels[i - 1]);
2532
+
2533
+ std::size_t total_bytes = offsets[header.size - 1] + node_bytes_(levels[header.size - 1]);
2534
+ if (file.size() < total_bytes) {
2535
+ reset();
2536
+ return result.failed("File is corrupted and can't fit all the nodes");
2559
2537
  }
2560
2538
 
2561
- byte_t* file = (byte_t*)MapViewOfFile(mapping_handle, FILE_MAP_READ, 0, 0, file_length);
2562
- if (file == 0) {
2563
- CloseHandle(mapping_handle);
2564
- CloseHandle(file_handle);
2565
- return result.failed("View the map failed!");
2539
+ // Submit metadata and reserve memory
2540
+ index_limits_t limits;
2541
+ limits.members = header.size;
2542
+ if (!reserve(limits)) {
2543
+ reset();
2544
+ return result.failed("Out of memory");
2566
2545
  }
2567
- viewed_file_.file_handle = file_handle;
2568
- viewed_file_.mapping_handle = mapping_handle;
2569
- viewed_file_.ptr = file;
2570
- viewed_file_.length = file_length;
2571
- #else
2572
-
2573
- #if defined(USEARCH_DEFINED_LINUX)
2574
- int descriptor = open(file_path, O_RDONLY | O_NOATIME);
2575
- #else
2576
- int descriptor = open(file_path, O_RDONLY);
2577
- #endif
2578
-
2579
- // Estimate the file size
2580
- struct stat file_stat;
2581
- int fstat_status = fstat(descriptor, &file_stat);
2582
- if (fstat_status < 0) {
2583
- close(descriptor);
2584
- return result.failed(std::strerror(errno));
2546
+ nodes_count_ = header.size;
2547
+ max_level_ = static_cast<level_t>(header.max_level);
2548
+ entry_slot_ = static_cast<compressed_slot_t>(header.entry_slot);
2549
+
2550
+ // Rapidly address all the nodes
2551
+ for (std::size_t i = 0; i != header.size; ++i) {
2552
+ nodes_[i] = node_t{(byte_t*)file.data() + offsets[i]};
2553
+ progress(i, header.size);
2585
2554
  }
2555
+ viewed_file_ = std::move(file);
2556
+ return {};
2557
+ }
2586
2558
 
2587
- // Map the entire file
2588
- byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_PRIVATE, descriptor, 0);
2589
- if (file == MAP_FAILED) {
2590
- close(descriptor);
2591
- return result.failed(std::strerror(errno));
2592
- }
2593
- viewed_file_.file_descriptor = descriptor;
2594
- viewed_file_.ptr = file;
2595
- viewed_file_.length = file_stat.st_size;
2596
- #endif // Platform specific code
2559
+ #pragma endregion
2597
2560
 
2598
- // Read the header
2599
- {
2600
- file_head_t state{file};
2601
- if (state.bytes_per_label != sizeof(label_t)) {
2602
- reset_view_();
2603
- return result.failed("Incompatible label type!");
2604
- }
2605
- if (state.bytes_per_id != sizeof(id_t)) {
2606
- reset_view_();
2607
- return result.failed("Incompatible ID type!");
2608
- }
2561
+ /**
2562
+ * @brief Performs compaction on the whole HNSW index, purging some entries
2563
+ * and links to them, while also generating a more efficient mapping,
2564
+ * putting the more frequently used entries closer together.
2565
+ *
2566
+ *
2567
+ * Scans the whole collection, removing the links leading towards
2568
+ * banned entries. This essentially isolates some nodes from the rest
2569
+ * of the graph, while keeping their outgoing links, in case the node
2570
+ * is structurally relevant and has a crucial role in the index.
2571
+ * It won't reclaim the memory.
2572
+ *
2573
+ * @param[in] allow_member Predicate to mark nodes for isolation.
2574
+ * @param[in] executor Thread-pool to execute the job in parallel.
2575
+ * @param[in] progress Callback to report the execution progress.
2576
+ */
2577
+ template <typename values_at, typename metric_at, //
2578
+ typename slot_transition_at = dummy_label_to_label_mapping_t, //
2579
+ typename executor_at = dummy_executor_t, //
2580
+ typename progress_at = dummy_progress_t, //
2581
+ typename prefetch_at = dummy_prefetch_t>
2582
+ void compact( //
2583
+ values_at&& values, //
2584
+ metric_at&& metric, //
2585
+ slot_transition_at&& slot_transition, //
2609
2586
 
2610
- config_.connectivity = state.connectivity;
2611
- config_.vector_alignment = state.vector_alignment;
2612
- pre_ = precompute_(config_);
2587
+ executor_at&& executor = executor_at{}, //
2588
+ progress_at&& progress = progress_at{}, //
2589
+ prefetch_at&& prefetch = prefetch_at{}) noexcept {
2590
+
2591
+ // Export all the keys, slots, and levels.
2592
+ // Partition them with the predicate.
2593
+ // Sort the allowed entries in descending order of their level.
2594
+ // Create a new array mapping old slots to the new ones (INT_MAX for deleted items).
2595
+ struct slot_level_t {
2596
+ compressed_slot_t old_slot;
2597
+ compressed_slot_t cluster;
2598
+ level_t level;
2599
+ };
2600
+ using slot_level_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<slot_level_t>;
2601
+ buffer_gt<slot_level_t, slot_level_allocator_t> slots_and_levels(size());
2602
+
2603
+ // For every bottom level node, determine its parent cluster
2604
+ executor.execute_bulk(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
2605
+ context_t& context = contexts_[thread_idx];
2606
+ std::size_t cluster = search_for_one_( //
2607
+ values[citerator_at(old_slot)], //
2608
+ metric, prefetch, //
2609
+ entry_slot_, max_level_, 0, context);
2610
+ slots_and_levels[old_slot] = { //
2611
+ static_cast<compressed_slot_t>(old_slot), //
2612
+ static_cast<compressed_slot_t>(cluster), //
2613
+ node_at_(old_slot).level()};
2614
+ });
2613
2615
 
2614
- index_limits_t limits;
2615
- limits.members = state.size;
2616
- limits.threads_add = 0;
2617
- if (!reserve(limits))
2618
- return result.failed("Out of memory!");
2616
+ // Where the actual permutation happens:
2617
+ std::sort(slots_and_levels.begin(), slots_and_levels.end(), [](slot_level_t const& a, slot_level_t const& b) {
2618
+ return a.level == b.level ? a.cluster < b.cluster : a.level > b.level;
2619
+ });
2619
2620
 
2620
- size_ = state.size;
2621
- max_level_ = static_cast<level_t>(state.max_level);
2622
- entry_id_ = static_cast<id_t>(state.entry_idx);
2623
- }
2621
+ using size_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<std::size_t>;
2622
+ buffer_gt<std::size_t, size_allocator_t> old_slot_to_new(slots_and_levels.size());
2623
+ for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot)
2624
+ old_slot_to_new[slots_and_levels[new_slot].old_slot] = new_slot;
2624
2625
 
2625
- // Locate every node packed into file
2626
- std::size_t progress_bytes = sizeof(file_header_t);
2627
- std::size_t const size = size_;
2628
- for (std::size_t i = 0; i != size; ++i) {
2629
- byte_t* tape = (byte_t*)(file + progress_bytes);
2630
- dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
2631
- level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
2632
-
2633
- std::size_t node_bytes = node_bytes_(dim, level);
2634
- std::size_t node_vector_bytes = dim * sizeof(scalar_t);
2635
- nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
2636
- progress_bytes += node_bytes;
2637
- progress(i, size);
2638
- }
2626
+ // Erase all the incoming links
2627
+ buffer_gt<node_t, nodes_allocator_t> reordered_nodes(slots_and_levels.size());
2628
+ tape_allocator_t reordered_tape;
2639
2629
 
2640
- return {};
2641
- }
2630
+ for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
2631
+ std::size_t old_slot = slots_and_levels[new_slot].old_slot;
2632
+ node_t old_node = node_at_(old_slot);
2642
2633
 
2643
- #pragma endregion
2634
+ std::size_t node_bytes = node_bytes_(old_node.level());
2635
+ byte_t* new_data = (byte_t*)reordered_tape.allocate(node_bytes);
2636
+ node_t new_node{new_data};
2637
+ std::memcpy(new_data, old_node.tape(), node_bytes);
2644
2638
 
2645
- struct join_result_t {
2646
- error_t error{};
2647
- std::size_t intersection_size{};
2648
- std::size_t engagements{};
2649
- std::size_t cycles{};
2650
- std::size_t measurements{};
2639
+ for (level_t level = 0; level <= old_node.level(); ++level)
2640
+ for (misaligned_ref_gt<compressed_slot_t> neighbor : neighbors_(new_node, level))
2641
+ neighbor = static_cast<compressed_slot_t>(old_slot_to_new[compressed_slot_t(neighbor)]);
2651
2642
 
2652
- explicit operator bool() const noexcept { return !error; }
2653
- join_result_t failed(error_t message) noexcept {
2654
- error = std::move(message);
2655
- return std::move(*this);
2643
+ reordered_nodes[new_slot] = new_node;
2644
+
2645
+ progress(new_slot, slots_and_levels.size());
2656
2646
  }
2657
- };
2658
2647
 
2659
- /**
2660
- * @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
2661
- * to perform fast one-to-one matching between two large collections
2662
- * of vectors, using approximate nearest neighbors search.
2663
- *
2664
- * @param[inout] first_to_second Container to map ::first labels to ::second.
2665
- * @param[inout] second_to_first Container to map ::second labels to ::first.
2666
- * @param[in] executor Thread-pool to execute the job in parallel.
2667
- * @param[in] progress Callback to report the execution progress.
2668
- */
2669
- template < //
2670
- typename first_to_second_at = dummy_label_to_label_mapping_t, //
2671
- typename second_to_first_at = dummy_label_to_label_mapping_t, //
2672
- typename executor_at = dummy_executor_t, //
2673
- typename progress_at = dummy_progress_t //
2674
- >
2675
- static join_result_t join( //
2676
- index_gt const& first, index_gt const& second, //
2677
- join_config_t config = {}, //
2678
- first_to_second_at&& first_to_second = first_to_second_at{}, //
2679
- second_to_first_at&& second_to_first = second_to_first_at{}, //
2680
- executor_at&& executor = executor_at{}, //
2681
- progress_at&& progress = progress_at{}) noexcept {
2648
+ for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
2649
+ std::size_t old_slot = slots_and_levels[new_slot].old_slot;
2650
+ slot_transition(node_at_(old_slot).ckey(), //
2651
+ static_cast<compressed_slot_t>(old_slot), //
2652
+ static_cast<compressed_slot_t>(new_slot));
2653
+ }
2682
2654
 
2683
- if (second.size() < first.size())
2684
- return join_small_and_big_( //
2685
- second, first, //
2686
- config, //
2687
- std::forward<second_to_first_at>(second_to_first), //
2688
- std::forward<first_to_second_at>(first_to_second), //
2689
- std::forward<executor_at>(executor), //
2690
- std::forward<progress_at>(progress));
2691
- else
2692
- return join_small_and_big_( //
2693
- first, second, //
2694
- config, //
2695
- std::forward<first_to_second_at>(first_to_second), //
2696
- std::forward<second_to_first_at>(second_to_first), //
2697
- std::forward<executor_at>(executor), //
2698
- std::forward<progress_at>(progress));
2655
+ nodes_ = std::move(reordered_nodes);
2656
+ tape_allocator_ = std::move(reordered_tape);
2657
+ entry_slot_ = old_slot_to_new[entry_slot_];
2699
2658
  }
2700
2659
 
2701
2660
  /**
@@ -2720,293 +2679,80 @@ class index_gt {
2720
2679
  progress_at&& progress = progress_at{}) noexcept {
2721
2680
 
2722
2681
  // Erase all the incoming links
2723
- executor.execute_bulk(size(), [&](std::size_t, std::size_t node_idx) {
2724
- node_t node = node_with_id_(node_idx);
2682
+ std::size_t nodes_count = size();
2683
+ executor.execute_bulk(nodes_count, [&](std::size_t, std::size_t node_idx) {
2684
+ node_t node = node_at_(node_idx);
2725
2685
  for (level_t level = 0; level <= node.level(); ++level) {
2726
2686
  neighbors_ref_t neighbors = neighbors_(node, level);
2727
2687
  std::size_t old_size = neighbors.size();
2728
2688
  neighbors.clear();
2729
2689
  for (std::size_t i = 0; i != old_size; ++i) {
2730
- id_t neighbor_id = neighbors[i];
2731
- node_t neighbor = node_with_id_(neighbor_id);
2732
- if (allow_member(member_cref_t{neighbor.label(), neighbor.vector_view(), neighbor_id}))
2733
- neighbors.push_back(neighbor_id);
2690
+ compressed_slot_t neighbor_slot = neighbors[i];
2691
+ node_t neighbor = node_at_(neighbor_slot);
2692
+ if (allow_member(member_cref_t{neighbor.ckey(), neighbor_slot}))
2693
+ neighbors.push_back(neighbor_slot);
2734
2694
  }
2735
2695
  }
2696
+ progress(node_idx, nodes_count);
2736
2697
  });
2737
2698
  }
2738
2699
 
2739
2700
  private:
2740
- template <typename first_to_second_at, typename second_to_first_at, typename executor_at, typename progress_at>
2741
- static join_result_t join_small_and_big_( //
2742
- index_gt const& men, index_gt const& women, //
2743
- join_config_t config, //
2744
- first_to_second_at&& man_to_woman, //
2745
- second_to_first_at&& woman_to_man, //
2746
- executor_at&& executor, //
2747
- progress_at&& progress) noexcept {
2748
-
2749
- join_result_t result;
2750
-
2751
- // Sanity checks and argument validation:
2752
- if (&men == &women)
2753
- return result.failed("Can't join with itself, consider copying");
2754
- if (config.max_proposals == 0)
2755
- config.max_proposals = std::log(men.size()) + executor.size();
2756
- config.max_proposals = (std::min)(men.size(), config.max_proposals);
2757
-
2758
- using proposals_count_t = std::uint16_t;
2759
- using ids_allocator_t = typename allocator_traits_t::template rebind_alloc<id_t>;
2760
- dynamic_allocator_t& alloc = men.dynamic_allocator_;
2761
-
2762
- // Create an atomic queue, as a ring structure, from/to which
2763
- // free men will be added/pulled.
2764
- std::mutex free_men_mutex{};
2765
- ring_gt<id_t, ids_allocator_t> free_men;
2766
- free_men.reserve(men.size());
2767
- for (std::size_t i = 0; i != men.size(); ++i)
2768
- free_men.push(static_cast<id_t>(i));
2769
-
2770
- // We are gonna need some temporary memory.
2771
- proposals_count_t* proposal_counts = (proposals_count_t*)alloc.allocate(sizeof(proposals_count_t) * men.size());
2772
- id_t* man_to_woman_ids = (id_t*)alloc.allocate(sizeof(id_t) * men.size());
2773
- id_t* woman_to_man_ids = (id_t*)alloc.allocate(sizeof(id_t) * women.size());
2774
- if (!proposal_counts || !man_to_woman_ids || !woman_to_man_ids)
2775
- return result.failed("Can't temporary mappings");
2776
-
2777
- id_t missing_id;
2778
- std::memset((void*)&missing_id, 0xFF, sizeof(id_t));
2779
- std::memset((void*)man_to_woman_ids, 0xFF, sizeof(id_t) * men.size());
2780
- std::memset((void*)woman_to_man_ids, 0xFF, sizeof(id_t) * women.size());
2781
- std::memset(proposal_counts, 0, sizeof(proposals_count_t) * men.size());
2782
-
2783
- // Define locks, to limit concurrent accesses to `man_to_woman_ids` and `woman_to_man_ids`.
2784
- visits_bitset_t men_locks, women_locks;
2785
- if (!men_locks.resize(men.size()) || !women_locks.resize(women.size()))
2786
- return result.failed("Can't allocate locks");
2787
-
2788
- // Accumulate statistics from all the contexts,
2789
- // to have a baseline to compare with, by the time the `join` is finished.
2790
- std::size_t old_measurements{};
2791
- std::size_t old_cycles{};
2792
- for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
2793
- old_measurements += women.contexts_[thread_idx].measurements_count;
2794
- old_cycles += women.contexts_[thread_idx].iteration_cycles;
2795
- }
2796
- std::atomic<std::size_t> rounds{0};
2797
- std::atomic<std::size_t> engagements{0};
2798
-
2799
- // Concurrently process all the men
2800
- executor.execute_bulk([&](std::size_t thread_idx) {
2801
- context_t& context = women.contexts_[thread_idx];
2802
- search_config_t search_config;
2803
- search_config.expansion = config.expansion;
2804
- search_config.exact = config.exact;
2805
- search_config.thread = thread_idx;
2806
- id_t free_man_id;
2807
-
2808
- // While there exist a free man who still has a woman to propose to.
2809
- while (true) {
2810
- std::size_t passed_rounds = 0;
2811
- std::size_t total_rounds = 0;
2812
- {
2813
- std::unique_lock<std::mutex> pop_lock(free_men_mutex);
2814
- if (!free_men.try_pop(free_man_id))
2815
- // Primary exit path, we have exhausted the list of candidates
2816
- break;
2817
- passed_rounds = ++rounds;
2818
- total_rounds = passed_rounds + free_men.size();
2819
- }
2820
- progress(passed_rounds, total_rounds);
2821
- while (men_locks.atomic_set(free_man_id))
2822
- ;
2823
-
2824
- node_t free_man = men.node_with_id_(free_man_id);
2825
- proposals_count_t& free_man_proposals = proposal_counts[free_man_id];
2826
- if (free_man_proposals >= config.max_proposals)
2827
- continue;
2828
-
2829
- // Find the closest woman, to whom this man hasn't proposed yet.
2830
- free_man_proposals++;
2831
- search_result_t candidates = women.search(free_man.vector_view(), free_man_proposals, search_config);
2832
- if (!candidates) {
2833
- // TODO:
2834
- }
2835
-
2836
- match_t match = candidates.back();
2837
- member_cref_t woman = match.member;
2838
- while (women_locks.atomic_set(woman.id))
2839
- ;
2840
-
2841
- id_t husband_id = woman_to_man_ids[woman.id];
2842
- bool woman_is_free = husband_id == missing_id;
2843
- if (woman_is_free) {
2844
- // Engagement
2845
- man_to_woman_ids[free_man_id] = woman.id;
2846
- woman_to_man_ids[woman.id] = free_man_id;
2847
- engagements++;
2848
- } else {
2849
- distance_t distance_from_husband = context.measure(woman.vector, men.node_with_id_(husband_id));
2850
- distance_t distance_from_candidate = match.distance;
2851
- if (distance_from_husband > distance_from_candidate) {
2852
- // Break-up
2853
- while (men_locks.atomic_set(husband_id))
2854
- ;
2855
- man_to_woman_ids[husband_id] = missing_id;
2856
- men_locks.atomic_reset(husband_id);
2857
-
2858
- // New Engagement
2859
- man_to_woman_ids[free_man_id] = woman.id;
2860
- woman_to_man_ids[woman.id] = free_man_id;
2861
- engagements++;
2862
-
2863
- std::unique_lock<std::mutex> push_lock(free_men_mutex);
2864
- free_men.push(husband_id);
2865
- } else {
2866
- std::unique_lock<std::mutex> push_lock(free_men_mutex);
2867
- free_men.push(free_man_id);
2868
- }
2869
- }
2870
-
2871
- men_locks.atomic_reset(free_man_id);
2872
- women_locks.atomic_reset(woman.id);
2873
- }
2874
- });
2875
-
2876
- // Export the IDs into labels:
2877
- std::size_t intersection_size = 0;
2878
- for (std::size_t i = 0; i != men.size(); ++i) {
2879
- id_t woman_id = man_to_woman_ids[i];
2880
- if (woman_id != missing_id) {
2881
- label_t man = men.node_with_id_(i).label();
2882
- label_t woman = women.node_with_id_(woman_id).label();
2883
- man_to_woman[man] = woman;
2884
- woman_to_man[woman] = man;
2885
- intersection_size++;
2886
- }
2887
- }
2888
-
2889
- // Deallocate memory
2890
- alloc.deallocate((byte_t*)proposal_counts, sizeof(proposals_count_t) * men.size());
2891
- alloc.deallocate((byte_t*)man_to_woman_ids, sizeof(id_t) * men.size());
2892
- alloc.deallocate((byte_t*)woman_to_man_ids, sizeof(id_t) * women.size());
2893
-
2894
- // Export stats
2895
- result.engagements = engagements;
2896
- result.intersection_size = intersection_size;
2897
- for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
2898
- result.measurements += women.contexts_[thread_idx].measurements_count;
2899
- result.cycles += women.contexts_[thread_idx].iteration_cycles;
2900
- }
2901
- result.measurements -= old_measurements;
2902
- result.cycles -= old_cycles;
2903
- return result;
2904
- }
2905
-
2906
- void reset_view_() noexcept {
2907
- if (!viewed_file_)
2908
- return;
2909
- #if defined(USEARCH_DEFINED_WINDOWS)
2910
- UnmapViewOfFile(viewed_file_.ptr);
2911
- CloseHandle(viewed_file_.mapping_handle);
2912
- CloseHandle(viewed_file_.file_handle);
2913
-
2914
- #else
2915
- munmap(viewed_file_.ptr, viewed_file_.length);
2916
- close(viewed_file_.file_descriptor);
2917
- #endif
2918
- viewed_file_ = {};
2919
- }
2920
-
2921
2701
  inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
2922
2702
  precomputed_constants_t pre;
2923
- pre.connectivity_max_base = config.connectivity * base_level_multiple_();
2924
2703
  pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
2925
- pre.neighbors_bytes = config.connectivity * sizeof(id_t) + sizeof(neighbors_count_t);
2926
- pre.neighbors_base_bytes = pre.connectivity_max_base * sizeof(id_t) + sizeof(neighbors_count_t);
2704
+ pre.neighbors_bytes = config.connectivity * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
2705
+ pre.neighbors_base_bytes = config.connectivity_base * sizeof(compressed_slot_t) + sizeof(neighbors_count_t);
2927
2706
  return pre;
2928
2707
  }
2929
2708
 
2930
- inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
2931
- inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
2932
- return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
2933
- }
2934
-
2935
2709
  using span_bytes_t = span_gt<byte_t>;
2936
- struct node_bytes_split_t {
2937
- span_bytes_t tape{};
2938
- span_bytes_t vector{};
2939
-
2940
- node_bytes_split_t() {}
2941
- node_bytes_split_t(span_bytes_t tape, span_bytes_t vector) noexcept : tape(tape), vector(vector) {}
2942
2710
 
2943
- std::size_t memory_usage() const noexcept { return tape.size() + vector.size(); }
2944
- bool colocated() const noexcept { return tape.end() == vector.begin(); }
2945
- operator node_t() const noexcept { return node_t{tape.begin(), reinterpret_cast<scalar_t*>(vector.begin())}; }
2946
- };
2947
-
2948
- inline node_bytes_split_t node_bytes_split_(node_t node) const noexcept {
2949
- std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
2950
- std::size_t bytes_in_tape = node_head_bytes_() + levels_bytes;
2951
- return {{node.tape(), bytes_in_tape}, {(byte_t*)node.vector(), node_vector_bytes_(node)}};
2711
+ inline span_bytes_t node_bytes_(node_t node) const noexcept { return {node.tape(), node_bytes_(node.level())}; }
2712
+ inline std::size_t node_bytes_(level_t level) const noexcept {
2713
+ return node_head_bytes_() + node_neighbors_bytes_(level);
2714
+ }
2715
+ inline std::size_t node_neighbors_bytes_(node_t node) const noexcept { return node_neighbors_bytes_(node.level()); }
2716
+ inline std::size_t node_neighbors_bytes_(level_t level) const noexcept {
2717
+ return pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
2952
2718
  }
2953
2719
 
2954
- inline std::size_t node_vector_bytes_(dim_t dim) const noexcept { return dim * sizeof(scalar_t); }
2955
- inline std::size_t node_vector_bytes_(node_t node) const noexcept { return node_vector_bytes_(node.dim()); }
2956
-
2957
- node_bytes_split_t node_malloc_(dim_t dims_to_store, level_t level) noexcept {
2958
-
2959
- std::size_t vector_bytes = node_vector_bytes_(dims_to_store);
2960
- std::size_t node_bytes = node_bytes_(dims_to_store, level);
2961
- std::size_t non_vector_bytes = node_bytes - vector_bytes;
2962
-
2720
+ span_bytes_t node_malloc_(level_t level) noexcept {
2721
+ std::size_t node_bytes = node_bytes_(level);
2963
2722
  byte_t* data = (byte_t*)tape_allocator_.allocate(node_bytes);
2964
- if (!data)
2965
- return node_bytes_split_t{};
2966
- return {{data, non_vector_bytes}, {data + non_vector_bytes, vector_bytes}};
2723
+ return data ? span_bytes_t{data, node_bytes} : span_bytes_t{};
2967
2724
  }
2968
2725
 
2969
- node_t node_make_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
2970
- node_bytes_split_t node_bytes = node_malloc_(vector.size() * store_vector, level);
2971
- if (store_vector) {
2972
- std::memset(node_bytes.tape.data(), 0, node_bytes.tape.size());
2973
- std::memcpy(node_bytes.vector.data(), vector.data(), node_bytes.vector.size());
2974
- } else {
2975
- std::memset(node_bytes.tape.data(), 0, node_bytes.memory_usage());
2976
- }
2977
- node_t node = node_bytes;
2978
- node.label(label);
2979
- node.dim(static_cast<dim_t>(vector.size()));
2726
+ node_t node_make_(key_t key, level_t level) noexcept {
2727
+ span_bytes_t node_bytes = node_malloc_(level);
2728
+ if (!node_bytes)
2729
+ return {};
2730
+
2731
+ std::memset(node_bytes.data(), 0, node_bytes.size());
2732
+ node_t node{(byte_t*)node_bytes.data()};
2733
+ node.key(key);
2980
2734
  node.level(level);
2981
2735
  return node;
2982
2736
  }
2983
2737
 
2984
- node_t node_make_copy_(node_bytes_split_t old_bytes) noexcept {
2985
- if (old_bytes.colocated()) {
2986
- byte_t* data = (byte_t*)tape_allocator_.allocate(old_bytes.memory_usage());
2987
- std::memcpy(data, old_bytes.tape.data(), old_bytes.memory_usage());
2988
- return node_t{data, reinterpret_cast<scalar_t*>(data + old_bytes.tape.size())};
2989
- } else {
2990
- node_t old_node = old_bytes;
2991
- node_bytes_split_t node_bytes = node_malloc_(old_node.vector_view().size(), old_node.level());
2992
- std::memcpy(node_bytes.tape.data(), old_bytes.tape.data(), old_bytes.tape.size());
2993
- std::memcpy(node_bytes.vector.data(), old_bytes.vector.data(), old_bytes.vector.size());
2994
- return node_bytes;
2995
- }
2738
+ node_t node_make_copy_(span_bytes_t old_bytes) noexcept {
2739
+ byte_t* data = (byte_t*)tape_allocator_.allocate(old_bytes.size());
2740
+ if (!data)
2741
+ return {};
2742
+ std::memcpy(data, old_bytes.data(), old_bytes.size());
2743
+ return node_t{data};
2996
2744
  }
2997
2745
 
2998
- void node_free_(std::size_t id) noexcept {
2999
-
2746
+ void node_free_(std::size_t idx) noexcept {
3000
2747
  if (viewed_file_)
3001
2748
  return;
3002
2749
 
3003
- node_t& node = nodes_[id];
3004
- std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_bytes_split_(node).colocated();
3005
- tape_allocator_.deallocate(node.tape(), node_bytes);
2750
+ node_t& node = nodes_[idx];
2751
+ tape_allocator_.deallocate(node.tape(), node_bytes_(node).size());
3006
2752
  node = node_t{};
3007
2753
  }
3008
2754
 
3009
- inline node_t node_with_id_(std::size_t idx) const noexcept { return nodes_[idx]; }
2755
+ inline node_t node_at_(std::size_t idx) const noexcept { return nodes_[idx]; }
3010
2756
  inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
3011
2757
 
3012
2758
  inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
@@ -3019,91 +2765,102 @@ class index_gt {
3019
2765
 
3020
2766
  struct node_lock_t {
3021
2767
  visits_bitset_t& bitset;
3022
- std::size_t idx;
3023
-
3024
- inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
2768
+ std::size_t slot;
2769
+ inline ~node_lock_t() noexcept { bitset.atomic_reset(slot); }
3025
2770
  };
3026
2771
 
3027
- inline node_lock_t node_lock_(std::size_t idx) const noexcept {
3028
- while (nodes_mutexes_.atomic_set(idx))
2772
+ inline node_lock_t node_lock_(std::size_t slot) const noexcept {
2773
+ while (nodes_mutexes_.atomic_set(slot))
3029
2774
  ;
3030
- return {nodes_mutexes_, idx};
2775
+ return {nodes_mutexes_, slot};
3031
2776
  }
3032
2777
 
3033
- void connect_node_across_levels_( //
3034
- id_t node_id, vector_view_t vector, //
3035
- id_t entry_id, level_t max_level, level_t target_level, //
3036
- add_config_t const& config, context_t& context) usearch_noexcept_m {
2778
+ template <typename value_at, typename metric_at, typename prefetch_at>
2779
+ void connect_node_across_levels_( //
2780
+ value_at&& value, metric_at&& metric, prefetch_at&& prefetch, //
2781
+ std::size_t node_slot, std::size_t entry_slot, level_t max_level, level_t target_level, //
2782
+ index_update_config_t const& config, context_t& context) usearch_noexcept_m {
3037
2783
 
3038
2784
  // Go down the level, tracking only the closest match
3039
- id_t closest_id = search_for_one_(entry_id, vector, max_level, target_level, context);
2785
+ std::size_t closest_slot = search_for_one_( //
2786
+ value, metric, prefetch, //
2787
+ entry_slot, max_level, target_level, context);
3040
2788
 
3041
2789
  // From `target_level` down perform proper extensive search
3042
2790
  for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
3043
2791
  // TODO: Handle out of memory conditions
3044
- search_to_insert_(closest_id, vector, level, config.expansion, context);
3045
- closest_id = connect_new_node_(node_id, level, context);
3046
- reconnect_neighbor_nodes_(node_id, level, context);
2792
+ search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
2793
+ closest_slot = connect_new_node_(metric, node_slot, level, context);
2794
+ reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
3047
2795
  }
3048
2796
  }
3049
2797
 
3050
- id_t connect_new_node_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
2798
+ template <typename metric_at>
2799
+ std::size_t connect_new_node_( //
2800
+ metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
3051
2801
 
3052
- node_t new_node = node_with_id_(new_id);
2802
+ node_t new_node = node_at_(new_slot);
3053
2803
  top_candidates_t& top = context.top_candidates;
3054
2804
 
3055
- // Outgoing links from `new_id`:
2805
+ // Outgoing links from `new_slot`:
3056
2806
  neighbors_ref_t new_neighbors = neighbors_(new_node, level);
3057
2807
  {
3058
2808
  usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
3059
- candidates_view_t top_view = refine_(top, config_.connectivity, context);
2809
+ candidates_view_t top_view = refine_(metric, config_.connectivity, top, context);
3060
2810
 
3061
2811
  for (std::size_t idx = 0; idx != top_view.size(); idx++) {
3062
2812
  usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
3063
- usearch_assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
3064
- new_neighbors.push_back(top_view[idx].id);
2813
+ usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
2814
+ new_neighbors.push_back(top_view[idx].slot);
3065
2815
  }
3066
2816
  }
3067
2817
 
3068
2818
  return new_neighbors[0];
3069
2819
  }
3070
2820
 
3071
- void reconnect_neighbor_nodes_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
2821
+ template <typename value_at, typename metric_at>
2822
+ void reconnect_neighbor_nodes_( //
2823
+ metric_at&& metric, std::size_t new_slot, value_at&& value, level_t level,
2824
+ context_t& context) usearch_noexcept_m {
3072
2825
 
3073
- node_t new_node = node_with_id_(new_id);
2826
+ node_t new_node = node_at_(new_slot);
3074
2827
  top_candidates_t& top = context.top_candidates;
3075
2828
  neighbors_ref_t new_neighbors = neighbors_(new_node, level);
3076
2829
 
3077
2830
  // Reverse links from the neighbors:
3078
- std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
3079
- for (id_t close_id : new_neighbors) {
3080
- node_t close_node = node_with_id_(close_id);
3081
- node_lock_t close_lock = node_lock_(close_id);
2831
+ std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
2832
+ for (compressed_slot_t close_slot : new_neighbors) {
2833
+ if (close_slot == new_slot)
2834
+ continue;
2835
+ node_lock_t close_lock = node_lock_(close_slot);
2836
+ node_t close_node = node_at_(close_slot);
3082
2837
 
3083
2838
  neighbors_ref_t close_header = neighbors_(close_node, level);
3084
2839
  usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
3085
- usearch_assert_m(close_id != new_id, "Self-loops are impossible");
2840
+ usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
3086
2841
  usearch_assert_m(level <= close_node.level(), "Linking to missing level");
3087
2842
 
3088
- // If `new_id` is already present in the neighboring connections of `close_id`
2843
+ // If `new_slot` is already present in the neighboring connections of `close_slot`
3089
2844
  // then no need to modify any connections or run the heuristics.
3090
2845
  if (close_header.size() < connectivity_max) {
3091
- close_header.push_back(new_id);
2846
+ close_header.push_back(static_cast<compressed_slot_t>(new_slot));
3092
2847
  continue;
3093
2848
  }
3094
2849
 
3095
2850
  // To fit a new connection we need to drop an existing one.
3096
2851
  top.clear();
3097
2852
  usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
3098
- top.insert_reserved({context.measure(new_node, close_node), new_id});
3099
- for (id_t successor_id : close_header)
3100
- top.insert_reserved({context.measure(node_with_id_(successor_id), close_node), successor_id});
2853
+ top.insert_reserved(
2854
+ {context.measure(value, citerator_at(close_slot), metric), static_cast<compressed_slot_t>(new_slot)});
2855
+ for (compressed_slot_t successor_slot : close_header)
2856
+ top.insert_reserved(
2857
+ {context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
3101
2858
 
3102
2859
  // Export the results:
3103
2860
  close_header.clear();
3104
- candidates_view_t top_view = refine_(top, connectivity_max, context);
2861
+ candidates_view_t top_view = refine_(metric, connectivity_max, top, context);
3105
2862
  for (std::size_t idx = 0; idx != top_view.size(); idx++)
3106
- close_header.push_back(top_view[idx].id);
2863
+ close_header.push_back(top_view[idx].slot);
3107
2864
  }
3108
2865
  }
3109
2866
 
@@ -3113,31 +2870,105 @@ class index_gt {
3113
2870
  return (level_t)r;
3114
2871
  }
3115
2872
 
3116
- id_t search_for_one_( //
3117
- id_t closest_id, vector_view_t query, //
3118
- level_t begin_level, level_t end_level, //
3119
- context_t& context) const noexcept {
2873
+ struct candidates_range_t;
2874
+ class candidates_iterator_t {
2875
+ friend struct candidates_range_t;
2876
+
2877
+ index_gt const& index_;
2878
+ neighbors_ref_t neighbors_;
2879
+ visits_bitset_t& visits_;
2880
+ std::size_t current_;
2881
+
2882
+ candidates_iterator_t& skip_missing() noexcept {
2883
+ while (current_ != neighbors_.size()) {
2884
+ compressed_slot_t neighbor_slot = neighbors_[current_];
2885
+ if (visits_.test(neighbor_slot))
2886
+ current_++;
2887
+ else
2888
+ break;
2889
+ }
2890
+ return *this;
2891
+ }
2892
+
2893
+ public:
2894
+ using element_t = compressed_slot_t;
2895
+ using iterator_category = std::forward_iterator_tag;
2896
+ using value_type = element_t;
2897
+ using difference_type = std::ptrdiff_t;
2898
+ using pointer = misaligned_ptr_gt<element_t>;
2899
+ using reference = misaligned_ref_gt<element_t>;
2900
+
2901
+ reference operator*() const noexcept { return slot(); }
2902
+ candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_bitset_t& visits,
2903
+ std::size_t progress) noexcept
2904
+ : index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
2905
+ candidates_iterator_t operator++(int) noexcept {
2906
+ return candidates_iterator_t(index_, visits_, neighbors_, current_ + 1).skip_missing();
2907
+ }
2908
+ candidates_iterator_t& operator++() noexcept {
2909
+ ++current_;
2910
+ skip_missing();
2911
+ return *this;
2912
+ }
2913
+ bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
2914
+ bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
2915
+
2916
+ key_t key() const noexcept { return index_->node_at_(slot()).key(); }
2917
+ compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
2918
+ friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
2919
+ friend inline key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
2920
+ };
2921
+
2922
+ struct candidates_range_t {
2923
+ index_gt const& index;
2924
+ neighbors_ref_t neighbors;
2925
+ visits_bitset_t& visits;
2926
+
2927
+ candidates_iterator_t begin() const noexcept {
2928
+ return candidates_iterator_t{index, neighbors, visits, 0}.skip_missing();
2929
+ }
2930
+ candidates_iterator_t end() const noexcept { return {index, neighbors, visits, neighbors.size()}; }
2931
+ };
2932
+
2933
+ template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
2934
+ std::size_t search_for_one_( //
2935
+ value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
2936
+ std::size_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
2937
+
2938
+ visits_bitset_t& visits = context.visits;
2939
+ visits.clear();
3120
2940
 
3121
- distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
2941
+ // Optional prefetching
2942
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
2943
+ prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
2944
+
2945
+ distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
3122
2946
  for (level_t level = begin_level; level > end_level; --level) {
3123
2947
  bool changed;
3124
2948
  do {
3125
2949
  changed = false;
3126
- node_t closest_node = node_with_id_(closest_id);
3127
- node_lock_t closest_lock = node_lock_(closest_id);
3128
- neighbors_ref_t closest_neighbors = neighbors_non_base_(closest_node, level);
3129
- for (id_t candidate_id : closest_neighbors) {
3130
- distance_t candidate_dist = context.measure(query, node_with_id_(candidate_id));
2950
+ node_lock_t closest_lock = node_lock_(closest_slot);
2951
+ neighbors_ref_t closest_neighbors = neighbors_non_base_(node_at_(closest_slot), level);
2952
+
2953
+ // Optional prefetching
2954
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
2955
+ candidates_range_t missing_candidates{*this, closest_neighbors, visits};
2956
+ prefetch(missing_candidates.begin(), missing_candidates.end());
2957
+ }
2958
+
2959
+ // Actual traversal
2960
+ for (compressed_slot_t candidate_slot : closest_neighbors) {
2961
+ distance_t candidate_dist = context.measure(query, citerator_at(candidate_slot), metric);
3131
2962
  if (candidate_dist < closest_dist) {
3132
2963
  closest_dist = candidate_dist;
3133
- closest_id = candidate_id;
2964
+ closest_slot = candidate_slot;
3134
2965
  changed = true;
3135
2966
  }
3136
2967
  }
3137
2968
  context.iteration_cycles++;
3138
2969
  } while (changed);
3139
2970
  }
3140
- return closest_id;
2971
+ return closest_slot;
3141
2972
  }
3142
2973
 
3143
2974
  /**
@@ -3145,8 +2976,11 @@ class index_gt {
3145
2976
  * Locks the nodes in the process, assuming other threads are updating neighbors lists.
3146
2977
  * @return `true` if procedure succeeded, `false` if run out of memory.
3147
2978
  */
3148
- bool search_to_insert_( //
3149
- id_t start_id, vector_view_t query, level_t level, std::size_t top_limit, context_t& context) noexcept {
2979
+ template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
2980
+ bool search_to_insert_( //
2981
+ value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
2982
+ std::size_t start_slot, std::size_t new_slot, level_t level, std::size_t top_limit,
2983
+ context_t& context) noexcept {
3150
2984
 
3151
2985
  visits_bitset_t& visits = context.visits;
3152
2986
  next_candidates_t& next = context.next_candidates; // pop min, push
@@ -3156,10 +2990,14 @@ class index_gt {
3156
2990
  next.clear();
3157
2991
  top.clear();
3158
2992
 
3159
- distance_t radius = context.measure(query, node_with_id_(start_id));
3160
- next.insert_reserved({-radius, start_id});
3161
- top.insert_reserved({radius, start_id});
3162
- visits.set(start_id);
2993
+ // Optional prefetching
2994
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
2995
+ prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
2996
+
2997
+ distance_t radius = context.measure(query, citerator_at(start_slot), metric);
2998
+ next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
2999
+ top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
3000
+ visits.set(start_slot);
3163
3001
 
3164
3002
  while (!next.empty()) {
3165
3003
 
@@ -3170,24 +3008,30 @@ class index_gt {
3170
3008
  next.pop();
3171
3009
  context.iteration_cycles++;
3172
3010
 
3173
- id_t candidate_id = candidacy.id;
3174
- node_t candidate_ref = node_with_id_(candidate_id);
3175
- node_lock_t candidate_lock = node_lock_(candidate_id);
3011
+ compressed_slot_t candidate_slot = candidacy.slot;
3012
+ if (new_slot == candidate_slot)
3013
+ continue;
3014
+ node_t candidate_ref = node_at_(candidate_slot);
3015
+ node_lock_t candidate_lock = node_lock_(candidate_slot);
3176
3016
  neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
3177
3017
 
3178
- prefetch_neighbors_(candidate_neighbors, visits);
3179
- for (id_t successor_id : candidate_neighbors) {
3180
- if (visits.test(successor_id))
3181
- continue;
3018
+ // Optional prefetching
3019
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
3020
+ candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
3021
+ prefetch(missing_candidates.begin(), missing_candidates.end());
3022
+ }
3182
3023
 
3183
- visits.set(successor_id);
3184
- distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
3024
+ for (compressed_slot_t successor_slot : candidate_neighbors) {
3025
+ if (visits.set(successor_slot))
3026
+ continue;
3185
3027
 
3028
+ // node_lock_t successor_lock = node_lock_(successor_slot);
3029
+ distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
3186
3030
  if (top.size() < top_limit || successor_dist < radius) {
3187
3031
  // This can substantially grow our priority queue:
3188
- next.insert({-successor_dist, successor_id});
3032
+ next.insert({-successor_dist, successor_slot});
3189
3033
  // This will automatically evict poor matches:
3190
- top.insert({successor_dist, successor_id}, top_limit);
3034
+ top.insert({successor_dist, successor_slot}, top_limit);
3191
3035
  radius = top.top().distance;
3192
3036
  }
3193
3037
  }
@@ -3200,10 +3044,10 @@ class index_gt {
3200
3044
  * Doesn't lock any nodes, assuming read-only simultaneous access.
3201
3045
  * @return `true` if procedure succeeded, `false` if run out of memory.
3202
3046
  */
3203
- template <typename predicate_at>
3204
- bool search_to_find_in_base_( //
3205
- id_t start_id, vector_view_t query, std::size_t expansion, context_t& context,
3206
- predicate_at&& predicate) const noexcept {
3047
+ template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
3048
+ bool search_to_find_in_base_( //
3049
+ value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
3050
+ std::size_t start_slot, std::size_t expansion, context_t& context) const noexcept {
3207
3051
 
3208
3052
  visits_bitset_t& visits = context.visits;
3209
3053
  next_candidates_t& next = context.next_candidates; // pop min, push
@@ -3214,10 +3058,14 @@ class index_gt {
3214
3058
  next.clear();
3215
3059
  top.clear();
3216
3060
 
3217
- distance_t radius = context.measure(query, node_with_id_(start_id));
3218
- next.insert_reserved({-radius, start_id});
3219
- top.insert_reserved({radius, start_id});
3220
- visits.set(start_id);
3061
+ // Optional prefetching
3062
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value)
3063
+ prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
3064
+
3065
+ distance_t radius = context.measure(query, citerator_at(start_slot), metric);
3066
+ next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
3067
+ top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
3068
+ visits.set(start_slot);
3221
3069
 
3222
3070
  while (!next.empty()) {
3223
3071
 
@@ -3228,28 +3076,29 @@ class index_gt {
3228
3076
  next.pop();
3229
3077
  context.iteration_cycles++;
3230
3078
 
3231
- id_t candidate_id = candidate.id;
3232
- neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
3079
+ neighbors_ref_t candidate_neighbors = neighbors_base_(node_at_(candidate.slot));
3233
3080
 
3234
- prefetch_neighbors_(candidate_neighbors, visits);
3235
- for (id_t successor_id : candidate_neighbors) {
3236
- if (visits.test(successor_id))
3237
- continue;
3081
+ // Optional prefetching
3082
+ if (!std::is_same<prefetch_at, dummy_prefetch_t>::value) {
3083
+ candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
3084
+ prefetch(missing_candidates.begin(), missing_candidates.end());
3085
+ }
3238
3086
 
3239
- visits.set(successor_id);
3240
- node_t successor = node_with_id_(successor_id);
3241
- distance_t successor_dist = context.measure(query, successor);
3087
+ for (compressed_slot_t successor_slot : candidate_neighbors) {
3088
+ if (visits.set(successor_slot))
3089
+ continue;
3242
3090
 
3091
+ distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
3243
3092
  if (top.size() < top_limit || successor_dist < radius) {
3244
3093
  // This can substantially grow our priority queue:
3245
- next.insert({-successor_dist, successor_id});
3246
- if (predicate( //
3247
- match_t{member_cref_t{successor.label(), successor.vector_view(), successor_id},
3248
- successor_dist})) {
3249
- // This will automatically evict poor matches:
3250
- top.insert({successor_dist, successor_id}, top_limit);
3251
- radius = top.top().distance;
3252
- }
3094
+ next.insert({-successor_dist, successor_slot});
3095
+ if (!is_dummy<predicate_at>())
3096
+ if (!predicate(member_cref_t{node_at_(successor_slot).ckey(), successor_slot}))
3097
+ continue;
3098
+
3099
+ // This will automatically evict poor matches:
3100
+ top.insert({successor_dist, successor_slot}, top_limit);
3101
+ radius = top.top().distance;
3253
3102
  }
3254
3103
  }
3255
3104
  }
@@ -3258,33 +3107,35 @@ class index_gt {
3258
3107
  }
3259
3108
 
3260
3109
  /**
3261
- * @brief Iterates through all managed vectors, without actually touching the index.
3110
+ * @brief Iterates through all members, without actually touching the index.
3262
3111
  */
3263
- template <typename predicate_at>
3264
- void search_exact_( //
3265
- vector_view_t query, std::size_t count, context_t& context, //
3266
- predicate_at&& predicate) const noexcept {
3112
+ template <typename value_at, typename metric_at, typename predicate_at>
3113
+ void search_exact_( //
3114
+ value_at&& query, metric_at&& metric, predicate_at&& predicate, //
3115
+ std::size_t count, context_t& context) const noexcept {
3267
3116
 
3268
3117
  top_candidates_t& top = context.top_candidates;
3269
3118
  top.clear();
3270
3119
  top.reserve(count);
3271
3120
  for (std::size_t i = 0; i != size(); ++i) {
3272
- id_t id = static_cast<id_t>(i);
3273
- node_t node = node_with_id_(i);
3274
- distance_t distance = context.measure(query, node);
3275
- if (predicate(match_t{member_cref_t{node.label(), node.vector_view(), id}, distance}))
3276
- top.insert(candidate_t{distance, id}, count);
3121
+ if (!is_dummy<predicate_at>())
3122
+ if (!predicate(at(i)))
3123
+ continue;
3124
+
3125
+ distance_t distance = context.measure(query, citerator_at(i), metric);
3126
+ top.insert(candidate_t{distance, static_cast<compressed_slot_t>(i)}, count);
3277
3127
  }
3278
3128
  }
3279
3129
 
3280
- void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
3281
-
3282
3130
  /**
3283
3131
  * @brief This algorithm from the original paper implements a heuristic,
3284
3132
  * that massively reduces the number of connections a point has,
3285
3133
  * to keep only the neighbors, that are from each other.
3286
3134
  */
3287
- candidates_view_t refine_(top_candidates_t& top, std::size_t needed, context_t& context) const noexcept {
3135
+ template <typename metric_at>
3136
+ candidates_view_t refine_( //
3137
+ metric_at&& metric, //
3138
+ std::size_t needed, top_candidates_t& top, context_t& context) const noexcept {
3288
3139
 
3289
3140
  top.sort_ascending();
3290
3141
  candidate_t* top_data = top.data();
@@ -3296,14 +3147,14 @@ class index_gt {
3296
3147
  std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
3297
3148
  while (submitted_count < needed && consumed_count < top_count) {
3298
3149
  candidate_t candidate = top_data[consumed_count];
3299
- node_t candidate_ref = node_with_id_(candidate.id);
3300
- distance_t candidate_dist = candidate.distance;
3301
3150
  bool good = true;
3302
3151
  for (std::size_t idx = 0; idx < submitted_count; idx++) {
3303
3152
  candidate_t submitted = top_data[idx];
3304
- node_t submitted_node = node_with_id_(submitted.id);
3305
- distance_t inter_result_dist = context.measure(submitted_node, candidate_ref);
3306
- if (inter_result_dist < candidate_dist) {
3153
+ distance_t inter_result_dist = context.measure( //
3154
+ citerator_at(candidate.slot), //
3155
+ citerator_at(submitted.slot), //
3156
+ metric);
3157
+ if (inter_result_dist < candidate.distance) {
3307
3158
  good = false;
3308
3159
  break;
3309
3160
  }
@@ -3321,44 +3172,218 @@ class index_gt {
3321
3172
  }
3322
3173
  };
3323
3174
 
3175
+ struct join_result_t {
3176
+ error_t error{};
3177
+ std::size_t intersection_size{};
3178
+ std::size_t engagements{};
3179
+ std::size_t visited_members{};
3180
+ std::size_t computed_distances{};
3181
+
3182
+ explicit operator bool() const noexcept { return !error; }
3183
+ join_result_t failed(error_t message) noexcept {
3184
+ error = std::move(message);
3185
+ return std::move(*this);
3186
+ }
3187
+ };
3188
+
3324
3189
  /**
3325
- * @brief Extracts metadata from pre-constructed index on disk, without loading it
3326
- * or mapping the whole binary file.
3190
+ * @brief Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
3191
+ * to perform fast one-to-one matching between two large collections
3192
+ * of vectors, using approximate nearest neighbors search.
3193
+ *
3194
+ * @param[inout] man_to_woman Container to map ::first keys to ::second.
3195
+ * @param[inout] woman_to_man Container to map ::second keys to ::first.
3196
+ * @param[in] executor Thread-pool to execute the job in parallel.
3197
+ * @param[in] progress Callback to report the execution progress.
3327
3198
  */
3328
- inline file_head_result_t index_metadata(char const* file_path) noexcept {
3329
-
3330
- file_head_result_t result;
3331
- file_header_t state_buffer{};
3332
- std::FILE* file = std::fopen(file_path, "rb");
3333
- if (!file)
3334
- return result.failed(std::strerror(errno));
3335
-
3336
- // Read the header
3337
- std::size_t read = std::fread(&state_buffer[0], sizeof(file_header_t), 1, file);
3338
- std::fclose(file);
3339
- if (!read)
3340
- return result.failed(std::strerror(errno));
3341
-
3342
- // Parse and validate the MIME type
3343
- file_head_t state{state_buffer};
3344
- if (std::strncmp(state.magic, default_magic(), std::strlen(default_magic())) != 0)
3345
- return result.failed("Wrong MIME type!");
3346
-
3347
- result.version_major = state.version_major;
3348
- result.version_minor = state.version_minor;
3349
- result.version_patch = state.version_patch;
3350
- result.metric = state.metric;
3351
- result.connectivity = state.connectivity;
3352
- result.max_level = state.max_level;
3353
- result.vector_alignment = state.vector_alignment;
3354
- result.bytes_per_label = state.bytes_per_label;
3355
- result.bytes_per_id = state.bytes_per_id;
3356
- result.scalar_kind = state.scalar_kind;
3357
- result.size = state.size;
3358
- result.entry_idx = state.entry_idx;
3359
- result.bytes_for_graphs = state.bytes_for_graphs;
3360
- result.bytes_for_vectors = state.bytes_for_vectors;
3361
- result.bytes_checksum = state.bytes_checksum;
3199
+ template < //
3200
+
3201
+ typename men_at, //
3202
+ typename women_at, //
3203
+ typename men_values_at, //
3204
+ typename women_values_at, //
3205
+ typename men_metric_at, //
3206
+ typename women_metric_at, //
3207
+
3208
+ typename man_to_woman_at = dummy_label_to_label_mapping_t, //
3209
+ typename woman_to_man_at = dummy_label_to_label_mapping_t, //
3210
+ typename executor_at = dummy_executor_t, //
3211
+ typename progress_at = dummy_progress_t //
3212
+ >
3213
+ static join_result_t join( //
3214
+ men_at const& men, //
3215
+ women_at const& women, //
3216
+ men_values_at const& men_values, //
3217
+ women_values_at const& women_values, //
3218
+ men_metric_at&& men_metric, //
3219
+ women_metric_at&& women_metric, //
3220
+
3221
+ index_join_config_t config = {}, //
3222
+ man_to_woman_at&& man_to_woman = man_to_woman_at{}, //
3223
+ woman_to_man_at&& woman_to_man = woman_to_man_at{}, //
3224
+ executor_at&& executor = executor_at{}, //
3225
+ progress_at&& progress = progress_at{}) noexcept {
3226
+
3227
+ if (women.size() < men.size())
3228
+ return unum::usearch::join( //
3229
+ women, men, //
3230
+ women_values, men_values, //
3231
+ std::forward<women_metric_at>(women_metric), std::forward<men_metric_at>(men_metric), //
3232
+
3233
+ config, //
3234
+ std::forward<woman_to_man_at>(woman_to_man), //
3235
+ std::forward<man_to_woman_at>(man_to_woman), //
3236
+ std::forward<executor_at>(executor), //
3237
+ std::forward<progress_at>(progress));
3238
+
3239
+ join_result_t result;
3240
+
3241
+ // Sanity checks and argument validation:
3242
+ if (&men == &women)
3243
+ return result.failed("Can't join with itself, consider copying");
3244
+
3245
+ if (config.max_proposals == 0)
3246
+ config.max_proposals = std::log(men.size()) + executor.size();
3247
+
3248
+ using proposals_count_t = std::uint16_t;
3249
+ config.max_proposals = (std::min)(men.size(), config.max_proposals);
3250
+
3251
+ using distance_t = typename men_at::distance_t;
3252
+ using dynamic_allocator_traits_t = typename men_at::dynamic_allocator_traits_t;
3253
+ using man_key_t = typename men_at::key_t;
3254
+ using woman_key_t = typename women_at::key_t;
3255
+
3256
+ // Use the `compressed_slot_t` type of the larger collection
3257
+ using compressed_slot_t = typename women_at::compressed_slot_t;
3258
+ using compressed_slot_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<compressed_slot_t>;
3259
+ using proposals_count_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<proposals_count_t>;
3260
+
3261
+ // Create an atomic queue, as a ring structure, from/to which
3262
+ // free men will be added/pulled.
3263
+ std::mutex free_men_mutex{};
3264
+ ring_gt<compressed_slot_t, compressed_slot_allocator_t> free_men;
3265
+ free_men.reserve(men.size());
3266
+ for (std::size_t i = 0; i != men.size(); ++i)
3267
+ free_men.push(static_cast<compressed_slot_t>(i));
3268
+
3269
+ // We are gonna need some temporary memory.
3270
+ buffer_gt<proposals_count_t, proposals_count_allocator_t> proposal_counts(men.size());
3271
+ buffer_gt<compressed_slot_t, compressed_slot_allocator_t> man_to_woman_slots(men.size());
3272
+ buffer_gt<compressed_slot_t, compressed_slot_allocator_t> woman_to_man_slots(women.size());
3273
+ if (!proposal_counts || !man_to_woman_slots || !woman_to_man_slots)
3274
+ return result.failed("Can't temporary mappings");
3275
+
3276
+ compressed_slot_t missing_slot;
3277
+ std::memset((void*)&missing_slot, 0xFF, sizeof(compressed_slot_t));
3278
+ std::memset((void*)man_to_woman_slots.data(), 0xFF, sizeof(compressed_slot_t) * men.size());
3279
+ std::memset((void*)woman_to_man_slots.data(), 0xFF, sizeof(compressed_slot_t) * women.size());
3280
+ std::memset(proposal_counts.data(), 0, sizeof(proposals_count_t) * men.size());
3281
+
3282
+ // Define locks, to limit concurrent accesses to `man_to_woman_slots` and `woman_to_man_slots`.
3283
+ visits_bitset_t men_locks(men.size()), women_locks(women.size());
3284
+ if (!men_locks || !women_locks)
3285
+ return result.failed("Can't allocate locks");
3286
+
3287
+ std::atomic<std::size_t> rounds{0};
3288
+ std::atomic<std::size_t> engagements{0};
3289
+ std::atomic<std::size_t> computed_distances{0};
3290
+ std::atomic<std::size_t> visited_members{0};
3291
+
3292
+ // Concurrently process all the men
3293
+ executor.execute_bulk([&](std::size_t thread_idx) {
3294
+ index_search_config_t search_config;
3295
+ search_config.expansion = config.expansion;
3296
+ search_config.exact = config.exact;
3297
+ search_config.thread = thread_idx;
3298
+ compressed_slot_t free_man_slot;
3299
+
3300
+ // While there exist a free man who still has a woman to propose to.
3301
+ while (true) {
3302
+ std::size_t passed_rounds = 0;
3303
+ std::size_t total_rounds = 0;
3304
+ {
3305
+ std::unique_lock<std::mutex> pop_lock(free_men_mutex);
3306
+ if (!free_men.try_pop(free_man_slot))
3307
+ // Primary exit path, we have exhausted the list of candidates
3308
+ break;
3309
+ passed_rounds = ++rounds;
3310
+ total_rounds = passed_rounds + free_men.size();
3311
+ }
3312
+ progress(passed_rounds, total_rounds);
3313
+ while (men_locks.atomic_set(free_man_slot))
3314
+ ;
3315
+
3316
+ proposals_count_t& free_man_proposals = proposal_counts[free_man_slot];
3317
+ if (free_man_proposals >= config.max_proposals)
3318
+ continue;
3319
+
3320
+ // Find the closest woman, to whom this man hasn't proposed yet.
3321
+ ++free_man_proposals;
3322
+ auto candidates = women.search(men_values[free_man_slot], free_man_proposals, women_metric, search_config);
3323
+ visited_members += candidates.visited_members;
3324
+ computed_distances += candidates.computed_distances;
3325
+ if (!candidates) {
3326
+ // TODO:
3327
+ }
3328
+
3329
+ auto match = candidates.back();
3330
+ auto woman = match.member;
3331
+ while (women_locks.atomic_set(woman.slot))
3332
+ ;
3333
+
3334
+ compressed_slot_t husband_slot = woman_to_man_slots[woman.slot];
3335
+ bool woman_is_free = husband_slot == missing_slot;
3336
+ if (woman_is_free) {
3337
+ // Engagement
3338
+ man_to_woman_slots[free_man_slot] = woman.slot;
3339
+ woman_to_man_slots[woman.slot] = free_man_slot;
3340
+ engagements++;
3341
+ } else {
3342
+ distance_t distance_from_husband = women_metric(women_values[woman.slot], men_values[husband_slot]);
3343
+ distance_t distance_from_candidate = match.distance;
3344
+ if (distance_from_husband > distance_from_candidate) {
3345
+ // Break-up
3346
+ while (men_locks.atomic_set(husband_slot))
3347
+ ;
3348
+ man_to_woman_slots[husband_slot] = missing_slot;
3349
+ men_locks.atomic_reset(husband_slot);
3350
+
3351
+ // New Engagement
3352
+ man_to_woman_slots[free_man_slot] = woman.slot;
3353
+ woman_to_man_slots[woman.slot] = free_man_slot;
3354
+ engagements++;
3355
+
3356
+ std::unique_lock<std::mutex> push_lock(free_men_mutex);
3357
+ free_men.push(husband_slot);
3358
+ } else {
3359
+ std::unique_lock<std::mutex> push_lock(free_men_mutex);
3360
+ free_men.push(free_man_slot);
3361
+ }
3362
+ }
3363
+
3364
+ men_locks.atomic_reset(free_man_slot);
3365
+ women_locks.atomic_reset(woman.slot);
3366
+ }
3367
+ });
3368
+
3369
+ // Export the "slots" into keys:
3370
+ std::size_t intersection_size = 0;
3371
+ for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
3372
+ compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
3373
+ if (woman_slot != missing_slot) {
3374
+ man_key_t man = men.at(man_slot).key;
3375
+ woman_key_t woman = women.at(woman_slot).key;
3376
+ man_to_woman[man] = woman;
3377
+ woman_to_man[woman] = man;
3378
+ intersection_size++;
3379
+ }
3380
+ }
3381
+
3382
+ // Export stats
3383
+ result.engagements = engagements;
3384
+ result.intersection_size = intersection_size;
3385
+ result.computed_distances = computed_distances;
3386
+ result.visited_members = visited_members;
3362
3387
  return result;
3363
3388
  }
3364
3389