usearch 2.11.1 → 2.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +100 -30
  2. package/binding.gyp +11 -12
  3. package/c/README.md +3 -0
  4. package/c/lib.cpp +50 -13
  5. package/c/test.c +60 -55
  6. package/c/usearch.h +16 -0
  7. package/cpp/CMakeLists.txt +6 -0
  8. package/cpp/README.md +34 -13
  9. package/cpp/bench.cpp +99 -47
  10. package/cpp/test.cpp +947 -132
  11. package/include/usearch/index.hpp +894 -272
  12. package/include/usearch/index_dense.hpp +418 -156
  13. package/include/usearch/index_plugins.hpp +99 -49
  14. package/javascript/dist/cjs/usearch.d.ts +39 -1
  15. package/javascript/dist/cjs/usearch.js +11 -1
  16. package/javascript/dist/esm/usearch.d.ts +39 -1
  17. package/javascript/dist/esm/usearch.js +11 -1
  18. package/javascript/tsconfig-esm.json +3 -2
  19. package/javascript/usearch.ts +18 -7
  20. package/package.json +2 -2
  21. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  22. package/prebuilds/linux-arm64/usearch.node +0 -0
  23. package/prebuilds/linux-x64/usearch.node +0 -0
  24. package/simsimd/.cmake-format.py +19 -0
  25. package/simsimd/CONTRIBUTING.md +49 -8
  26. package/simsimd/Cargo.lock +1 -1
  27. package/simsimd/Cargo.toml +10 -5
  28. package/simsimd/Package.swift +16 -0
  29. package/simsimd/README.md +69 -15
  30. package/simsimd/binding.gyp +14 -19
  31. package/simsimd/build.rs +2 -0
  32. package/simsimd/c/lib.c +18 -0
  33. package/simsimd/include/module.modulemap +4 -0
  34. package/simsimd/include/simsimd/binary.h +2 -0
  35. package/simsimd/include/simsimd/dot.h +466 -32
  36. package/simsimd/include/simsimd/probability.h +20 -10
  37. package/simsimd/include/simsimd/simsimd.h +444 -98
  38. package/simsimd/include/simsimd/spatial.h +480 -44
  39. package/simsimd/include/simsimd/types.h +161 -39
  40. package/simsimd/javascript/simsimd.ts +26 -2
  41. package/simsimd/package.json +1 -1
  42. package/simsimd/rust/benches/cosine.rs +8 -2
  43. package/simsimd/rust/benches/native.rs +240 -2
  44. package/simsimd/rust/benches/sqeuclidean.rs +8 -2
  45. package/simsimd/rust/lib.rs +50 -13
  46. package/simsimd/swift/source/SimSIMD.swift +123 -0
  47. package/simsimd/swift/tests/SimSIMDTests.swift +92 -0
  48. package/prebuilds/win32-ia32/usearch.node +0 -0
  49. package/prebuilds/win32-x64/usearch.node +0 -0
  50. package/simsimd/.releaserc +0 -87
@@ -1,16 +1,14 @@
1
1
  /**
2
- * @file index.hpp
3
- * @author Ash Vardanian
4
- * @brief Single-header Vector Search.
5
- * @date 2023-04-26
6
- *
7
- * @copyright Copyright (c) 2023
2
+ * @file index.hpp
3
+ * @author Ash Vardanian
4
+ * @brief Single-header Vector Search engine.
5
+ * @date April 26, 2023
8
6
  */
9
7
  #ifndef UNUM_USEARCH_HPP
10
8
  #define UNUM_USEARCH_HPP
11
9
 
12
10
  #define USEARCH_VERSION_MAJOR 2
13
- #define USEARCH_VERSION_MINOR 11
11
+ #define USEARCH_VERSION_MINOR 13
14
12
  #define USEARCH_VERSION_PATCH 1
15
13
 
16
14
  // Inferring C++ version
@@ -18,6 +16,9 @@
18
16
  #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
19
17
  #define USEARCH_DEFINED_CPP17
20
18
  #endif
19
+ #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) || __cplusplus >= 202002L)
20
+ #define USEARCH_DEFINED_CPP20
21
+ #endif
21
22
 
22
23
  // Inferring target OS: Windows, MacOS, or Linux
23
24
  #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
@@ -35,6 +36,14 @@
35
36
  #define USEARCH_DEFINED_GCC
36
37
  #endif
37
38
 
39
+ // The `#pragma region` and `#pragma endregion` are not supported by GCC 12 and older.
40
+ // But they are supported by GCC 13, all recent Clang versions, and MSVC.
41
+ #if defined(__GNUC__) && ((__GNUC__ > 13) || (__GNUC__ == 13 && __GNUC_MINOR__ >= 0))
42
+ #define USEARCH_USE_PRAGMA_REGION
43
+ #elif defined(__clang__) || defined(_MSC_VER)
44
+ #define USEARCH_USE_PRAGMA_REGION
45
+ #endif
46
+
38
47
  // Inferring hardware architecture: x86 vs Arm
39
48
  #if defined(__x86_64__)
40
49
  #define USEARCH_DEFINED_X86
@@ -86,17 +95,39 @@
86
95
  #include <thread> // `std::thread`
87
96
  #include <utility> // `std::pair`
88
97
 
98
+ // Helper macros for concatenation and stringification
99
+ #define usearch_concat_helper_m(a, b) a##b
100
+ #define usearch_concat_m(a, b) usearch_concat_helper_m(a, b)
101
+ #define usearch_stringify_helper_m(x) #x
102
+ #define usearch_stringify_m(x) usearch_stringify_helper_m(x)
103
+
89
104
  // Prefetching
90
105
  #if defined(USEARCH_DEFINED_GCC)
91
106
  // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
92
107
  // Zero means we are only going to read from that memory.
93
108
  // Three means high temporal locality and suggests to keep
94
109
  // the data in all layers of cache.
95
- #define prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
110
+ #define usearch_prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
96
111
  #elif defined(USEARCH_DEFINED_X86)
97
- #define prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
112
+ #define usearch_prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
98
113
  #else
99
- #define prefetch_m(ptr)
114
+ #define usearch_prefetch_m(ptr)
115
+ #endif
116
+
117
+ // Function profiling
118
+ #if defined(usearch_defined_x86)
119
+ #define usearch_profiled_m __attribute__((noinline))
120
+ #define usearch_profile_name_m(name) \
121
+ __asm__ volatile(".globl " usearch_stringify_m(usearch_concat_m(name, __COUNTER__)) "\n" usearch_stringify_m( \
122
+ usearch_concat_m(name, __COUNTER__)) ":")
123
+ #elif defined(usearch_defined_arm)
124
+ #define usearch_profiled_m __attribute__((noinline))
125
+ #define usearch_profile_name_m(name) \
126
+ __asm__ volatile(".global " usearch_stringify_m(usearch_concat_m(name, __COUNTER__)) "\n" usearch_stringify_m( \
127
+ usearch_concat_m(name, __COUNTER__)) ":")
128
+ #else
129
+ #define usearch_profiled_m
130
+ #define usearch_profile_name_m(name)
100
131
  #endif
101
132
 
102
133
  // Alignment
@@ -115,11 +146,25 @@
115
146
  #else
116
147
  #define usearch_assert_m(must_be_true, message) \
117
148
  if (!(must_be_true)) { \
118
- throw std::runtime_error(message); \
149
+ __usearch_raise_runtime_error(message); \
119
150
  }
120
151
  #define usearch_noexcept_m
121
152
  #endif
122
153
 
154
+ extern "C" {
155
+ /// @brief Helper function to simplify debugging - trace just one symbol - `__usearch_raise_runtime_error`.
156
+ /// Assuming the `extern C` block, the name won't be mangled.
157
+ inline static void __usearch_raise_runtime_error(char const* message) {
158
+ // On Windows we compile with `/EHc` flag, which specifies that functions
159
+ // with C linkage do not throw C++ exceptions.
160
+ #if !defined(__cpp_exceptions) || defined(USEARCH_DEFINED_WINDOWS)
161
+ std::terminate();
162
+ #else
163
+ throw std::runtime_error(message);
164
+ #endif
165
+ }
166
+ }
167
+
123
168
  namespace unum {
124
169
  namespace usearch {
125
170
 
@@ -168,6 +213,13 @@ template <typename at, typename other_at = at> at exchange(at& obj, other_at&& n
168
213
  return old_value;
169
214
  }
170
215
 
216
+ #if defined(USEARCH_DEFINED_CPP20)
217
+
218
+ template <typename at> void destroy_at(at* obj) { std::destroy_at(obj); }
219
+ template <typename at> void construct_at(at* obj) { std::construct_at(obj); }
220
+
221
+ #else
222
+
171
223
  /// @brief The `std::destroy_at` alternative for C++11.
172
224
  template <typename at, typename sfinae_at = at>
173
225
  typename std::enable_if<std::is_pod<sfinae_at>::value>::type destroy_at(at*) {}
@@ -184,6 +236,8 @@ typename std::enable_if<!std::is_pod<sfinae_at>::value>::type construct_at(at* o
184
236
  new (obj) at();
185
237
  }
186
238
 
239
+ #endif
240
+
187
241
  /**
188
242
  * @brief A reference to a misaligned memory location with a specific type.
189
243
  * It is needed to avoid Undefined Behavior when dereferencing addresses
@@ -223,27 +277,57 @@ template <typename at> class misaligned_ptr_gt {
223
277
  using pointer = misaligned_ptr_gt<element_t>;
224
278
  using reference = misaligned_ref_gt<element_t>;
225
279
 
280
+ misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
281
+
226
282
  reference operator*() const noexcept { return {ptr_}; }
227
283
  reference operator[](std::size_t i) noexcept { return reference(ptr_ + i * sizeof(element_t)); }
228
284
  value_type operator[](std::size_t i) const noexcept {
229
285
  return misaligned_load<element_t>(ptr_ + i * sizeof(element_t));
230
286
  }
231
287
 
232
- misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
233
- misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(element_t)); }
234
- misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(element_t)); }
235
- misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(element_t)); }
236
- misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(element_t)); }
237
-
238
- // clang-format off
239
- misaligned_ptr_gt& operator++() noexcept { ptr_ += sizeof(element_t); return *this; }
240
- misaligned_ptr_gt& operator--() noexcept { ptr_ -= sizeof(element_t); return *this; }
241
- misaligned_ptr_gt& operator+=(difference_type d) noexcept { ptr_ += d * sizeof(element_t); return *this; }
242
- misaligned_ptr_gt& operator-=(difference_type d) noexcept { ptr_ -= d * sizeof(element_t); return *this; }
243
- // clang-format on
244
-
245
- bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
246
- bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
288
+ misaligned_ptr_gt& operator++() noexcept {
289
+ ptr_ += sizeof(element_t);
290
+ return *this;
291
+ }
292
+ misaligned_ptr_gt& operator--() noexcept {
293
+ ptr_ -= sizeof(element_t);
294
+ return *this;
295
+ }
296
+ misaligned_ptr_gt operator++(int) noexcept {
297
+ misaligned_ptr_gt tmp = *this;
298
+ ++(*this);
299
+ return tmp;
300
+ }
301
+ misaligned_ptr_gt operator--(int) noexcept {
302
+ misaligned_ptr_gt tmp = *this;
303
+ --(*this);
304
+ return tmp;
305
+ }
306
+ misaligned_ptr_gt operator+(difference_type d) const noexcept {
307
+ return misaligned_ptr_gt(ptr_ + d * sizeof(element_t));
308
+ }
309
+ misaligned_ptr_gt operator-(difference_type d) const noexcept {
310
+ return misaligned_ptr_gt(ptr_ - d * sizeof(element_t));
311
+ }
312
+ difference_type operator-(const misaligned_ptr_gt& other) const noexcept {
313
+ return (ptr_ - other.ptr_) / sizeof(element_t);
314
+ }
315
+
316
+ misaligned_ptr_gt& operator+=(difference_type d) noexcept {
317
+ ptr_ += d * sizeof(element_t);
318
+ return *this;
319
+ }
320
+ misaligned_ptr_gt& operator-=(difference_type d) noexcept {
321
+ ptr_ -= d * sizeof(element_t);
322
+ return *this;
323
+ }
324
+
325
+ bool operator==(misaligned_ptr_gt const& other) const noexcept { return ptr_ == other.ptr_; }
326
+ bool operator!=(misaligned_ptr_gt const& other) const noexcept { return ptr_ != other.ptr_; }
327
+ bool operator<(misaligned_ptr_gt const& other) const noexcept { return ptr_ < other.ptr_; }
328
+ bool operator<=(misaligned_ptr_gt const& other) const noexcept { return ptr_ <= other.ptr_; }
329
+ bool operator>(misaligned_ptr_gt const& other) const noexcept { return ptr_ > other.ptr_; }
330
+ bool operator>=(misaligned_ptr_gt const& other) const noexcept { return ptr_ >= other.ptr_; }
247
331
  };
248
332
 
249
333
  /**
@@ -279,7 +363,8 @@ template <typename scalar_at, typename allocator_at = std::allocator<scalar_at>>
279
363
  for (std::size_t i = 0; i != size_; ++i)
280
364
  construct_at(data_ + i);
281
365
  }
282
- ~buffer_gt() noexcept {
366
+ ~buffer_gt() noexcept { reset(); }
367
+ void reset() noexcept {
283
368
  if (!std::is_trivially_destructible<scalar_at>::value)
284
369
  for (std::size_t i = 0; i != size_; ++i)
285
370
  destroy_at(data_ + i);
@@ -319,7 +404,8 @@ class error_t {
319
404
  char const* message_{};
320
405
 
321
406
  public:
322
- error_t(char const* message = nullptr) noexcept : message_(message) {}
407
+ error_t() noexcept : message_(nullptr) {}
408
+ error_t(char const* message) noexcept : message_(message) {}
323
409
  error_t& operator=(char const* message) noexcept {
324
410
  message_ = message;
325
411
  return *this;
@@ -332,11 +418,18 @@ class error_t {
332
418
  std::swap(message_, other.message_);
333
419
  return *this;
334
420
  }
421
+
422
+ /// @brief Checks if there was an error.
335
423
  explicit operator bool() const noexcept { return message_ != nullptr; }
424
+
425
+ /// @brief Returns the error message.
336
426
  char const* what() const noexcept { return message_; }
427
+
428
+ /// @brief Releases the error message, meaning the caller takes ownership.
337
429
  char const* release() noexcept { return exchange(message_, nullptr); }
338
430
 
339
431
  #if defined(__cpp_exceptions) || defined(__EXCEPTIONS)
432
+ /// @brief Destructor raises an exception if an error was recorded.
340
433
  ~error_t() noexcept(false) {
341
434
  #if defined(USEARCH_DEFINED_CPP17)
342
435
  if (message_ && std::uncaught_exceptions() == 0)
@@ -345,12 +438,17 @@ class error_t {
345
438
  #endif
346
439
  raise();
347
440
  }
441
+
442
+ /// @brief Throws an exception using to be caught by `try` / `catch`.
348
443
  void raise() noexcept(false) {
349
444
  if (message_)
350
445
  throw std::runtime_error(exchange(message_, nullptr));
351
446
  }
352
447
  #else
448
+ /// @brief Destructor terminates if an error was recorded.
353
449
  ~error_t() noexcept { raise(); }
450
+
451
+ /// @brief Terminates if an error was recorded.
354
452
  void raise() noexcept {
355
453
  if (message_)
356
454
  std::terminate();
@@ -363,7 +461,7 @@ class error_t {
363
461
  * or an error. It's used to avoid raising exception, and gracefully propagate
364
462
  * the error.
365
463
  *
366
- * @tparam result_at The type of the expected result.
464
+ * @tparam result_at The type of the expected result.
367
465
  */
368
466
  template <typename result_at> struct expected_gt {
369
467
  result_at result;
@@ -497,6 +595,64 @@ using bitset_t = bitset_gt<>;
497
595
  * @brief Similar to `std::priority_queue`, but allows raw access to underlying
498
596
  * memory, in case you want to shuffle it or sort. Good for collections
499
597
  * from 100s to 10'000s elements.
598
+ *
599
+ * In a max-heap, the heap property ensures that the value of each node is greater
600
+ * than or equal to the values of its children. This means that the largest element
601
+ * is always at the root of the heap.
602
+ *
603
+ * @section Heap Structures
604
+ *
605
+ * There are several designs of heaps. Binary heaps are the simplest & most common
606
+ * variant, that is easy to implement as a succint array. However, they are not the
607
+ * most efficient for all operations. Most importantly, @b melding (merging) of
608
+ * two heaps has linear complexity in time.
609
+ *
610
+ * +-----------------+---------+-----------+---------+--------------+---------+
611
+ * | Operation | find-max| delete-max| insert | increase-key | meld |
612
+ * +-----------------+---------+-----------+---------+--------------+---------+
613
+ * | Binary | Θ(1) | Θ(log n) | O(log n)| O(log n) | Θ(n) |
614
+ * | Leftist | Θ(1) | Θ(log n) | O(log n)| Θ(log n) | Θ(log n)|
615
+ * | Binomial | Θ(1) | Θ(log n) | Θ(1) | Θ(log n) | O(log n)|
616
+ * | Skew binomial | Θ(1) | Θ(log n) | Θ(1) | O(log n) | O(log n)|
617
+ * | Pairing | Θ(1) | O(log n) | Θ(1) | o(log n) | Θ(1) |
618
+ * | Rank-pairing | Θ(1) | O(log n) | Θ(1) | Θ(1) | Θ(1) |
619
+ * | Fibonacci | Θ(1) | O(log n) | Θ(1) | Θ(1) | Θ(1) |
620
+ * | Strict Fibonacci| Θ(1) | O(log n) | Θ(1) | Θ(1) | Θ(1) |
621
+ * | Brodal | Θ(1) | Θ(log n) | Θ(1) | Θ(1) | Θ(1) |
622
+ * | 2–3 heap | Θ(1) | O(log n) | Θ(1) | Θ(1) | O(log n)|
623
+ * +-----------------+---------+-----------+---------+--------------+---------+
624
+ *
625
+ * It's well known, that improved priority queue structures translate into better
626
+ * graph-transversal algorithms. For example, Dijkstra's algorithm can be sped up
627
+ * by using a Fibonacci heap for arbitrary weights. For integer weight bounded
628
+ * by L, Schrijver reported following time complexities in 2004:
629
+ *
630
+ * +------------+-------------------------------------+----------------------------+--------------------------+
631
+ * | Weights | Algorithm | Time complexity | Author |
632
+ * +------------+-------------------------------------+----------------------------+--------------------------+
633
+ * | R | | O(V^2 EL) | Ford 1956 |
634
+ * | R | Bellman–Ford algorithm | O(VE) | Shimbel 1955, Bellman |
635
+ * | | | | 1958, Moore 1959 |
636
+ * | R | | O(V^2 log V) | Dantzig 1960 |
637
+ * | R | Dijkstra's with list | O(V^2) | Leyzorek et al. 1957, |
638
+ * | | | | Dijkstra 1959... |
639
+ * | R | Dijkstra's with binary heap | O((E + V) log V) | Johnson 1977 |
640
+ * | R | Dijkstra's with Fibonacci heap | O(E + V log V) | Fredman & Tarjan 1984, |
641
+ * | | | | Fredman & Tarjan 1987 |
642
+ * | R | Quantum Dijkstra | O(√VE log^2 V) | Dürr et al. 2006 |
643
+ * | R | Dial's algorithm (Dijkstra's using | O(E + LV) | Dial 1969 |
644
+ * | | a bucket queue with L buckets) | | |
645
+ * | N | | O(E log log L) | Johnson 1981, Karlsson & |
646
+ * | | | | Poblete 1983 |
647
+ * | N | Gabow's algorithm | O(E log_E/V L) | Gabow 1983, Gabow 1985 |
648
+ * | N | | O(E + V √log L) | Ahuja et al. 1990 |
649
+ * | N | Thorup | O(E + V log log V) | Thorup 2004 |
650
+ * +------------+-------------------------------------+----------------------------+--------------------------+
651
+ *
652
+ * Possible improvements:
653
+ * - Randomized meldable heaps: https://en.wikipedia.org/wiki/Randomized_meldable_heap
654
+ * - D-ary heaps: https://en.wikipedia.org/wiki/D-ary_heap
655
+ * - B-heap: https://en.wikipedia.org/wiki/B-heap
500
656
  */
501
657
  template <typename element_at, //
502
658
  typename comparator_at = std::less<void>, // <void> is needed before C++14.
@@ -547,13 +703,25 @@ class max_heap_gt {
547
703
  inline bool empty() const noexcept { return !size_; }
548
704
  inline std::size_t size() const noexcept { return size_; }
549
705
  inline std::size_t capacity() const noexcept { return capacity_; }
706
+ inline element_t* data() noexcept { return elements_; }
707
+ inline element_t const* data() const noexcept { return elements_; }
708
+ inline void clear() noexcept { size_ = 0; }
709
+ inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
550
710
 
551
711
  /// @brief Selects the largest element in the heap.
552
712
  /// @return Reference to the stored element.
553
713
  inline element_t const& top() const noexcept { return elements_[0]; }
554
- inline void clear() noexcept { size_ = 0; }
555
714
 
556
- bool reserve(std::size_t new_capacity) noexcept {
715
+ /// @brief Invalidates the "max-heap" property, transforming into ascending range.
716
+ inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
717
+
718
+ /**
719
+ * @brief Ensures the heap has enough capacity for the specified number of elements.
720
+ * @param new_capacity The desired minimum capacity.
721
+ * @return True if the capacity was successfully increased, false otherwise.
722
+ */
723
+ usearch_profiled_m bool reserve(std::size_t new_capacity) noexcept {
724
+ usearch_profile_name_m(max_heap_reserve);
557
725
  if (new_capacity < capacity_)
558
726
  return true;
559
727
 
@@ -573,6 +741,11 @@ class max_heap_gt {
573
741
  return new_elements;
574
742
  }
575
743
 
744
+ /**
745
+ * @brief Inserts an element into the heap.
746
+ * @param element The element to be inserted.
747
+ * @return True if the element was successfully inserted, false otherwise.
748
+ */
576
749
  bool insert(element_t&& element) noexcept {
577
750
  if (!reserve(size_ + 1))
578
751
  return false;
@@ -581,13 +754,33 @@ class max_heap_gt {
581
754
  return true;
582
755
  }
583
756
 
584
- inline void insert_reserved(element_t&& element) noexcept {
757
+ /**
758
+ * @brief Inserts an element into the heap without reserving additional space.
759
+ * @param element The element to be inserted.
760
+ */
761
+ usearch_profiled_m void insert_reserved(element_t&& element) noexcept {
762
+ usearch_profile_name_m(max_heap_insert_reserved);
585
763
  new (&elements_[size_]) element_t(element);
586
764
  size_++;
587
765
  shift_up(size_ - 1);
588
766
  }
589
767
 
590
- inline element_t pop() noexcept {
768
+ /**
769
+ * @brief Inserts multiple elements into the heap.
770
+ * @param elements Pointer to the elements to be inserted.
771
+ * @return True if the elements were successfully inserted, false otherwise.
772
+ */
773
+ inline bool insert_many(element_t const* elements) noexcept {
774
+ // Wikipedia describes a procedure, due to Floyd, which constructs a heap from an array in linear time.
775
+ // It also mentions a procedure for merging two heaps, of sizes 𝑛 and 𝑘, in time 𝑂(𝑘+log𝑘log𝑛).
776
+ // Altogether, we can add 𝑘 elements to a heap of length 𝑛 in time 𝑂(𝑘+log𝑘log𝑛): first build a heap containing
777
+ // 𝑘 elements to be inserted (takes 𝑂(𝑘) time), then merge that with the heap of size 𝑛 (takes 𝑂(𝑘+log𝑘log𝑛)
778
+ // time). Compare this to repeated insertion, which would run in time 𝑂(𝑘log𝑛).
779
+ return false;
780
+ }
781
+
782
+ usearch_profiled_m element_t pop() noexcept {
783
+ usearch_profile_name_m(max_heap_pop);
591
784
  element_t result = top();
592
785
  std::swap(elements_[0], elements_[size_ - 1]);
593
786
  size_--;
@@ -596,24 +789,29 @@ class max_heap_gt {
596
789
  return result;
597
790
  }
598
791
 
599
- /** @brief Invalidates the "max-heap" property, transforming into ascending range. */
600
- inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
601
- inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
602
-
603
- inline element_t* data() noexcept { return elements_; }
604
- inline element_t const* data() const noexcept { return elements_; }
605
-
606
792
  private:
607
- inline std::size_t parent_idx(std::size_t i) const noexcept { return (i - 1u) / 2u; }
608
- inline std::size_t left_child_idx(std::size_t i) const noexcept { return (i * 2u) + 1u; }
609
- inline std::size_t right_child_idx(std::size_t i) const noexcept { return (i * 2u) + 2u; }
793
+ static std::size_t parent_idx(std::size_t i) noexcept { return (i - 1u) / 2u; }
794
+ static std::size_t left_child_idx(std::size_t i) noexcept { return (i * 2u) + 1u; }
795
+ static std::size_t right_child_idx(std::size_t i) noexcept { return (i * 2u) + 2u; }
610
796
  static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
611
797
 
798
+ /**
799
+ * @brief Shifts an element up to maintain the heap property.
800
+ * This operation is called when a new element is @b added at the end of the heap.
801
+ * The element is moved up until the heap property is restored.
802
+ * @param i Index of the element to be shifted up.
803
+ */
612
804
  void shift_up(std::size_t i) noexcept {
613
805
  for (; i && less(elements_[parent_idx(i)], elements_[i]); i = parent_idx(i))
614
806
  std::swap(elements_[parent_idx(i)], elements_[i]);
615
807
  }
616
808
 
809
+ /**
810
+ * @brief Shifts an element down to maintain the heap property.
811
+ * This operation is called when the root element is @b removed and the last element is moved to the root.
812
+ * The element is moved down until the heap property is restored.
813
+ * @param i Index of the element to be shifted down.
814
+ */
617
815
  void shift_down(std::size_t i) noexcept {
618
816
  std::size_t max_idx = i;
619
817
 
@@ -736,6 +934,18 @@ class sorted_buffer_gt {
736
934
  return true;
737
935
  }
738
936
 
937
+ inline bool insert_sorted(element_t const* elements, std::size_t elements_count, std::size_t limit) noexcept {
938
+ if (!size_) {
939
+ }
940
+ // If we are inserting elements, we only perform full-scale binary search once,
941
+ // and then only compute successive insertion offsets based only on the tail.
942
+ std::size_t slot = std::lower_bound(elements_, elements_ + size_, elements[0], &less) - elements_;
943
+ if (slot == limit)
944
+ return false;
945
+ std::size_t to_move = size_ - slot - (size_ == limit);
946
+ std::size_t next_slot = std::lower_bound(elements_ + slot, elements_ + size_, elements[1], &less) - elements_;
947
+ }
948
+
739
949
  inline element_t pop() noexcept {
740
950
  size_--;
741
951
  element_t result = elements_[size_];
@@ -758,9 +968,7 @@ class sorted_buffer_gt {
758
968
  #endif
759
969
 
760
970
  /**
761
- * @brief Five-byte integer type to address node clouds with over 4B entries.
762
- *
763
- * @note Avoid usage in 32bit environment
971
+ * @brief Five-byte integer type to address node clouds with over 4B entries.
764
972
  */
765
973
  class usearch_pack_m uint40_t {
766
974
  unsigned char octets[5];
@@ -772,7 +980,10 @@ class usearch_pack_m uint40_t {
772
980
 
773
981
  public:
774
982
  inline uint40_t() noexcept { broadcast(0); }
775
- inline uint40_t(std::uint32_t n) noexcept { std::memcpy(&octets[1], &n, 4); }
983
+ inline uint40_t(std::uint32_t n) noexcept {
984
+ std::memcpy(&octets, &n, 4);
985
+ octets[4] = 0;
986
+ }
776
987
 
777
988
  #ifdef USEARCH_64BIT_ENV
778
989
  inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, &n, 5); }
@@ -789,22 +1000,38 @@ class usearch_pack_m uint40_t {
789
1000
  std::memcpy(octets, &n, 5);
790
1001
  #else
791
1002
  std::memcpy(octets, &n, 4);
792
- #endif
1003
+ octets[4] = 0;
1004
+ #endif // USEARCH_64BIT_ENV
793
1005
  }
794
- #endif
1006
+ #endif // USEARCH_DEFINED_CLANG && USEARCH_DEFINED_APPLE
795
1007
 
796
1008
  inline operator std::size_t() const noexcept {
797
1009
  std::size_t result = 0;
798
1010
  #ifdef USEARCH_64BIT_ENV
799
1011
  std::memcpy(&result, octets, 5);
800
1012
  #else
801
- std::memcpy(&result, octets + 1, 4);
1013
+ std::memcpy(&result, octets, 4);
802
1014
  #endif
803
1015
  return result;
804
1016
  }
805
1017
 
806
1018
  inline static uint40_t max() noexcept { return uint40_t{}.broadcast(0xFF); }
807
1019
  inline static uint40_t min() noexcept { return uint40_t{}.broadcast(0); }
1020
+
1021
+ inline bool operator==(uint40_t const& other) const noexcept { return std::memcmp(octets, other.octets, 5) == 0; }
1022
+ inline bool operator!=(uint40_t const& other) const noexcept { return !(*this == other); }
1023
+ inline bool operator>(uint40_t const& other) const noexcept { return other < *this; }
1024
+ inline bool operator<=(uint40_t const& other) const noexcept { return !(*this > other); }
1025
+ inline bool operator>=(uint40_t const& other) const noexcept { return !(*this < other); }
1026
+ inline bool operator<(uint40_t const& other) const noexcept {
1027
+ for (int i = 0; i < 5; ++i) {
1028
+ if (octets[4 - i] < other.octets[4 - i])
1029
+ return true;
1030
+ if (octets[4 - i] > other.octets[4 - i])
1031
+ return false;
1032
+ }
1033
+ return false;
1034
+ }
808
1035
  };
809
1036
 
810
1037
  #if defined(USEARCH_DEFINED_WINDOWS)
@@ -813,12 +1040,33 @@ class usearch_pack_m uint40_t {
813
1040
 
814
1041
  static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
815
1042
 
816
- // clang-format off
817
- template <typename key_at, typename std::enable_if<std::is_integral<key_at>::value>::type* = nullptr> key_at default_free_value() { return std::numeric_limits<key_at>::max(); }
818
- template <typename key_at, typename std::enable_if<std::is_same<key_at, uint40_t>::value>::type* = nullptr> uint40_t default_free_value() { return uint40_t::max(); }
819
- template <typename key_at, typename std::enable_if<!std::is_integral<key_at>::value && !std::is_same<key_at, uint40_t>::value>::type* = nullptr> key_at default_free_value() { return key_at(); }
820
- // clang-format on
1043
+ /**
1044
+ * @brief Reflection-helper to get the default "unused" value for a given type.
1045
+ * Needed to initialize hash-sets and bit-sets.
1046
+ */
1047
+ template <typename element_at> struct default_free_value_gt {
1048
+ template <typename sfinae_element_at = element_at,
1049
+ typename std::enable_if<std::is_integral<sfinae_element_at>::value>::type* = nullptr>
1050
+ static sfinae_element_at value() noexcept {
1051
+ return std::numeric_limits<element_at>::max();
1052
+ }
1053
+ template <typename sfinae_element_at = element_at,
1054
+ typename std::enable_if<!std::is_integral<sfinae_element_at>::value>::type* = nullptr>
1055
+ static sfinae_element_at value() noexcept {
1056
+ return element_at();
1057
+ }
1058
+ };
1059
+
1060
+ template <> struct default_free_value_gt<uint40_t> {
1061
+ static uint40_t value() noexcept { return uint40_t::max(); }
1062
+ };
1063
+
1064
+ template <typename element_at> element_at default_free_value() { return default_free_value_gt<element_at>::value(); }
821
1065
 
1066
+ /**
1067
+ * @brief Adapter to allow definining arbitrary hash functions for keys and slots.
1068
+ * It's added, as overloading `std::hash` is not recommended by the standard.
1069
+ */
822
1070
  template <typename element_at> struct hash_gt {
823
1071
  std::size_t operator()(element_at const& element) const noexcept { return std::hash<element_at>{}(element); }
824
1072
  };
@@ -829,6 +1077,7 @@ template <> struct hash_gt<uint40_t> {
829
1077
 
830
1078
  /**
831
1079
  * @brief Minimalistic hash-set implementation to track visited nodes during graph traversal.
1080
+ * In our primary usecase, its a sparse alternative to a bit-set.
832
1081
  *
833
1082
  * It doesn't support deletion of separate objects, but supports `clear`-ing all at once.
834
1083
  * It expects `reserve` to be called ahead of all insertions, so no resizes are needed.
@@ -896,6 +1145,10 @@ class growing_hash_set_gt {
896
1145
  growing_hash_set_gt(growing_hash_set_gt const&) = delete;
897
1146
  growing_hash_set_gt& operator=(growing_hash_set_gt const&) = delete;
898
1147
 
1148
+ /**
1149
+ * @brief Checks if the element is already in the hash-set.
1150
+ * @return `true` if the element is already in the hash-set.
1151
+ */
899
1152
  inline bool test(element_t const& elem) const noexcept {
900
1153
  std::size_t index = hasher_(elem) & (capacity_ - 1);
901
1154
  while (slots_[index] != default_free_value<element_t>()) {
@@ -908,7 +1161,7 @@ class growing_hash_set_gt {
908
1161
  }
909
1162
 
910
1163
  /**
911
- *
1164
+ * @brief Inserts an element into the hash-set.
912
1165
  * @return Similar to `bitset_gt`, returns the previous value.
913
1166
  */
914
1167
  inline bool set(element_t const& elem) noexcept {
@@ -925,6 +1178,10 @@ class growing_hash_set_gt {
925
1178
  return false;
926
1179
  }
927
1180
 
1181
+ /**
1182
+ * @brief Extends the capacity of the hash-set.
1183
+ * @return `true` if enough capacity is available, `false` if memory allocation failed.
1184
+ */
928
1185
  bool reserve(std::size_t new_capacity) noexcept {
929
1186
  new_capacity = (new_capacity * 5u) / 3u;
930
1187
  if (new_capacity <= capacity_)
@@ -1007,7 +1264,7 @@ class ring_gt {
1007
1264
  size_t size() const noexcept {
1008
1265
  if (empty_)
1009
1266
  return 0;
1010
- else if (head_ >= tail_)
1267
+ else if (head_ > tail_)
1011
1268
  return head_ - tail_;
1012
1269
  else
1013
1270
  return capacity_ - (tail_ - head_);
@@ -1052,7 +1309,9 @@ class ring_gt {
1052
1309
  return true;
1053
1310
  }
1054
1311
 
1055
- void push(element_t const& value) noexcept {
1312
+ void push(element_t const& value) usearch_noexcept_m {
1313
+ usearch_assert_m(capacity() > 0, "Ring buffer is not initialized");
1314
+ usearch_assert_m(size() < capacity(), "Ring buffer is full");
1056
1315
  elements_[head_] = value;
1057
1316
  head_ = (head_ + 1) % capacity_;
1058
1317
  empty_ = false;
@@ -1060,7 +1319,7 @@ class ring_gt {
1060
1319
 
1061
1320
  bool try_push(element_t const& value) noexcept {
1062
1321
  if (head_ == tail_ && !empty_)
1063
- return false; // elements_ is full
1322
+ return false; // `elements_` is full
1064
1323
 
1065
1324
  return push(value);
1066
1325
  return true;
@@ -1113,20 +1372,49 @@ struct index_config_t {
1113
1372
  std::size_t connectivity_base = default_connectivity() * 2;
1114
1373
 
1115
1374
  inline index_config_t() = default;
1116
- inline index_config_t(std::size_t c) noexcept
1117
- : connectivity(c ? c : default_connectivity()), connectivity_base(c ? c * 2 : default_connectivity() * 2) {}
1118
- inline index_config_t(std::size_t c, std::size_t cb) noexcept
1119
- : connectivity(c), connectivity_base((std::max)(c, cb)) {}
1375
+ inline index_config_t(std::size_t c, std::size_t cb = 0) noexcept : connectivity(c), connectivity_base(cb) {}
1376
+
1377
+ /**
1378
+ * @brief Validates the configuration settings, updating them in-place.
1379
+ * @return Error message, if any.
1380
+ */
1381
+ inline error_t validate() noexcept {
1382
+ if (connectivity == 0)
1383
+ connectivity = default_connectivity();
1384
+ if (connectivity_base == 0)
1385
+ connectivity_base = connectivity * 2;
1386
+ if (connectivity < 2)
1387
+ return "Connectivity must be at least 2, otherwise the index degenerates into ropes";
1388
+ if (connectivity_base < connectivity)
1389
+ return "Base layer should be at least as connected as the rest of the graph";
1390
+ return {};
1391
+ }
1392
+
1393
+ /**
1394
+ * @brief Immutable function to check if the configuration is valid.
1395
+ * @return `true` if the configuration is valid.
1396
+ */
1397
+ inline bool is_valid() const noexcept { return connectivity >= 2 && connectivity_base >= connectivity; }
1120
1398
  };
1121
1399
 
1400
+ /**
1401
+ * @brief Growth settings for the index container.
1402
+ * Includes the upper bound for `::members` capacity,
1403
+ * and the number of read/write threads expected to work with the index.
1404
+ */
1122
1405
  struct index_limits_t {
1406
+ /// @brief Maximum number of entries in the index.
1123
1407
  std::size_t members = 0;
1408
+ /// @brief Max number of threads simultaneously updating entries.
1124
1409
  std::size_t threads_add = std::thread::hardware_concurrency();
1410
+ /// @brief Max number of threads simultaneously searching entries.
1125
1411
  std::size_t threads_search = std::thread::hardware_concurrency();
1126
1412
 
1127
1413
  inline index_limits_t(std::size_t n, std::size_t t) noexcept : members(n), threads_add(t), threads_search(t) {}
1128
1414
  inline index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
1415
+ /// @brief Returns the upper limit for the number of threads.
1129
1416
  inline std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
1417
+ /// @brief Returns the concurrency-level of the index - the minimum of thread counts.
1130
1418
  inline std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
1131
1419
  };
1132
1420
 
@@ -1404,8 +1692,10 @@ class input_file_t {
1404
1692
  serialization_result_t read(void* begin, std::size_t length) noexcept {
1405
1693
  serialization_result_t result;
1406
1694
  std::size_t read = std::fread(begin, length, 1, file_);
1407
- if (length && !read)
1408
- return result.failed(std::feof(file_) ? "End of file reached!" : std::strerror(errno));
1695
+ if (length && !read) {
1696
+ bool reached_eof = std::feof(file_);
1697
+ return result.failed(reached_eof ? "End of file reached!" : std::strerror(errno));
1698
+ }
1409
1699
  return result;
1410
1700
  }
1411
1701
  void close() noexcept {
@@ -1563,6 +1853,14 @@ class memory_mapped_file_t {
1563
1853
  }
1564
1854
  };
1565
1855
 
1856
+ /**
1857
+ * @brief Metadata header for the serialized index.
1858
+ *
1859
+ * This structure is very minimalistic by design. It contains no information
1860
+ * about the capacity of the index, so you'll have to `reserve` after loading.
1861
+ * It also contains no info on the metric or key types, so you'll have to store
1862
+ * that information elsewhere, like we do in `index_dense_head_t`.
1863
+ */
1566
1864
  struct index_serialized_header_t {
1567
1865
  std::uint64_t size = 0;
1568
1866
  std::uint64_t connectivity = 0;
@@ -1608,8 +1906,8 @@ template <typename key_at> inline key_at get_key(member_ref_gt<key_at> const& m)
1608
1906
  * be seen as a network of keys, accelerating approximate @b Value~>Key visited_members.
1609
1907
  *
1610
1908
  * Unlike most implementations, this one is generic anc can be used for any search,
1611
- * not just within equi-dimensional vectors. Examples range from texts to similar Chess
1612
- * positions.
1909
+ * not just within equi-dimensional vectors. Examples range from Texts to similar Chess
1910
+ * positions, Geo-Spatial Search, and even Graphs.
1613
1911
  *
1614
1912
  * @tparam key_at
1615
1913
  * The type of primary objects stored in the index.
@@ -1695,6 +1993,7 @@ class index_gt {
1695
1993
  using dynamic_allocator_t = dynamic_allocator_at;
1696
1994
  using tape_allocator_t = tape_allocator_at;
1697
1995
  static_assert(sizeof(vector_key_t) >= sizeof(compressed_slot_t), "Having tiny keys doesn't make sense.");
1996
+ static_assert(std::is_signed<distance_t>::value, "Distance must be a signed type, as we use the unary minus.");
1698
1997
 
1699
1998
  using member_cref_t = member_cref_gt<vector_key_t>;
1700
1999
  using member_ref_t = member_ref_gt<vector_key_t>;
@@ -1705,10 +2004,17 @@ class index_gt {
1705
2004
 
1706
2005
  friend class index_gt;
1707
2006
  member_iterator_gt() noexcept {}
1708
- member_iterator_gt(index_t* index, std::size_t slot) noexcept : index_(index), slot_(slot) {}
2007
+ member_iterator_gt(index_t* index, compressed_slot_t slot) noexcept : index_(index), slot_(slot) {}
2008
+
2009
+ template <int> ref_t call_key(std::true_type) const noexcept {
2010
+ return ref_t{index_->node_at_(slot_).ckey(), slot_};
2011
+ }
2012
+ template <int> ref_t call_key(std::false_type) const noexcept {
2013
+ return ref_t{index_->node_at_(slot_).key(), slot_};
2014
+ }
1709
2015
 
1710
2016
  index_t* index_{};
1711
- std::size_t slot_{};
2017
+ compressed_slot_t slot_{};
1712
2018
 
1713
2019
  public:
1714
2020
  using iterator_category = std::random_access_iterator_tag;
@@ -1717,22 +2023,21 @@ class index_gt {
1717
2023
  using pointer = void;
1718
2024
  using reference = ref_t;
1719
2025
 
1720
- reference operator*() const noexcept { return {index_->node_at_(slot_).key(), slot_}; }
1721
- vector_key_t key() const noexcept { return index_->node_at_(slot_).key(); }
2026
+ reference operator*() const noexcept { return call_key<0>(std::is_const<index_t>()); }
2027
+ vector_key_t key() const noexcept { return index_->node_at_(slot_).ckey(); }
1722
2028
 
1723
- friend inline std::size_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
2029
+ friend inline compressed_slot_t get_slot(member_iterator_gt const& it) noexcept { return it.slot_; }
1724
2030
  friend inline vector_key_t get_key(member_iterator_gt const& it) noexcept { return it.key(); }
1725
2031
 
1726
- member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, slot_ + 1); }
1727
- member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, slot_ - 1); }
1728
- member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, slot_ + d); }
1729
- member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, slot_ - d); }
1730
-
1731
2032
  // clang-format off
1732
- member_iterator_gt& operator++() noexcept { slot_ += 1; return *this; }
1733
- member_iterator_gt& operator--() noexcept { slot_ -= 1; return *this; }
1734
- member_iterator_gt& operator+=(difference_type d) noexcept { slot_ += d; return *this; }
1735
- member_iterator_gt& operator-=(difference_type d) noexcept { slot_ -= d; return *this; }
2033
+ member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + 1)); }
2034
+ member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - 1)); }
2035
+ member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + d)); }
2036
+ member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - d)); }
2037
+ member_iterator_gt& operator++() noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + 1); return *this; }
2038
+ member_iterator_gt& operator--() noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - 1); return *this; }
2039
+ member_iterator_gt& operator+=(difference_type d) noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) + d); return *this; }
2040
+ member_iterator_gt& operator-=(difference_type d) noexcept { slot_ = static_cast<compressed_slot_t>(static_cast<std::size_t>(slot_) - d); return *this; }
1736
2041
  bool operator==(member_iterator_gt const& other) const noexcept { return index_ == other.index_ && slot_ == other.slot_; }
1737
2042
  bool operator!=(member_iterator_gt const& other) const noexcept { return index_ != other.index_ || slot_ != other.slot_; }
1738
2043
  // clang-format on
@@ -1823,8 +2128,10 @@ class index_gt {
1823
2128
  node_t& operator=(node_t const&) = default;
1824
2129
 
1825
2130
  misaligned_ref_gt<vector_key_t const> ckey() const noexcept { return {tape_}; }
1826
- misaligned_ref_gt<vector_key_t> key() const noexcept { return {tape_}; }
1827
- misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(vector_key_t)}; }
2131
+ misaligned_ref_gt<vector_key_t const> ckey() noexcept { return {tape_}; }
2132
+ misaligned_ref_gt<vector_key_t const> key() const noexcept { return {tape_}; }
2133
+ misaligned_ref_gt<vector_key_t> key() noexcept { return {tape_}; }
2134
+ misaligned_ref_gt<level_t> level() noexcept { return {tape_ + sizeof(vector_key_t)}; }
1828
2135
 
1829
2136
  void key(vector_key_t v) noexcept { return misaligned_store<vector_key_t>(tape_, v); }
1830
2137
  void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(vector_key_t), v); }
@@ -1842,16 +2149,22 @@ class index_gt {
1842
2149
  class neighbors_ref_t {
1843
2150
  byte_t* tape_;
1844
2151
 
1845
- static constexpr std::size_t shift(std::size_t i = 0) {
2152
+ static constexpr std::size_t shift(std::size_t i = 0) noexcept {
1846
2153
  return sizeof(neighbors_count_t) + sizeof(compressed_slot_t) * i;
1847
2154
  }
1848
2155
 
1849
2156
  public:
2157
+ using iterator = misaligned_ptr_gt<compressed_slot_t>;
2158
+ using const_iterator = misaligned_ptr_gt<compressed_slot_t const>;
2159
+ using value_type = compressed_slot_t;
2160
+
1850
2161
  neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
1851
2162
  misaligned_ptr_gt<compressed_slot_t> begin() noexcept { return tape_ + shift(); }
1852
2163
  misaligned_ptr_gt<compressed_slot_t> end() noexcept { return begin() + size(); }
1853
2164
  misaligned_ptr_gt<compressed_slot_t const> begin() const noexcept { return tape_ + shift(); }
1854
2165
  misaligned_ptr_gt<compressed_slot_t const> end() const noexcept { return begin() + size(); }
2166
+ misaligned_ptr_gt<compressed_slot_t const> cbegin() noexcept { return tape_ + shift(); }
2167
+ misaligned_ptr_gt<compressed_slot_t const> cend() noexcept { return cbegin() + size(); }
1855
2168
  compressed_slot_t operator[](std::size_t i) const noexcept {
1856
2169
  return misaligned_load<compressed_slot_t>(tape_ + shift(i));
1857
2170
  }
@@ -1859,7 +2172,7 @@ class index_gt {
1859
2172
  void clear() noexcept {
1860
2173
  neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1861
2174
  std::memset(tape_, 0, shift(n));
1862
- // misaligned_store<neighbors_count_t>(tape_, 0);
2175
+ misaligned_store<neighbors_count_t>(tape_, 0);
1863
2176
  }
1864
2177
  void push_back(compressed_slot_t slot) noexcept {
1865
2178
  neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
@@ -1879,29 +2192,55 @@ class index_gt {
1879
2192
  visits_hash_set_t visits{};
1880
2193
  std::default_random_engine level_generator{};
1881
2194
  std::size_t iteration_cycles{};
1882
- std::size_t computed_distances_count{};
2195
+ std::size_t computed_distances{};
2196
+ std::size_t computed_distances_in_refines{};
2197
+ std::size_t computed_distances_in_reverse_refines{};
1883
2198
 
2199
+ /// @brief Heterogeneous distance calculation.
1884
2200
  template <typename value_at, typename metric_at, typename entry_at> //
1885
2201
  inline distance_t measure(value_at const& first, entry_at const& second, metric_at&& metric) noexcept {
1886
2202
  static_assert( //
1887
2203
  std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
1888
2204
  "Unexpected type");
1889
2205
 
1890
- computed_distances_count++;
2206
+ computed_distances++;
1891
2207
  return metric(first, second);
1892
2208
  }
1893
2209
 
2210
+ /// @brief Homogeneous distance calculation.
1894
2211
  template <typename metric_at, typename entry_at> //
1895
2212
  inline distance_t measure(entry_at const& first, entry_at const& second, metric_at&& metric) noexcept {
1896
2213
  static_assert( //
1897
2214
  std::is_same<entry_at, member_cref_t>::value || std::is_same<entry_at, member_citerator_t>::value,
1898
2215
  "Unexpected type");
1899
2216
 
1900
- computed_distances_count++;
2217
+ computed_distances++;
1901
2218
  return metric(first, second);
1902
2219
  }
2220
+
2221
+ /// @brief Heterogeneous batch distance calculation.
2222
+ template <typename value_at, typename metric_at, typename entries_at, typename candidate_allowed_at,
2223
+ typename transform_at,
2224
+ typename callback_at> //
2225
+ inline void measure_batch(value_at const& first, entries_at const& second_entries, metric_at&& metric,
2226
+ candidate_allowed_at&& candidate_allowed, transform_at&& transform,
2227
+ callback_at&& callback) noexcept {
2228
+
2229
+ using entry_t = typename std::remove_reference<decltype(second_entries[0])>::type;
2230
+ metric.batch(first, second_entries, candidate_allowed, transform,
2231
+ [&](entry_t const& entry, distance_t distance) {
2232
+ callback(entry, distance);
2233
+ computed_distances++;
2234
+ });
2235
+ }
1903
2236
  };
1904
2237
 
2238
+ /// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
2239
+ mutable std::atomic<std::size_t> nodes_capacity_{};
2240
+
2241
+ /// @brief Number of "slots" already storing non-null nodes.
2242
+ mutable std::atomic<std::size_t> nodes_count_{};
2243
+
1905
2244
  index_config_t config_{};
1906
2245
  index_limits_t limits_{};
1907
2246
 
@@ -1911,12 +2250,6 @@ class index_gt {
1911
2250
  precomputed_constants_t pre_{};
1912
2251
  memory_mapped_file_t viewed_file_{};
1913
2252
 
1914
- /// @brief Number of "slots" available for `node_t` objects. Equals to @b `limits_.members`.
1915
- usearch_align_m mutable std::atomic<std::size_t> nodes_capacity_{};
1916
-
1917
- /// @brief Number of "slots" already storing non-null nodes.
1918
- usearch_align_m mutable std::atomic<std::size_t> nodes_count_{};
1919
-
1920
2253
  /// @brief Controls access to `max_level_` and `entry_slot_`.
1921
2254
  /// If any thread is updating those values, no other threads can `add()` or `search()`.
1922
2255
  std::mutex global_mutex_{};
@@ -1929,7 +2262,7 @@ class index_gt {
1929
2262
 
1930
2263
  using nodes_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<node_t>;
1931
2264
 
1932
- /// @brief C-style array of `node_t` smart-pointers.
2265
+ /// @brief C-style array of `node_t` smart-pointers. Use `compressed_slot_t` for indexing.
1933
2266
  buffer_gt<node_t, nodes_allocator_t> nodes_{};
1934
2267
 
1935
2268
  /// @brief Mutex, that limits concurrent access to `nodes_`.
@@ -1948,17 +2281,33 @@ class index_gt {
1948
2281
  index_config_t const& config() const noexcept { return config_; }
1949
2282
  index_limits_t const& limits() const noexcept { return limits_; }
1950
2283
  bool is_immutable() const noexcept { return bool(viewed_file_); }
2284
+ explicit operator bool() const noexcept { return config_.is_valid(); }
1951
2285
 
1952
2286
  /**
2287
+ * @brief Default index constructor, suitable only for stateless allocators.
2288
+ * @warning Consider `index_gt::make` instead, or explicitly convert to `bool` to check if the index is valid.
1953
2289
  * @section Exceptions
1954
- * Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
2290
+ * Doesn't throw, unless the ::dynamic_allocator's and ::tape_allocator's throw on move-construction.
1955
2291
  */
1956
2292
  explicit index_gt( //
1957
- index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
1958
- tape_allocator_t tape_allocator = {}) noexcept
1959
- : config_(config), limits_(0, 0), dynamic_allocator_(std::move(dynamic_allocator)),
1960
- tape_allocator_(std::move(tape_allocator)), pre_(precompute_(config)), nodes_count_(0u), max_level_(-1),
1961
- entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
2293
+ dynamic_allocator_t dynamic_allocator = {}, tape_allocator_t tape_allocator = {}) noexcept(false)
2294
+ : nodes_capacity_(0u), nodes_count_(0u), config_(), limits_(0, 0),
2295
+ dynamic_allocator_(std::move(dynamic_allocator)), tape_allocator_(std::move(tape_allocator)),
2296
+ pre_(precompute_({})), max_level_(-1), entry_slot_(0u), nodes_(), nodes_mutexes_(), contexts_() {}
2297
+
2298
+ /**
2299
+ * @brief Default index constructor, suitable only for stateless allocators.
2300
+ * @warning Consider `index_gt::make` instead, or explicitly convert to `bool` to check if the index is valid.
2301
+ * @section Exceptions
2302
+ * Doesn't throw, unless the ::dynamic_allocator's and ::tape_allocator's throw on move-construction.
2303
+ */
2304
+ explicit index_gt(index_config_t config, dynamic_allocator_t dynamic_allocator = {},
2305
+ tape_allocator_t tape_allocator = {}) noexcept(false)
2306
+ : index_gt(dynamic_allocator, tape_allocator) {
2307
+ config.validate();
2308
+ config_ = config;
2309
+ pre_ = precompute_(config);
2310
+ }
1962
2311
 
1963
2312
  /**
1964
2313
  * @brief Clones the structure with the same hyper-parameters, but without contents.
@@ -1974,17 +2323,56 @@ class index_gt {
1974
2323
  return *this;
1975
2324
  }
1976
2325
 
1977
- struct copy_result_t {
1978
- error_t error;
2326
+ struct state_result_t {
1979
2327
  index_gt index;
2328
+ error_t error;
1980
2329
 
1981
2330
  explicit operator bool() const noexcept { return !error; }
1982
- copy_result_t failed(error_t message) noexcept {
1983
- error = std::move(message);
1984
- return std::move(*this);
2331
+ state_result_t failed(error_t message) noexcept { return {std::move(index), std::move(message)}; }
2332
+ operator index_gt&&() && {
2333
+ if (error)
2334
+ __usearch_raise_runtime_error(error.what());
2335
+ return std::move(index);
1985
2336
  }
1986
2337
  };
2338
+ using copy_result_t = state_result_t;
1987
2339
 
2340
+ /**
2341
+ * @brief The recommended way to initialize the index, as unlike the constructor,
2342
+ * it can fail with an error message, without raising an exception.
2343
+ *
2344
+ * @param[in] config The configuration specs of the index.
2345
+ * @param[in] dynamic_allocator The allocator for temporary buffers and thread contexts, like priority queues.
2346
+ * @param[in] tape_allocator The allocator for the primary allocations of nodes and vectors.
2347
+ */
2348
+ static state_result_t make( //
2349
+ index_config_t config = {}, dynamic_allocator_t dynamic_allocator = {},
2350
+ tape_allocator_t tape_allocator = {}) noexcept {
2351
+
2352
+ state_result_t result;
2353
+ result.error = config.validate();
2354
+ if (result.error)
2355
+ return result;
2356
+
2357
+ index_gt index;
2358
+ index.config_ = std::move(config);
2359
+ index.dynamic_allocator_ = std::move(dynamic_allocator);
2360
+ index.tape_allocator_ = std::move(tape_allocator);
2361
+ index.pre_ = precompute_(index.config_);
2362
+ index.nodes_count_ = 0u;
2363
+ index.max_level_ = -1;
2364
+ index.entry_slot_ = 0u;
2365
+
2366
+ result.index = std::move(index);
2367
+ return result;
2368
+ }
2369
+
2370
+ /**
2371
+ * @brief The recommended way to copy the index, as unlike the copy-constructor,
2372
+ * it can fail with an error message, without raising an exception.
2373
+ *
2374
+ * @param[in] config The configuration specs for the copy-operation. Currently unused.
2375
+ */
1988
2376
  copy_result_t copy(index_copy_config_t config = {}) const noexcept {
1989
2377
  copy_result_t result;
1990
2378
  index_gt& other = result.index;
@@ -2006,22 +2394,24 @@ class index_gt {
2006
2394
  return result;
2007
2395
  }
2008
2396
 
2009
- member_citerator_t cbegin() const noexcept { return {this, 0}; }
2010
- member_citerator_t cend() const noexcept { return {this, size()}; }
2011
- member_citerator_t begin() const noexcept { return {this, 0}; }
2012
- member_citerator_t end() const noexcept { return {this, size()}; }
2013
- member_iterator_t begin() noexcept { return {this, 0}; }
2014
- member_iterator_t end() noexcept { return {this, size()}; }
2397
+ member_citerator_t cbegin() const noexcept { return {this, static_cast<compressed_slot_t>(0u)}; }
2398
+ member_citerator_t cend() const noexcept { return {this, static_cast<compressed_slot_t>(size())}; }
2399
+ member_citerator_t begin() const noexcept { return {this, static_cast<compressed_slot_t>(0u)}; }
2400
+ member_citerator_t end() const noexcept { return {this, static_cast<compressed_slot_t>(size())}; }
2401
+ member_iterator_t begin() noexcept { return {this, static_cast<compressed_slot_t>(0u)}; }
2402
+ member_iterator_t end() noexcept { return {this, static_cast<compressed_slot_t>(size())}; }
2015
2403
 
2016
- member_ref_t at(std::size_t slot) noexcept { return {nodes_[slot].key(), slot}; }
2017
- member_cref_t at(std::size_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
2018
- member_iterator_t iterator_at(std::size_t slot) noexcept { return {this, slot}; }
2019
- member_citerator_t citerator_at(std::size_t slot) const noexcept { return {this, slot}; }
2404
+ member_ref_t at(compressed_slot_t slot) noexcept { return {nodes_[slot].key(), slot}; }
2405
+ member_cref_t at(compressed_slot_t slot) const noexcept { return {nodes_[slot].ckey(), slot}; }
2406
+ member_iterator_t iterator_at(compressed_slot_t slot) noexcept { return {this, slot}; }
2407
+ member_citerator_t citerator_at(compressed_slot_t slot) const noexcept { return {this, slot}; }
2020
2408
 
2021
2409
  dynamic_allocator_t const& dynamic_allocator() const noexcept { return dynamic_allocator_; }
2022
2410
  tape_allocator_t const& tape_allocator() const noexcept { return tape_allocator_; }
2023
2411
 
2412
+ #if defined(USEARCH_USE_PRAGMA_REGION)
2024
2413
  #pragma region Adjusting Configuration
2414
+ #endif
2025
2415
 
2026
2416
  /**
2027
2417
  * @brief Erases all the vectors from the index.
@@ -2089,13 +2479,20 @@ class index_gt {
2089
2479
  * @brief Increases the `capacity()` of the index to allow adding more vectors.
2090
2480
  * @return `true` on success, `false` on memory allocation errors.
2091
2481
  */
2092
- bool reserve(index_limits_t limits) usearch_noexcept_m {
2482
+ bool try_reserve(index_limits_t limits) usearch_noexcept_m {
2093
2483
 
2094
2484
  if (limits.threads_add <= limits_.threads_add //
2095
2485
  && limits.threads_search <= limits_.threads_search //
2096
2486
  && limits.members <= limits_.members)
2097
2487
  return true;
2098
2488
 
2489
+ // In some cases, we don't want to update the number of members,
2490
+ // just want to make sure that future reserves use the new thread limits.
2491
+ if (!limits.members && !size()) {
2492
+ limits_ = limits;
2493
+ return true;
2494
+ }
2495
+
2099
2496
  nodes_mutexes_t new_mutexes(limits.members);
2100
2497
  buffer_gt<node_t, nodes_allocator_t> new_nodes(limits.members);
2101
2498
  buffer_gt<context_t, contexts_allocator_t> new_contexts(limits.threads());
@@ -2114,16 +2511,27 @@ class index_gt {
2114
2511
  return true;
2115
2512
  }
2116
2513
 
2514
+ /**
2515
+ * @brief Increases the `capacity()` of the index to allow adding more vectors.
2516
+ * @warning Unlike STL, won't throw exceptions on memory allocations, so check the return value.
2517
+ * @return `true` on success, `false` on memory allocation errors.
2518
+ */
2519
+ bool reserve(index_limits_t limits) usearch_noexcept_m { return try_reserve(limits); }
2520
+
2521
+ #if defined(USEARCH_USE_PRAGMA_REGION)
2117
2522
  #pragma endregion
2118
2523
 
2119
2524
  #pragma region Construction and Search
2525
+ #endif
2120
2526
 
2121
2527
  struct add_result_t {
2122
2528
  error_t error{};
2123
2529
  std::size_t new_size{};
2124
2530
  std::size_t visited_members{};
2125
2531
  std::size_t computed_distances{};
2126
- std::size_t slot{};
2532
+ std::size_t computed_distances_in_refines{};
2533
+ std::size_t computed_distances_in_reverse_refines{};
2534
+ compressed_slot_t slot{};
2127
2535
 
2128
2536
  explicit operator bool() const noexcept { return !error; }
2129
2537
  add_result_t failed(error_t message) noexcept {
@@ -2168,15 +2576,15 @@ class index_gt {
2168
2576
  top_candidates_t const* top_{};
2169
2577
 
2170
2578
  friend class index_gt;
2171
- inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept
2172
- : nodes_(index.nodes_), top_(&top) {}
2579
+ inline search_result_t(index_gt const& index, top_candidates_t const* top) noexcept
2580
+ : nodes_(index.nodes_), top_(top) {}
2173
2581
 
2174
2582
  public:
2175
- /** @brief Number of search results found. */
2583
+ /** @brief Number of search results found. */
2176
2584
  std::size_t count{};
2177
- /** @brief Number of graph nodes traversed. */
2585
+ /** @brief Number of graph nodes traversed. */
2178
2586
  std::size_t visited_members{};
2179
- /** @brief Number of times the distances were computed. */
2587
+ /** @brief Number of times the distances were computed. */
2180
2588
  std::size_t computed_distances{};
2181
2589
  error_t error{};
2182
2590
 
@@ -2310,57 +2718,76 @@ class index_gt {
2310
2718
 
2311
2719
  // Determining how much memory to allocate for the node depends on the target level
2312
2720
  std::unique_lock<std::mutex> new_level_lock(global_mutex_);
2313
- level_t max_level_copy = max_level_; // Copy under lock
2314
- std::size_t entry_idx_copy = entry_slot_; // Copy under lock
2315
- level_t target_level = choose_random_level_(context.level_generator);
2721
+ level_t max_level_copy = max_level_; // Copy under lock
2722
+ compressed_slot_t entry_slot_copy = static_cast<compressed_slot_t>(entry_slot_); // Copy under lock
2723
+ level_t new_target_level = choose_random_level_(context.level_generator);
2316
2724
 
2317
2725
  // Make sure we are not overflowing
2318
2726
  std::size_t capacity = nodes_capacity_.load();
2319
- std::size_t new_slot = nodes_count_.fetch_add(1);
2320
- if (new_slot >= capacity) {
2727
+ std::size_t old_size = nodes_count_.fetch_add(1);
2728
+ if (old_size >= capacity) {
2321
2729
  nodes_count_.fetch_sub(1);
2322
2730
  return result.failed("Reserve capacity ahead of insertions!");
2323
2731
  }
2324
2732
 
2325
2733
  // Allocate the neighbors
2326
- node_t node = node_make_(key, target_level);
2327
- if (!node) {
2734
+ node_t new_node = node_make_(key, new_target_level);
2735
+ if (!new_node) {
2328
2736
  nodes_count_.fetch_sub(1);
2329
2737
  return result.failed("Out of memory!");
2330
2738
  }
2331
- if (target_level <= max_level_copy)
2739
+ if (new_target_level <= max_level_copy)
2332
2740
  new_level_lock.unlock();
2333
2741
 
2334
- nodes_[new_slot] = node;
2335
- result.new_size = new_slot + 1;
2336
- result.slot = new_slot;
2337
- callback(at(new_slot));
2338
- node_lock_t new_lock = node_lock_(new_slot);
2742
+ nodes_[old_size] = new_node;
2743
+ result.new_size = old_size + 1;
2744
+ compressed_slot_t new_slot = result.slot = static_cast<compressed_slot_t>(old_size);
2745
+ callback(at(result.slot));
2339
2746
 
2340
2747
  // Do nothing for the first element
2341
- if (!new_slot) {
2342
- entry_slot_ = new_slot;
2343
- max_level_ = target_level;
2748
+ if (!old_size) {
2749
+ entry_slot_ = result.slot;
2750
+ max_level_ = new_target_level;
2344
2751
  return result;
2345
2752
  }
2346
2753
 
2347
2754
  // Pull stats
2348
- result.computed_distances = context.computed_distances_count;
2755
+ result.computed_distances = context.computed_distances;
2756
+ result.computed_distances_in_refines = context.computed_distances_in_refines;
2757
+ result.computed_distances_in_reverse_refines = context.computed_distances_in_reverse_refines;
2349
2758
  result.visited_members = context.iteration_cycles;
2350
2759
 
2351
- connect_node_across_levels_( //
2352
- value, metric, prefetch, //
2353
- new_slot, entry_idx_copy, max_level_copy, target_level, //
2354
- config, context);
2760
+ // Go down the level, tracking only the closest match
2761
+ compressed_slot_t closest_slot = search_for_one_( //
2762
+ value, metric, prefetch, //
2763
+ entry_slot_copy, max_level_copy, new_target_level, context);
2764
+
2765
+ // From `new_target_level` down - perform proper extensive search
2766
+ for (level_t level = (std::min)(new_target_level, max_level_copy); level >= 0; --level) {
2767
+ // TODO: Handle out of memory conditions
2768
+ search_to_insert_(value, metric, prefetch, closest_slot, level, config.expansion, context);
2769
+ candidates_view_t closest_view;
2770
+ {
2771
+ node_lock_t new_lock = node_lock_(new_slot);
2772
+ neighbors_(new_node, level).clear();
2773
+ closest_view = form_links_to_closest_(metric, new_slot, level, context);
2774
+ closest_slot = closest_view[0].slot;
2775
+ }
2776
+ form_reverse_links_(metric, new_slot, closest_view, value, level, context);
2777
+ }
2355
2778
 
2356
2779
  // Normalize stats
2357
- result.computed_distances = context.computed_distances_count - result.computed_distances;
2780
+ result.computed_distances = context.computed_distances - result.computed_distances;
2781
+ result.computed_distances_in_refines =
2782
+ context.computed_distances_in_refines - result.computed_distances_in_refines;
2783
+ result.computed_distances_in_reverse_refines =
2784
+ context.computed_distances_in_reverse_refines - result.computed_distances_in_reverse_refines;
2358
2785
  result.visited_members = context.iteration_cycles - result.visited_members;
2359
2786
 
2360
2787
  // Updating the entry point if needed
2361
- if (target_level > max_level_copy) {
2788
+ if (new_target_level > max_level_copy) {
2362
2789
  entry_slot_ = new_slot;
2363
- max_level_ = target_level;
2790
+ max_level_ = new_target_level;
2364
2791
  }
2365
2792
  return result;
2366
2793
  }
@@ -2368,6 +2795,10 @@ class index_gt {
2368
2795
  /**
2369
2796
  * @brief Update an existing entry. Thread-safe. Supports @b heterogeneous lookups.
2370
2797
  *
2798
+ * ! It's assumed that different threads aren't updating the same entry at the same time.
2799
+ * ! The state won't be corrupted, but no transactional guarantees are provided and the
2800
+ * ! resulting value & neighbors list may be inconsistent.
2801
+ *
2371
2802
  * @tparam metric_at
2372
2803
  * A function responsible for computing the distance @b (dis-similarity) between two objects.
2373
2804
  * It should be callable into distinctly different scenarios:
@@ -2399,9 +2830,13 @@ class index_gt {
2399
2830
  callback_at&& callback = callback_at{}, //
2400
2831
  prefetch_at&& prefetch = prefetch_at{}) usearch_noexcept_m {
2401
2832
 
2833
+ // Someone is gonna fuzz this, so let's make sure we cover the basics
2834
+ if (!config.expansion)
2835
+ config.expansion = default_expansion_add();
2836
+
2402
2837
  usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
2403
2838
  add_result_t result;
2404
- std::size_t old_slot = iterator.slot_;
2839
+ compressed_slot_t updated_slot = iterator.slot_;
2405
2840
 
2406
2841
  // Make sure we have enough local memory to perform this request
2407
2842
  context_t& context = contexts_[config.thread];
@@ -2419,30 +2854,60 @@ class index_gt {
2419
2854
  if (!next.reserve(config.expansion))
2420
2855
  return result.failed("Out of memory!");
2421
2856
 
2422
- node_lock_t new_lock = node_lock_(old_slot);
2423
- node_t node = node_at_(old_slot);
2857
+ node_t updated_node = node_at_(updated_slot);
2858
+ level_t updated_node_level = updated_node.level();
2424
2859
 
2425
- level_t node_level = node.level();
2426
- span_bytes_t node_bytes = node_bytes_(node);
2427
- std::memset(node_bytes.data(), 0, node_bytes.size());
2428
- node.level(node_level);
2860
+ // Copy entry coordinates under locks
2861
+ level_t max_level_copy;
2862
+ compressed_slot_t entry_slot_copy;
2863
+ {
2864
+ std::unique_lock<std::mutex> new_level_lock(global_mutex_);
2865
+ max_level_copy = max_level_; // Copy under lock
2866
+ entry_slot_copy = static_cast<compressed_slot_t>(entry_slot_); // Copy under lock
2867
+ }
2429
2868
 
2430
2869
  // Pull stats
2431
- result.computed_distances = context.computed_distances_count;
2870
+ result.computed_distances = context.computed_distances;
2432
2871
  result.visited_members = context.iteration_cycles;
2433
2872
 
2434
- connect_node_across_levels_( //
2435
- value, metric, prefetch, //
2436
- old_slot, entry_slot_, max_level_, node_level, //
2437
- config, context);
2438
- node.key(key);
2873
+ // If we are updating the entry node itself, it won't contain any neighbors,
2874
+ // so we should traverse a level down to find the closest match.
2875
+ if (updated_node_level == max_level_copy)
2876
+ updated_node_level--;
2877
+
2878
+ // Go down the level, tracking only the closest match;
2879
+ // It may even be equal to the `updated_slot`
2880
+ compressed_slot_t closest_slot = search_for_one_( //
2881
+ value, metric, prefetch, //
2882
+ entry_slot_copy, max_level_copy, updated_node_level, context);
2883
+
2884
+ // From `updated_node_level` down - perform proper extensive search
2885
+ for (level_t level = (std::min)(updated_node_level, max_level_copy); level >= 0; --level) {
2886
+ if (!search_to_update_(value, metric, prefetch, closest_slot, updated_slot, level, config.expansion,
2887
+ context))
2888
+ return result.failed("Out of memory!");
2889
+
2890
+ candidates_view_t closest_view;
2891
+ {
2892
+ node_lock_t updated_lock = node_lock_(updated_slot);
2893
+ // TODO: Go through existing neighbors removing reverse links
2894
+ // for (compressed_slot_t slot : neighbors_(updated_node, level))
2895
+ // remove_link_(slot, updated_slot, level);
2896
+ neighbors_(updated_node, level).clear();
2897
+ closest_view = form_links_to_closest_(metric, updated_slot, level, context);
2898
+ if (closest_view.size())
2899
+ closest_slot = closest_view[0].slot;
2900
+ }
2901
+ form_reverse_links_(metric, updated_slot, closest_view, value, level, context);
2902
+ }
2903
+ updated_node.key(key);
2439
2904
 
2440
2905
  // Normalize stats
2441
- result.computed_distances = context.computed_distances_count - result.computed_distances;
2906
+ result.computed_distances = context.computed_distances - result.computed_distances;
2442
2907
  result.visited_members = context.iteration_cycles - result.visited_members;
2443
- result.slot = old_slot;
2908
+ result.slot = updated_slot;
2444
2909
 
2445
- callback(at(old_slot));
2910
+ callback(at(updated_slot));
2446
2911
  return result;
2447
2912
  }
2448
2913
 
@@ -2467,16 +2932,28 @@ class index_gt {
2467
2932
  metric_at&& metric, //
2468
2933
  index_search_config_t config = {}, //
2469
2934
  predicate_at&& predicate = predicate_at{}, //
2470
- prefetch_at&& prefetch = prefetch_at{}) const noexcept {
2935
+ prefetch_at&& prefetch = prefetch_at{}) const usearch_noexcept_m {
2471
2936
 
2472
- context_t& context = contexts_[config.thread];
2473
- top_candidates_t& top = context.top_candidates;
2474
- search_result_t result{*this, top};
2475
- if (!nodes_count_)
2937
+ // Someone is gonna fuzz this, so let's make sure we cover the basics
2938
+ if (!wanted)
2939
+ return search_result_t{};
2940
+
2941
+ // Expansion factor set to zero is equivalent to the default value
2942
+ if (!config.expansion)
2943
+ config.expansion = default_expansion_search();
2944
+
2945
+ // Using references is cleaner, but would result in UBSan false positives
2946
+ context_t* context_ptr = contexts_.data() + config.thread;
2947
+ top_candidates_t* top_ptr = context_ptr ? &context_ptr->top_candidates : nullptr;
2948
+ search_result_t result{*this, top_ptr};
2949
+ if (!nodes_count_.load(std::memory_order_relaxed))
2476
2950
  return result;
2477
2951
 
2952
+ usearch_assert_m(contexts_.size() > config.thread, "Thread index out of bounds");
2953
+ context_t& context = *context_ptr;
2954
+ top_candidates_t& top = *top_ptr;
2478
2955
  // Go down the level, tracking only the closest match
2479
- result.computed_distances = context.computed_distances_count;
2956
+ result.computed_distances = context.computed_distances;
2480
2957
  result.visited_members = context.iteration_cycles;
2481
2958
 
2482
2959
  if (config.exact) {
@@ -2486,12 +2963,14 @@ class index_gt {
2486
2963
  } else {
2487
2964
  next_candidates_t& next = context.next_candidates;
2488
2965
  std::size_t expansion = (std::max)(config.expansion, wanted);
2966
+ usearch_assert_m(expansion > 0, "Expansion factor can't be a zero!");
2489
2967
  if (!next.reserve(expansion))
2490
2968
  return result.failed("Out of memory!");
2491
2969
  if (!top.reserve(expansion))
2492
2970
  return result.failed("Out of memory!");
2493
2971
 
2494
- std::size_t closest_slot = search_for_one_(query, metric, prefetch, entry_slot_, max_level_, 0, context);
2972
+ compressed_slot_t closest_slot = search_for_one_(
2973
+ query, metric, prefetch, static_cast<compressed_slot_t>(entry_slot_), max_level_, 0, context);
2495
2974
 
2496
2975
  // For bottom layer we need a more optimized procedure
2497
2976
  if (!search_to_find_in_base_(query, metric, predicate, prefetch, closest_slot, expansion, context))
@@ -2502,7 +2981,7 @@ class index_gt {
2502
2981
  top.shrink(wanted);
2503
2982
 
2504
2983
  // Normalize stats
2505
- result.computed_distances = context.computed_distances_count - result.computed_distances;
2984
+ result.computed_distances = context.computed_distances - result.computed_distances;
2506
2985
  result.visited_members = context.iteration_cycles - result.visited_members;
2507
2986
  result.count = top.size();
2508
2987
  return result;
@@ -2537,7 +3016,7 @@ class index_gt {
2537
3016
  return result.failed("No clusters to identify");
2538
3017
 
2539
3018
  // Go down the level, tracking only the closest match
2540
- result.computed_distances = context.computed_distances_count;
3019
+ result.computed_distances = context.computed_distances;
2541
3020
  result.visited_members = context.iteration_cycles;
2542
3021
 
2543
3022
  next_candidates_t& next = context.next_candidates;
@@ -2545,21 +3024,24 @@ class index_gt {
2545
3024
  if (!next.reserve(expansion))
2546
3025
  return result.failed("Out of memory!");
2547
3026
 
2548
- result.cluster.member = at(search_for_one_(query, metric, prefetch, entry_slot_, max_level_,
2549
- static_cast<level_t>(level - 1), context));
3027
+ result.cluster.member =
3028
+ at(search_for_one_(query, metric, prefetch, static_cast<compressed_slot_t>(entry_slot_), max_level_,
3029
+ static_cast<level_t>(level <= 0 ? 0 : level - 1), context));
2550
3030
  result.cluster.distance = context.measure(query, result.cluster.member, metric);
2551
3031
 
2552
3032
  // Normalize stats
2553
- result.computed_distances = context.computed_distances_count - result.computed_distances;
3033
+ result.computed_distances = context.computed_distances - result.computed_distances;
2554
3034
  result.visited_members = context.iteration_cycles - result.visited_members;
2555
3035
 
2556
3036
  (void)predicate;
2557
3037
  return result;
2558
3038
  }
2559
3039
 
3040
+ #if defined(USEARCH_USE_PRAGMA_REGION)
2560
3041
  #pragma endregion
2561
3042
 
2562
3043
  #pragma region Metadata
3044
+ #endif
2563
3045
 
2564
3046
  struct stats_t {
2565
3047
  std::size_t nodes{};
@@ -2568,6 +3050,9 @@ class index_gt {
2568
3050
  std::size_t allocated_bytes{};
2569
3051
  };
2570
3052
 
3053
+ /**
3054
+ * @brief Aggregates stats on the number of nodes, edges, and memory usage across all levels.
3055
+ */
2571
3056
  stats_t stats() const noexcept {
2572
3057
  stats_t result{};
2573
3058
 
@@ -2586,10 +3071,17 @@ class index_gt {
2586
3071
  return result;
2587
3072
  }
2588
3073
 
3074
+ /**
3075
+ * @brief Aggregates stats on the number of nodes, edges, and memory usage up to a specific level.
3076
+ *
3077
+ * The `level` parameter is zero-based, where `0` is the base level.
3078
+ * For example, `level=1` will include the base level and the first level of connections.
3079
+ */
2589
3080
  stats_t stats(std::size_t level) const noexcept {
2590
3081
  stats_t result{};
2591
-
2592
3082
  std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
3083
+ std::size_t max_edges_per_node = !level ? config_.connectivity_base : config_.connectivity;
3084
+
2593
3085
  for (std::size_t i = 0; i != size(); ++i) {
2594
3086
  node_t node = node_at_(i);
2595
3087
  if (static_cast<std::size_t>(node.level()) < level)
@@ -2600,11 +3092,17 @@ class index_gt {
2600
3092
  result.allocated_bytes += node_head_bytes_() + neighbors_bytes;
2601
3093
  }
2602
3094
 
2603
- std::size_t max_edges_per_node = level ? config_.connectivity_base : config_.connectivity;
2604
3095
  result.max_edges = result.nodes * max_edges_per_node;
2605
3096
  return result;
2606
3097
  }
2607
3098
 
3099
+ /**
3100
+ * @brief Aggregates stats on the number of nodes, edges, and memory usage up to a specific level,
3101
+ * simultaneously exporting the stats for each level into the `stats_per_level` C-style array.
3102
+ *
3103
+ * The `max_level` parameter is zero-based, where `0` is the base level.
3104
+ * For example, `max_level=1` will include the base level and the first level of connections.
3105
+ */
2608
3106
  stats_t stats(stats_t* stats_per_level, std::size_t max_level) const noexcept {
2609
3107
 
2610
3108
  std::size_t head_bytes = node_head_bytes_();
@@ -2663,9 +3161,11 @@ class index_gt {
2663
3161
 
2664
3162
  std::size_t memory_usage_per_node(level_t level) const noexcept { return node_bytes_(level); }
2665
3163
 
3164
+ #if defined(USEARCH_USE_PRAGMA_REGION)
2666
3165
  #pragma endregion
2667
3166
 
2668
3167
  #pragma region Serialization
3168
+ #endif
2669
3169
 
2670
3170
  /**
2671
3171
  * @brief Estimate the binary length (in bytes) of the serialized index.
@@ -2732,6 +3232,7 @@ class index_gt {
2732
3232
  serialization_result_t result;
2733
3233
 
2734
3234
  // Remove previously stored objects
3235
+ index_limits_t old_limits = limits_;
2735
3236
  reset();
2736
3237
 
2737
3238
  // Pull basic metadata
@@ -2756,9 +3257,15 @@ class index_gt {
2756
3257
  // Submit metadata
2757
3258
  config_.connectivity = header.connectivity;
2758
3259
  config_.connectivity_base = header.connectivity_base;
3260
+ error_t error = config_.validate();
3261
+ if (error)
3262
+ return result.failed(std::move(error));
3263
+
2759
3264
  pre_ = precompute_(config_);
2760
3265
  index_limits_t limits;
2761
3266
  limits.members = header.size;
3267
+ limits.threads_add = (std::max<std::size_t>)(1, old_limits.threads_add);
3268
+ limits.threads_search = (std::max<std::size_t>)(1, old_limits.threads_search);
2762
3269
  if (!reserve(limits)) {
2763
3270
  reset();
2764
3271
  return result.failed("Out of memory");
@@ -2808,8 +3315,12 @@ class index_gt {
2808
3315
  },
2809
3316
  std::forward<progress_at>(progress));
2810
3317
 
2811
- if (!stream_result)
3318
+ if (!stream_result) {
3319
+ // Drop generic messages like "end of file reached" in favor
3320
+ // of more specific messages from the stream
3321
+ io_result.error.release();
2812
3322
  return stream_result;
3323
+ }
2813
3324
  return io_result;
2814
3325
  }
2815
3326
 
@@ -2857,8 +3368,12 @@ class index_gt {
2857
3368
  },
2858
3369
  std::forward<progress_at>(progress));
2859
3370
 
2860
- if (!stream_result)
3371
+ if (!stream_result) {
3372
+ // Drop generic messages like "end of file reached" in favor
3373
+ // of more specific messages from the stream
3374
+ io_result.error.release();
2861
3375
  return stream_result;
3376
+ }
2862
3377
  return io_result;
2863
3378
  }
2864
3379
 
@@ -2897,6 +3412,7 @@ class index_gt {
2897
3412
  progress_at&& progress = {}) noexcept {
2898
3413
 
2899
3414
  // Remove previously stored objects
3415
+ index_limits_t old_limits = limits_;
2900
3416
  reset();
2901
3417
 
2902
3418
  serialization_result_t result = file.open_if_not();
@@ -2923,6 +3439,10 @@ class index_gt {
2923
3439
 
2924
3440
  config_.connectivity = header.connectivity;
2925
3441
  config_.connectivity_base = header.connectivity_base;
3442
+ error_t error = config_.validate();
3443
+ if (error)
3444
+ return result.failed(std::move(error));
3445
+
2926
3446
  pre_ = precompute_(config_);
2927
3447
  misaligned_ptr_gt<level_t> levels{(byte_t*)file.data() + offset + sizeof(header)};
2928
3448
  offsets[0u] = offset + sizeof(header) + sizeof(level_t) * header.size;
@@ -2938,6 +3458,8 @@ class index_gt {
2938
3458
  // Submit metadata and reserve memory
2939
3459
  index_limits_t limits;
2940
3460
  limits.members = header.size;
3461
+ limits.threads_add = (std::max<std::size_t>)(1, old_limits.threads_add);
3462
+ limits.threads_search = (std::max<std::size_t>)(1, old_limits.threads_search);
2941
3463
  if (!reserve(limits)) {
2942
3464
  reset();
2943
3465
  return result.failed("Out of memory");
@@ -2956,23 +3478,21 @@ class index_gt {
2956
3478
  return {};
2957
3479
  }
2958
3480
 
3481
+ #if defined(USEARCH_USE_PRAGMA_REGION)
2959
3482
  #pragma endregion
3483
+ #endif
2960
3484
 
2961
3485
  /**
2962
3486
  * @brief Performs compaction on the whole HNSW index, purging some entries
2963
3487
  * and links to them, while also generating a more efficient mapping,
2964
3488
  * putting the more frequently used entries closer together.
2965
3489
  *
2966
- *
2967
- * Scans the whole collection, removing the links leading towards
2968
- * banned entries. This essentially isolates some nodes from the rest
2969
- * of the graph, while keeping their outgoing links, in case the node
2970
- * is structurally relevant and has a crucial role in the index.
2971
- * It won't reclaim the memory.
2972
- *
2973
- * @param[in] allow_member Predicate to mark nodes for isolation.
3490
+ * @param[in] values A []-subscriptable object, providing access to the values.
3491
+ * @param[in] metric Callable object measuring distance between any ::values and present objects.
3492
+ * @param[in] slot_transition Callable object to inform changes in slot assignments.
2974
3493
  * @param[in] executor Thread-pool to execute the job in parallel.
2975
3494
  * @param[in] progress Callback to report the execution progress.
3495
+ * @param[in] prefetch Callable object to prefetch data into the cache.
2976
3496
  */
2977
3497
  template <typename values_at, typename metric_at, //
2978
3498
  typename slot_transition_at = dummy_key_to_key_mapping_t, //
@@ -3006,16 +3526,14 @@ class index_gt {
3006
3526
  std::size_t const total = 3 * slots_and_levels.size();
3007
3527
 
3008
3528
  // For every bottom level node, determine its parent cluster
3009
- executor.dynamic(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
3529
+ executor.dynamic(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot_as_uint) {
3010
3530
  context_t& context = contexts_[thread_idx];
3011
- std::size_t cluster = search_for_one_( //
3012
- values[citerator_at(old_slot)], //
3013
- metric, prefetch, //
3014
- entry_slot_, max_level_, 0, context);
3015
- slots_and_levels[old_slot] = { //
3016
- static_cast<compressed_slot_t>(old_slot), //
3017
- static_cast<compressed_slot_t>(cluster), //
3018
- node_at_(old_slot).level()};
3531
+ compressed_slot_t old_slot = static_cast<compressed_slot_t>(old_slot_as_uint);
3532
+ compressed_slot_t cluster = search_for_one_( //
3533
+ values[citerator_at(old_slot)], //
3534
+ metric, prefetch, //
3535
+ static_cast<compressed_slot_t>(entry_slot_), max_level_, 0, context);
3536
+ slots_and_levels[old_slot] = {old_slot, cluster, node_at_(old_slot).level()};
3019
3537
  ++processed;
3020
3538
  if (thread_idx == 0)
3021
3539
  do_tasks = progress(processed.load(), total);
@@ -3198,90 +3716,90 @@ class index_gt {
3198
3716
  return {nodes_mutexes_, slot};
3199
3717
  }
3200
3718
 
3201
- template <typename value_at, typename metric_at, typename prefetch_at>
3202
- void connect_node_across_levels_( //
3203
- value_at&& value, metric_at&& metric, prefetch_at&& prefetch, //
3204
- std::size_t node_slot, std::size_t entry_slot, level_t max_level, level_t target_level, //
3205
- index_update_config_t const& config, context_t& context) usearch_noexcept_m {
3206
-
3207
- // Go down the level, tracking only the closest match
3208
- std::size_t closest_slot = search_for_one_( //
3209
- value, metric, prefetch, //
3210
- entry_slot, max_level, target_level, context);
3211
-
3212
- // From `target_level` down perform proper extensive search
3213
- for (level_t level = (std::min)(target_level, max_level); level >= 0; --level) {
3214
- // TODO: Handle out of memory conditions
3215
- search_to_insert_(value, metric, prefetch, closest_slot, node_slot, level, config.expansion, context);
3216
- closest_slot = connect_new_node_(metric, node_slot, level, context);
3217
- reconnect_neighbor_nodes_(metric, node_slot, value, level, context);
3719
+ struct node_conditional_lock_t {
3720
+ nodes_mutexes_t& mutexes;
3721
+ std::size_t slot;
3722
+ inline ~node_conditional_lock_t() noexcept {
3723
+ if (slot != std::numeric_limits<std::size_t>::max())
3724
+ mutexes.atomic_reset(slot);
3218
3725
  }
3726
+ };
3727
+
3728
+ inline node_conditional_lock_t node_try_conditional_lock_(std::size_t slot, bool condition,
3729
+ bool& failed_to_acquire) const noexcept {
3730
+ failed_to_acquire = condition ? nodes_mutexes_.atomic_set(slot) : false;
3731
+ return {nodes_mutexes_, failed_to_acquire ? std::numeric_limits<std::size_t>::max() : slot};
3219
3732
  }
3220
3733
 
3221
- template <typename metric_at>
3222
- std::size_t connect_new_node_( //
3734
+ template <typename metric_at, bool require_non_empty_ak = false>
3735
+ candidates_view_t form_links_to_closest_( //
3223
3736
  metric_at&& metric, std::size_t new_slot, level_t level, context_t& context) usearch_noexcept_m {
3224
3737
 
3225
3738
  node_t new_node = node_at_(new_slot);
3226
3739
  top_candidates_t& top = context.top_candidates;
3740
+ usearch_assert_m(top.size() || !require_non_empty_ak, "No candidates found");
3741
+ candidates_view_t top_view =
3742
+ refine_(metric, config_.connectivity, top, context, context.computed_distances_in_refines);
3743
+ usearch_assert_m(top_view.size() || !require_non_empty_ak, "This would lead to isolated nodes");
3227
3744
 
3228
3745
  // Outgoing links from `new_slot`:
3229
3746
  neighbors_ref_t new_neighbors = neighbors_(new_node, level);
3230
- {
3231
- usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
3232
- candidates_view_t top_view = refine_(metric, config_.connectivity, top, context);
3233
-
3234
- for (std::size_t idx = 0; idx != top_view.size(); idx++) {
3235
- usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
3236
- usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
3237
- new_neighbors.push_back(top_view[idx].slot);
3238
- }
3747
+ usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
3748
+ for (std::size_t idx = 0; idx != top_view.size(); idx++) {
3749
+ usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
3750
+ usearch_assert_m(level <= node_at_(top_view[idx].slot).level(), "Linking to missing level");
3751
+ new_neighbors.push_back(top_view[idx].slot);
3239
3752
  }
3240
3753
 
3241
- return new_neighbors[0];
3754
+ return top_view;
3242
3755
  }
3243
3756
 
3244
3757
  template <typename value_at, typename metric_at>
3245
- void reconnect_neighbor_nodes_( //
3246
- metric_at&& metric, std::size_t new_slot, value_at&& value, level_t level,
3247
- context_t& context) usearch_noexcept_m {
3758
+ void form_reverse_links_( //
3759
+ metric_at&& metric, compressed_slot_t new_slot, candidates_view_t new_neighbors, value_at&& value,
3760
+ level_t level, context_t& context) usearch_noexcept_m {
3248
3761
 
3249
- node_t new_node = node_at_(new_slot);
3250
3762
  top_candidates_t& top = context.top_candidates;
3251
- neighbors_ref_t new_neighbors = neighbors_(new_node, level);
3763
+ std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
3252
3764
 
3253
3765
  // Reverse links from the neighbors:
3254
- std::size_t const connectivity_max = level ? config_.connectivity : config_.connectivity_base;
3255
- for (compressed_slot_t close_slot : new_neighbors) {
3766
+ for (auto new_neighbor : new_neighbors) {
3767
+ compressed_slot_t close_slot = new_neighbor.slot;
3256
3768
  if (close_slot == new_slot)
3257
3769
  continue;
3258
3770
  node_lock_t close_lock = node_lock_(close_slot);
3259
3771
  node_t close_node = node_at_(close_slot);
3260
-
3261
3772
  neighbors_ref_t close_header = neighbors_(close_node, level);
3262
- usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
3773
+
3774
+ // The node may have no neighbors only in one case, when it's the first one in the index,
3775
+ // but that is problematic to track in multi-threaded environments, where the order of insertion
3776
+ // is not guaranteed.
3777
+ // usearch_assert_m(close_header.size() || new_slot == 1, "Possible corruption - isolated node");
3778
+ usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption - overflow");
3263
3779
  usearch_assert_m(close_slot != new_slot, "Self-loops are impossible");
3264
3780
  usearch_assert_m(level <= close_node.level(), "Linking to missing level");
3265
3781
 
3266
3782
  // If `new_slot` is already present in the neighboring connections of `close_slot`
3267
3783
  // then no need to modify any connections or run the heuristics.
3268
3784
  if (close_header.size() < connectivity_max) {
3269
- close_header.push_back(static_cast<compressed_slot_t>(new_slot));
3785
+ close_header.push_back(new_slot);
3270
3786
  continue;
3271
3787
  }
3272
3788
 
3273
3789
  // To fit a new connection we need to drop an existing one.
3274
3790
  top.clear();
3275
- usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
3276
- top.insert_reserved(
3277
- {context.measure(value, citerator_at(close_slot), metric), static_cast<compressed_slot_t>(new_slot)});
3791
+ usearch_assert_m((top.capacity() >= (close_header.size() + 1)),
3792
+ "The memory must have been reserved in `add`");
3793
+ top.insert_reserved({context.measure(value, citerator_at(close_slot), metric), new_slot});
3278
3794
  for (compressed_slot_t successor_slot : close_header)
3279
3795
  top.insert_reserved(
3280
3796
  {context.measure(citerator_at(close_slot), citerator_at(successor_slot), metric), successor_slot});
3281
3797
 
3282
3798
  // Export the results:
3283
3799
  close_header.clear();
3284
- candidates_view_t top_view = refine_(metric, connectivity_max, top, context);
3800
+ candidates_view_t top_view =
3801
+ refine_(metric, connectivity_max, top, context, context.computed_distances_in_reverse_refines);
3802
+ usearch_assert_m(top_view.size(), "This would lead to isolated nodes");
3285
3803
  for (std::size_t idx = 0; idx != top_view.size(); idx++)
3286
3804
  close_header.push_back(top_view[idx].slot);
3287
3805
  }
@@ -3323,12 +3841,12 @@ class index_gt {
3323
3841
  using pointer = misaligned_ptr_gt<element_t>;
3324
3842
  using reference = misaligned_ref_gt<element_t>;
3325
3843
 
3326
- reference operator*() const noexcept { return slot(); }
3844
+ value_type operator*() const noexcept { return neighbors_[current_]; }
3327
3845
  candidates_iterator_t(index_gt const& index, neighbors_ref_t neighbors, visits_hash_set_t& visits,
3328
3846
  std::size_t progress) noexcept
3329
3847
  : index_(index), neighbors_(neighbors), visits_(visits), current_(progress) {}
3330
3848
  candidates_iterator_t operator++(int) noexcept {
3331
- return candidates_iterator_t(index_, visits_, neighbors_, current_ + 1).skip_missing();
3849
+ return candidates_iterator_t(index_, neighbors_, visits_, current_ + 1).skip_missing();
3332
3850
  }
3333
3851
  candidates_iterator_t& operator++() noexcept {
3334
3852
  ++current_;
@@ -3338,7 +3856,7 @@ class index_gt {
3338
3856
  bool operator==(candidates_iterator_t const& other) noexcept { return current_ == other.current_; }
3339
3857
  bool operator!=(candidates_iterator_t const& other) noexcept { return current_ != other.current_; }
3340
3858
 
3341
- vector_key_t key() const noexcept { return index_->node_at_(slot()).key(); }
3859
+ vector_key_t key() const noexcept { return index_.node_at_(slot()).key(); }
3342
3860
  compressed_slot_t slot() const noexcept { return neighbors_[current_]; }
3343
3861
  friend inline std::size_t get_slot(candidates_iterator_t const& it) noexcept { return it.slot(); }
3344
3862
  friend inline vector_key_t get_key(candidates_iterator_t const& it) noexcept { return it.key(); }
@@ -3356,16 +3874,16 @@ class index_gt {
3356
3874
  };
3357
3875
 
3358
3876
  template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
3359
- std::size_t search_for_one_( //
3877
+ compressed_slot_t search_for_one_( //
3360
3878
  value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
3361
- std::size_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
3879
+ compressed_slot_t closest_slot, level_t begin_level, level_t end_level, context_t& context) const noexcept {
3362
3880
 
3363
3881
  visits_hash_set_t& visits = context.visits;
3364
3882
  visits.clear();
3365
3883
 
3366
3884
  // Optional prefetching
3367
3885
  if (!is_dummy<prefetch_at>())
3368
- prefetch(citerator_at(closest_slot), citerator_at(closest_slot + 1));
3886
+ prefetch(citerator_at(closest_slot), citerator_at(closest_slot) + 1);
3369
3887
 
3370
3888
  distance_t closest_dist = context.measure(query, citerator_at(closest_slot), metric);
3371
3889
  for (level_t level = begin_level; level > end_level; --level) {
@@ -3390,6 +3908,7 @@ class index_gt {
3390
3908
  changed = true;
3391
3909
  }
3392
3910
  }
3911
+
3393
3912
  context.iteration_cycles++;
3394
3913
  } while (changed);
3395
3914
  }
@@ -3404,8 +3923,7 @@ class index_gt {
3404
3923
  template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
3405
3924
  bool search_to_insert_( //
3406
3925
  value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
3407
- std::size_t start_slot, std::size_t new_slot, level_t level, std::size_t top_limit,
3408
- context_t& context) noexcept {
3926
+ compressed_slot_t start_slot, level_t level, std::size_t top_limit, context_t& context) noexcept {
3409
3927
 
3410
3928
  visits_hash_set_t& visits = context.visits;
3411
3929
  next_candidates_t& next = context.next_candidates; // pop min, push
@@ -3414,18 +3932,21 @@ class index_gt {
3414
3932
  visits.clear();
3415
3933
  next.clear();
3416
3934
  top.clear();
3935
+
3936
+ // At the very least we are going to explore the starting node and its neighbors
3417
3937
  if (!visits.reserve(config_.connectivity_base + 1u))
3418
3938
  return false;
3419
3939
 
3420
3940
  // Optional prefetching
3421
3941
  if (!is_dummy<prefetch_at>())
3422
- prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
3942
+ prefetch(citerator_at(start_slot), citerator_at(start_slot) + 1);
3423
3943
 
3424
3944
  distance_t radius = context.measure(query, citerator_at(start_slot), metric);
3425
- next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
3426
- top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
3427
- visits.set(static_cast<compressed_slot_t>(start_slot));
3945
+ next.insert_reserved({-radius, start_slot});
3946
+ top.insert_reserved({radius, start_slot});
3947
+ visits.set(start_slot);
3428
3948
 
3949
+ // The primary loop of the graph traversal
3429
3950
  while (!next.empty()) {
3430
3951
 
3431
3952
  candidate_t candidacy = next.top();
@@ -3436,8 +3957,6 @@ class index_gt {
3436
3957
  context.iteration_cycles++;
3437
3958
 
3438
3959
  compressed_slot_t candidate_slot = candidacy.slot;
3439
- if (new_slot == candidate_slot)
3440
- continue;
3441
3960
  node_t candidate_ref = node_at_(candidate_slot);
3442
3961
  node_lock_t candidate_lock = node_lock_(candidate_slot);
3443
3962
  neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
@@ -3456,6 +3975,8 @@ class index_gt {
3456
3975
  if (visits.set(successor_slot))
3457
3976
  continue;
3458
3977
 
3978
+ // We don't access the neighbors of the `successor_slot` node,
3979
+ // so we don't have to lock it.
3459
3980
  // node_lock_t successor_lock = node_lock_(successor_slot);
3460
3981
  distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
3461
3982
  if (top.size() < top_limit || successor_dist < radius) {
@@ -3470,6 +3991,95 @@ class index_gt {
3470
3991
  return true;
3471
3992
  }
3472
3993
 
3994
+ /**
3995
+ * @brief Traverses a layer of a graph, to find the best neighbors list for updated node.
3996
+ * Locks the nodes in the process, assuming other threads are updating neighbors lists.
3997
+ * @return `true` if procedure succeeded, `false` if run out of memory.
3998
+ */
3999
+ template <typename value_at, typename metric_at, typename prefetch_at = dummy_prefetch_t>
4000
+ bool search_to_update_( //
4001
+ value_at&& query, metric_at&& metric, prefetch_at&& prefetch, //
4002
+ compressed_slot_t start_slot, compressed_slot_t updated_slot, level_t level, std::size_t top_limit,
4003
+ context_t& context) noexcept {
4004
+
4005
+ visits_hash_set_t& visits = context.visits;
4006
+ next_candidates_t& next = context.next_candidates; // pop min, push
4007
+ top_candidates_t& top = context.top_candidates; // pop max, push
4008
+
4009
+ visits.clear();
4010
+ next.clear();
4011
+ top.clear();
4012
+
4013
+ // At the very least we are going to explore the starting node and its neighbors
4014
+ if (!visits.reserve(config_.connectivity_base + 1u))
4015
+ return false;
4016
+
4017
+ // Optional prefetching
4018
+ if (!is_dummy<prefetch_at>())
4019
+ prefetch(citerator_at(start_slot), citerator_at(start_slot) + 1);
4020
+
4021
+ distance_t radius = context.measure(query, citerator_at(start_slot), metric);
4022
+ next.insert_reserved({-radius, start_slot});
4023
+ visits.set(start_slot);
4024
+ if (start_slot != updated_slot)
4025
+ top.insert_reserved({radius, start_slot});
4026
+
4027
+ // The primary loop of the graph traversal
4028
+ while (!next.empty()) {
4029
+
4030
+ candidate_t candidacy = next.top();
4031
+ if ((-candidacy.distance) > radius && top.size() == top_limit)
4032
+ break;
4033
+
4034
+ next.pop();
4035
+ context.iteration_cycles++;
4036
+
4037
+ compressed_slot_t candidate_slot = candidacy.slot;
4038
+ node_t candidate_ref = node_at_(candidate_slot);
4039
+
4040
+ // The trickiest part of update-heavy workloads is mitigating dead-locks
4041
+ // in connected nodes during traversal. A "good enough" solution would be
4042
+ // to skip concurrent access, assuming the other "close" node is gonna add
4043
+ // this one when forming reverse connections.
4044
+ bool failed_to_acquire = false;
4045
+ node_conditional_lock_t candidate_lock =
4046
+ node_try_conditional_lock_(candidate_slot, updated_slot != candidate_slot, failed_to_acquire);
4047
+ if (failed_to_acquire)
4048
+ continue;
4049
+ neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
4050
+
4051
+ // Optional prefetching
4052
+ if (!is_dummy<prefetch_at>()) {
4053
+ candidates_range_t missing_candidates{*this, candidate_neighbors, visits};
4054
+ prefetch(missing_candidates.begin(), missing_candidates.end());
4055
+ }
4056
+
4057
+ // Assume the worst-case when reserving memory
4058
+ if (!visits.reserve(visits.size() + candidate_neighbors.size()))
4059
+ return false;
4060
+
4061
+ for (compressed_slot_t successor_slot : candidate_neighbors) {
4062
+ if (visits.set(successor_slot))
4063
+ continue;
4064
+
4065
+ // We don't access the neighbors of the `successor_slot` node,
4066
+ // so we don't have to lock it.
4067
+ // node_conditional_lock_t successor_lock =
4068
+ // node_try_conditional_lock_(successor_slot, updated_slot != successor_slot);
4069
+ distance_t successor_dist = context.measure(query, citerator_at(successor_slot), metric);
4070
+ if (top.size() < top_limit || successor_dist < radius) {
4071
+ // This can substantially grow our priority queue:
4072
+ next.insert({-successor_dist, successor_slot});
4073
+ // This will automatically evict poor matches:
4074
+ if (updated_slot != successor_slot)
4075
+ top.insert({successor_dist, successor_slot}, top_limit);
4076
+ radius = top.top().distance;
4077
+ }
4078
+ }
4079
+ }
4080
+ return true;
4081
+ }
4082
+
3473
4083
  /**
3474
4084
  * @brief Traverses the @b base layer of a graph, to find a close match.
3475
4085
  * Doesn't lock any nodes, assuming read-only simultaneous access.
@@ -3478,7 +4088,7 @@ class index_gt {
3478
4088
  template <typename value_at, typename metric_at, typename predicate_at, typename prefetch_at>
3479
4089
  bool search_to_find_in_base_( //
3480
4090
  value_at&& query, metric_at&& metric, predicate_at&& predicate, prefetch_at&& prefetch, //
3481
- std::size_t start_slot, std::size_t expansion, context_t& context) const noexcept {
4091
+ compressed_slot_t start_slot, std::size_t expansion, context_t& context) const usearch_noexcept_m {
3482
4092
 
3483
4093
  visits_hash_set_t& visits = context.visits;
3484
4094
  next_candidates_t& next = context.next_candidates; // pop min, push
@@ -3493,20 +4103,24 @@ class index_gt {
3493
4103
 
3494
4104
  // Optional prefetching
3495
4105
  if (!is_dummy<prefetch_at>())
3496
- prefetch(citerator_at(start_slot), citerator_at(start_slot + 1));
4106
+ prefetch(citerator_at(start_slot), citerator_at(start_slot) + 1);
3497
4107
 
3498
4108
  distance_t radius = context.measure(query, citerator_at(start_slot), metric);
3499
- next.insert_reserved({-radius, static_cast<compressed_slot_t>(start_slot)});
3500
- visits.set(static_cast<compressed_slot_t>(start_slot));
4109
+ usearch_assert_m(next.capacity(), "The `max_heap_gt` must have been reserved in the search entry point");
4110
+ next.insert_reserved({-radius, start_slot});
4111
+ visits.set(start_slot);
3501
4112
 
3502
4113
  // Don't populate the top list if the predicate is not satisfied
3503
- if (is_dummy<predicate_at>() || predicate(member_cref_t{node_at_(start_slot).ckey(), start_slot}))
3504
- top.insert_reserved({radius, static_cast<compressed_slot_t>(start_slot)});
4114
+ if (is_dummy<predicate_at>() || predicate(member_cref_t{node_at_(start_slot).ckey(), start_slot})) {
4115
+ usearch_assert_m(top.capacity(),
4116
+ "The `sorted_buffer_gt` must have been reserved in the search entry point");
4117
+ top.insert_reserved({radius, start_slot});
4118
+ }
3505
4119
 
3506
4120
  while (!next.empty()) {
3507
4121
 
3508
4122
  candidate_t candidate = next.top();
3509
- if ((-candidate.distance) > radius)
4123
+ if ((-candidate.distance) > radius && top.size() == top_limit)
3510
4124
  break;
3511
4125
 
3512
4126
  next.pop();
@@ -3555,12 +4169,13 @@ class index_gt {
3555
4169
  top.clear();
3556
4170
  top.reserve(count);
3557
4171
  for (std::size_t i = 0; i != size(); ++i) {
4172
+ auto slot = static_cast<compressed_slot_t>(i);
3558
4173
  if (!is_dummy<predicate_at>())
3559
- if (!predicate(at(i)))
4174
+ if (!predicate(at(slot)))
3560
4175
  continue;
3561
4176
 
3562
- distance_t distance = context.measure(query, citerator_at(i), metric);
3563
- top.insert(candidate_t{distance, static_cast<compressed_slot_t>(i)}, count);
4177
+ distance_t distance = context.measure(query, citerator_at(slot), metric);
4178
+ top.insert(candidate_t{distance, slot}, count);
3564
4179
  }
3565
4180
  }
3566
4181
 
@@ -3570,22 +4185,27 @@ class index_gt {
3570
4185
  * to keep only the neighbors, that are from each other.
3571
4186
  */
3572
4187
  template <typename metric_at>
3573
- candidates_view_t refine_( //
3574
- metric_at&& metric, //
3575
- std::size_t needed, top_candidates_t& top, context_t& context) const noexcept {
4188
+ candidates_view_t refine_( //
4189
+ metric_at&& metric, //
4190
+ std::size_t needed, top_candidates_t& top, context_t& context, //
4191
+ std::size_t& refines_counter) const noexcept {
3576
4192
 
3577
- top.sort_ascending();
4193
+ // Avoid expensive computation, if the set is already small
3578
4194
  candidate_t* top_data = top.data();
3579
4195
  std::size_t const top_count = top.size();
3580
4196
  if (top_count < needed)
3581
4197
  return {top_data, top_count};
3582
4198
 
4199
+ // Sort before processing
4200
+ top.sort_ascending();
4201
+
3583
4202
  std::size_t submitted_count = 1;
3584
4203
  std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
3585
4204
  while (submitted_count < needed && consumed_count < top_count) {
3586
4205
  candidate_t candidate = top_data[consumed_count];
3587
4206
  bool good = true;
3588
- for (std::size_t idx = 0; idx < submitted_count; idx++) {
4207
+ std::size_t idx = 0;
4208
+ for (; idx < submitted_count; idx++) {
3589
4209
  candidate_t submitted = top_data[idx];
3590
4210
  distance_t inter_result_dist = context.measure( //
3591
4211
  citerator_at(candidate.slot), //
@@ -3596,6 +4216,7 @@ class index_gt {
3596
4216
  break;
3597
4217
  }
3598
4218
  }
4219
+ refines_counter += idx;
3599
4220
 
3600
4221
  if (good) {
3601
4222
  top_data[submitted_count] = top_data[consumed_count];
@@ -3777,11 +4398,12 @@ static join_result_t join( //
3777
4398
  bool woman_is_free = husband_slot == missing_slot;
3778
4399
  if (woman_is_free) {
3779
4400
  // Engagement
3780
- man_to_woman_slots[free_man_slot] = woman.slot;
4401
+ man_to_woman_slots[free_man_slot] = static_cast<compressed_slot_t>(woman.slot);
3781
4402
  woman_to_man_slots[woman.slot] = free_man_slot;
3782
4403
  engagements++;
3783
4404
  } else {
3784
- distance_t distance_from_husband = women_metric(women_values[woman.slot], men_values[husband_slot]);
4405
+ distance_t distance_from_husband =
4406
+ women_metric(women_values[static_cast<compressed_slot_t>(woman.slot)], men_values[husband_slot]);
3785
4407
  distance_t distance_from_candidate = match.distance;
3786
4408
  if (distance_from_husband > distance_from_candidate) {
3787
4409
  // Break-up
@@ -3791,7 +4413,7 @@ static join_result_t join( //
3791
4413
  men_locks.atomic_reset(husband_slot);
3792
4414
 
3793
4415
  // New Engagement
3794
- man_to_woman_slots[free_man_slot] = woman.slot;
4416
+ man_to_woman_slots[free_man_slot] = static_cast<compressed_slot_t>(woman.slot);
3795
4417
  woman_to_man_slots[woman.slot] = free_man_slot;
3796
4418
  engagements++;
3797
4419
 
@@ -3816,7 +4438,7 @@ static join_result_t join( //
3816
4438
  for (std::size_t man_slot = 0; man_slot != men.size(); ++man_slot) {
3817
4439
  compressed_slot_t woman_slot = man_to_woman_slots[man_slot];
3818
4440
  if (woman_slot != missing_slot) {
3819
- man_key_t man = men.at(man_slot).key;
4441
+ man_key_t man = men.at(static_cast<compressed_slot_t>(man_slot)).key;
3820
4442
  woman_key_t woman = women.at(woman_slot).key;
3821
4443
  man_to_woman[man] = woman;
3822
4444
  woman_to_man[woman] = man;