usearch 0.4.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.clang-tidy +60 -0
  2. package/.github/workflows/release.yml +42 -16
  3. package/.github/workflows/update_version.sh +1 -0
  4. package/.vscode/launch.json +22 -0
  5. package/.vscode/settings.json +5 -0
  6. package/.vscode/tasks.json +4 -4
  7. package/CITATION.cff +11 -0
  8. package/CMakeLists.txt +108 -95
  9. package/Cargo.toml +1 -1
  10. package/Dockerfile +14 -0
  11. package/README.md +8 -7
  12. package/VERSION +1 -1
  13. package/binding.gyp +2 -2
  14. package/build.rs +2 -2
  15. package/cmake/config.cmake.in +15 -0
  16. package/cmake/pkg-config.pc.in +4 -0
  17. package/cmake/usearchConfigVersion.cmake.in +20 -0
  18. package/conanfile.py +1 -1
  19. package/docs/benchmarks.md +11 -2
  20. package/docs/{Compilation.md → compilation.md} +54 -27
  21. package/include/usearch/usearch.hpp +664 -390
  22. package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +17 -16
  23. package/javascript/lib.cpp +13 -13
  24. package/javascript/test.js +1 -1
  25. package/objc/USearchObjective.mm +46 -29
  26. package/objc/include/USearchObjective.h +18 -26
  27. package/package.json +1 -1
  28. package/pyproject.toml +3 -2
  29. package/python/lib.cpp +179 -120
  30. package/python/{bench.py → scripts/bench.py} +6 -1
  31. package/python/scripts/bench_params.py +112 -0
  32. package/python/{test.py → scripts/test.py} +32 -13
  33. package/python/usearch/client.py +100 -14
  34. package/python/usearch/server.py +104 -23
  35. package/rust/lib.cpp +15 -14
  36. package/rust/lib.hpp +5 -5
  37. package/rust/lib.rs +3 -3
  38. package/setup.py +7 -0
  39. package/src/CMakeLists.txt +107 -0
  40. package/src/bench.cpp +45 -15
  41. package/src/{advanced.hpp → punned.hpp} +227 -199
  42. package/src/test.cpp +62 -0
  43. package/swift/Test.swift +1 -1
  44. package/wolfram/lib.cpp +14 -14
@@ -9,51 +9,84 @@
9
9
  #ifndef UNUM_USEARCH_H
10
10
  #define UNUM_USEARCH_H
11
11
 
12
+ // Inferring C++ version
13
+ #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
14
+ #define USEARCH_IS_CPP17
15
+ #endif
16
+
17
+ // Inferring target OS
12
18
  #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
13
- #define _USE_MATH_DEFINES
19
+ #define USEARCH_IS_WINDOWS
20
+ #elif defined(__APPLE__) && defined(__MACH__)
21
+ #define USEARCH_IS_APPLE
22
+ #elif defined(__linux__)
23
+ #define USEARCH_IS_LINUX
24
+ #endif
14
25
 
15
- #include <Windows.h>
26
+ // Inferring the compiler
27
+ #if defined(__clang__)
28
+ #define USEARCH_IS_CLANG
29
+ #elif defined(__GNUC__)
30
+ #define USEARCH_IS_GCC
31
+ #endif
16
32
 
17
- #define usearch_pack_m
18
- #define usearch_align_m __declspec(align(64))
19
- #define WINDOWS
33
+ // Inferring hardware architecture: x86 vs Arm
34
+ #if defined(__x86_64__)
35
+ #define USEARCH_IS_X86
36
+ #elif defined(__aarch64__)
37
+ #define USEARCH_IS_ARM
38
+ #endif
20
39
 
40
+ // OS-specific includes
41
+ #if defined(USEARCH_IS_WINDOWS)
42
+ #define _USE_MATH_DEFINES
43
+ #include <Windows.h>
44
+ #include <sys/stat.h> // `fstat` for file size
21
45
  #else
22
46
  #include <fcntl.h> // `fallocate`
23
47
  #include <stdlib.h> // `posix_memalign`
24
48
  #include <sys/mman.h> // `mmap`
49
+ #include <sys/stat.h> // `fstat` for file size
25
50
  #include <unistd.h> // `open`, `close`
26
-
27
- #define usearch_pack_m __attribute__((packed))
28
- #define usearch_align_m __attribute__((aligned(64)))
29
51
  #endif
30
52
 
31
- #include <sys/stat.h> // `fstat` for file size
32
-
53
+ // STL includes
33
54
  #include <algorithm> // `std::sort_heap`
34
55
  #include <atomic> // `std::atomic`
35
56
  #include <bitset> // `std::bitset`
36
57
  #include <climits> // `CHAR_BIT`
37
58
  #include <cmath> // `std::sqrt`
38
59
  #include <cstring> // `std::memset`
60
+ #include <iterator> // `std::reverse_iterator`
39
61
  #include <mutex> // `std::unique_lock` - replacement candidate
40
62
  #include <random> // `std::default_random_engine` - replacement candidate
41
63
  #include <stdexcept> // `std::runtime_exception`
42
- #include <utility> // `std::exchange`
64
+ #include <utility> // `std::exchange`, `std::pair`
43
65
  #include <vector> // `std::vector`
44
66
 
45
- #if defined(__GNUC__)
67
+ // Prefetching
68
+ #if defined(USEARCH_IS_GCC)
46
69
  // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
47
70
  // Zero means we are only going to read from that memory.
48
71
  // Three means high temporal locality and suggests to keep
49
72
  // the data in all layers of cache.
50
73
  #define prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
51
- #elif defined(__x86_64__)
74
+ #elif defined(USEARCH_IS_X86)
52
75
  #define prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
53
76
  #else
54
77
  #define prefetch_m(ptr)
55
78
  #endif
56
79
 
80
+ // Alignment
81
+ #if defined(USEARCH_IS_WINDOWS)
82
+ #define usearch_pack_m
83
+ #define usearch_align_m __declspec(align(64))
84
+ #else
85
+ #define usearch_pack_m __attribute__((packed))
86
+ #define usearch_align_m __attribute__((aligned(64)))
87
+ #endif
88
+
89
+ // Debugging
57
90
  #if defined(NDEBUG)
58
91
  #define assert_m(must_be_true, message)
59
92
  #else
@@ -68,11 +101,16 @@ namespace usearch {
68
101
 
69
102
  using f64_t = double;
70
103
  using f32_t = float;
104
+ using byte_t = char;
71
105
 
72
106
  template <typename at> at angle_to_radians(at angle) noexcept { return angle * M_PI / 180.f; }
73
107
 
74
108
  template <typename at> at square(at value) noexcept { return value * value; }
75
109
 
110
+ template <std::size_t multiple_ak> inline std::size_t divide_round_up(std::size_t num) noexcept {
111
+ return (num + multiple_ak - 1) / multiple_ak;
112
+ }
113
+
76
114
  inline std::size_t ceil2(std::size_t v) noexcept {
77
115
  v--;
78
116
  v |= v >> 1;
@@ -85,6 +123,68 @@ inline std::size_t ceil2(std::size_t v) noexcept {
85
123
  return v;
86
124
  }
87
125
 
126
+ template <typename at> void misaligned_store(void* ptr, at v) noexcept { std::memcpy(ptr, &v, sizeof(v)); }
127
+
128
+ template <typename at> at misaligned_load(void* ptr) noexcept {
129
+ at v;
130
+ std::memcpy(&v, ptr, sizeof(v));
131
+ return v;
132
+ }
133
+
134
+ template <typename at> class misaligned_ref_gt {
135
+ byte_t* ptr_;
136
+
137
+ public:
138
+ misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
139
+ operator at() const noexcept { return misaligned_load<at>(ptr_); }
140
+ misaligned_ref_gt& operator=(at const& v) noexcept {
141
+ misaligned_store<at>(ptr_, v);
142
+ return *this;
143
+ }
144
+ misaligned_ref_gt& operator=(at v) noexcept {
145
+ misaligned_store<at>(ptr_, v);
146
+ return *this;
147
+ }
148
+ };
149
+
150
+ template <typename at> class misaligned_ptr_gt {
151
+ byte_t* ptr_;
152
+
153
+ public:
154
+ using iterator_category = std::random_access_iterator_tag;
155
+ using value_type = at;
156
+ using difference_type = std::ptrdiff_t;
157
+ using pointer = misaligned_ptr_gt<at>;
158
+ using reference = misaligned_ref_gt<at>;
159
+
160
+ reference operator*() const noexcept { return {ptr_}; }
161
+
162
+ misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
163
+ misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(at)); }
164
+ misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(at)); }
165
+ misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(at)); }
166
+ misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(at)); }
167
+
168
+ misaligned_ptr_gt& operator++() noexcept {
169
+ ptr_ += sizeof(at);
170
+ return *this;
171
+ }
172
+ misaligned_ptr_gt& operator--() noexcept {
173
+ ptr_ -= sizeof(at);
174
+ return *this;
175
+ }
176
+ misaligned_ptr_gt& operator+=(difference_type d) noexcept {
177
+ ptr_ += d * sizeof(at);
178
+ return *this;
179
+ }
180
+ misaligned_ptr_gt& operator-=(difference_type d) noexcept {
181
+ ptr_ -= d * sizeof(at);
182
+ return *this;
183
+ }
184
+ bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
185
+ bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
186
+ };
187
+
88
188
  /**
89
189
  * @brief Inner (Dot) Product distance.
90
190
  */
@@ -95,12 +195,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
95
195
 
96
196
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
97
197
  result_type ab{};
98
- #if defined(__clang__)
198
+ #if defined(USEARCH_USE_OPENMP)
199
+ #pragma omp simd reduction(+ : ab)
200
+ #elif defined(USEARCH_IS_CLANG)
99
201
  #pragma clang loop vectorize(enable)
100
- #elif defined(__GNUC__)
202
+ #elif defined(USEARCH_IS_GCC)
101
203
  #pragma GCC ivdep
102
- #elif defined(_OPENMP)
103
- #pragma omp simd reduction(+ : ab)
104
204
  #endif
105
205
  for (std::size_t i = 0; i != dim; ++i)
106
206
  ab += result_t(a[i]) * result_t(b[i]);
@@ -121,18 +221,18 @@ template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
121
221
 
122
222
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
123
223
  result_t ab{}, a2{}, b2{};
124
- #if defined(__clang__)
224
+ #if defined(USEARCH_USE_OPENMP)
225
+ #pragma omp simd reduction(+ : ab, a2, b2)
226
+ #elif defined(USEARCH_IS_CLANG)
125
227
  #pragma clang loop vectorize(enable)
126
- #elif defined(__GNUC__)
228
+ #elif defined(USEARCH_IS_GCC)
127
229
  #pragma GCC ivdep
128
- #elif defined(_OPENMP)
129
- #pragma omp simd reduction(+ : ab, a2, b2)
130
230
  #endif
131
231
  for (std::size_t i = 0; i != dim; ++i)
132
232
  ab += result_t(a[i]) * result_t(b[i]), //
133
233
  a2 += square<result_t>(a[i]), //
134
234
  b2 += square<result_t>(b[i]);
135
- return 1 - ab / (std::sqrt(a2) * std::sqrt(b2));
235
+ return ab ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 0;
136
236
  }
137
237
  };
138
238
 
@@ -147,12 +247,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
147
247
 
148
248
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
149
249
  result_t ab_deltas_sq{};
150
- #if defined(__clang__)
250
+ #if defined(USEARCH_USE_OPENMP)
251
+ #pragma omp simd reduction(+ : ab_deltas_sq)
252
+ #elif defined(USEARCH_IS_CLANG)
151
253
  #pragma clang loop vectorize(enable)
152
- #elif defined(__GNUC__)
254
+ #elif defined(USEARCH_IS_GCC)
153
255
  #pragma GCC ivdep
154
- #elif defined(_OPENMP)
155
- #pragma omp simd reduction(+ : ab_deltas_sq)
156
256
  #endif
157
257
  for (std::size_t i = 0; i != dim; ++i)
158
258
  ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
@@ -169,17 +269,16 @@ template <typename scalar_at, typename result_at = std::size_t> struct hamming_g
169
269
  using result_t = result_at;
170
270
  using result_type = result_t;
171
271
 
172
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t elements,
173
- std::size_t = 0) const noexcept {
272
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
174
273
  result_t matches{};
175
- #if defined(__clang__)
274
+ #if defined(USEARCH_USE_OPENMP)
275
+ #pragma omp simd reduction(+ : matches)
276
+ #elif defined(USEARCH_IS_CLANG)
176
277
  #pragma clang loop vectorize(enable)
177
- #elif defined(__GNUC__)
278
+ #elif defined(USEARCH_IS_GCC)
178
279
  #pragma GCC ivdep
179
- #elif defined(_OPENMP)
180
- #pragma omp simd reduction(+ : matches)
181
280
  #endif
182
- for (std::size_t i = 0; i != elements; ++i)
281
+ for (std::size_t i = 0; i != dim; ++i)
183
282
  matches += a[i] != b[i];
184
283
  return matches;
185
284
  }
@@ -195,17 +294,17 @@ template <typename scalar_at, typename result_at = std::size_t> struct bit_hammi
195
294
  using result_t = result_at;
196
295
  using result_type = result_t;
197
296
  static_assert(std::is_unsigned<scalar_t>::value, "Hamming distance requires unsigned integral words");
198
- static constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
199
297
 
200
298
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words,
201
299
  std::size_t = 0) const noexcept {
300
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
202
301
  result_t matches{};
203
- #if defined(__clang__)
302
+ #if defined(USEARCH_USE_OPENMP)
303
+ #pragma omp simd reduction(+ : matches)
304
+ #elif defined(USEARCH_IS_CLANG)
204
305
  #pragma clang loop vectorize(enable)
205
- #elif defined(__GNUC__)
306
+ #elif defined(USEARCH_IS_GCC)
206
307
  #pragma GCC ivdep
207
- #elif defined(_OPENMP)
208
- #pragma omp simd reduction(+ : matches)
209
308
  #endif
210
309
  for (std::size_t i = 0; i != words; ++i)
211
310
  matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
@@ -252,12 +351,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct pearson_cor
252
351
  scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
253
352
  result_t a_sum{}, b_sum{}, ab_sum{};
254
353
  result_t a_sq_sum{}, b_sq_sum{};
255
- #if defined(__clang__)
354
+ #if defined(USEARCH_USE_OPENMP)
355
+ #pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
356
+ #elif defined(USEARCH_IS_CLANG)
256
357
  #pragma clang loop vectorize(enable)
257
- #elif defined(__GNUC__)
358
+ #elif defined(USEARCH_IS_GCC)
258
359
  #pragma GCC ivdep
259
- #elif defined(_OPENMP)
260
- #pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
261
360
  #endif
262
361
  for (std::size_t i = 0; i != dim; ++i) {
263
362
  a_sum += a[i];
@@ -300,19 +399,21 @@ template <typename scalar_at, typename result_at = scalar_at> struct haversine_g
300
399
  }
301
400
  };
302
401
 
303
- template <std::size_t multiple_ak> inline std::size_t divide_round_up(std::size_t num) noexcept {
304
- return (num + multiple_ak - 1) / multiple_ak;
305
- }
306
-
307
402
  /**
308
403
  * @brief Light-weight bitset implementation to track visited nodes during graph traversal.
404
+ * Extends basic functionality with atomic operations.
309
405
  */
310
406
  template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
311
407
  using allocator_t = allocator_at;
312
408
  using byte_t = typename allocator_t::value_type;
313
409
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
314
410
 
411
+ #if defined(USEARCH_IS_WINDOWS)
412
+ using slot_t = LONG volatile;
413
+ static_assert(sizeof(slot_t) == 8);
414
+ #else
315
415
  using slot_t = std::uint64_t;
416
+ #endif
316
417
 
317
418
  std::uint64_t* u64s_{};
318
419
  std::size_t slots_{};
@@ -322,6 +423,7 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
322
423
  visits_bitset_gt(std::size_t capacity) noexcept {
323
424
  slots_ = divide_round_up<64>(capacity);
324
425
  u64s_ = (slot_t*)allocator_t{}.allocate(slots_ * sizeof(slot_t));
426
+ clear();
325
427
  }
326
428
  ~visits_bitset_gt() noexcept { allocator_t{}.deallocate((byte_t*)u64s_, slots_ * sizeof(slot_t)), u64s_ = nullptr; }
327
429
  void clear() noexcept { std::memset(u64s_, 0, slots_ * sizeof(slot_t)); }
@@ -340,6 +442,42 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
340
442
  std::swap(slots_, other.slots_);
341
443
  return *this;
342
444
  }
445
+
446
+ #if defined(USEARCH_IS_WINDOWS)
447
+
448
+ inline bool atomic_test(std::size_t i) noexcept {
449
+ slot_t mask{1ul << (i & 63ul)};
450
+ return InterlockedOr(&u64s_[i / 64], 0) & mask;
451
+ }
452
+
453
+ inline bool atomic_set(std::size_t i) noexcept {
454
+ slot_t mask{1ul << (i & 63ul)};
455
+ return InterLockedOr(&u64s_[i / 64], mask) & mask;
456
+ }
457
+
458
+ inline void atomic_reset(std::size_t i) noexcept {
459
+ slot_t mask{1ul << (i & 63ul)};
460
+ InterLockedAnd((LONG volatile*)&u64s_[i / 64], ~mask);
461
+ }
462
+
463
+ #else
464
+
465
+ inline bool atomic_test(std::size_t i) noexcept {
466
+ slot_t mask{1ul << (i & 63ul)};
467
+ return __atomic_load_n(&u64s_[i / 64], __ATOMIC_RELAXED) & mask;
468
+ }
469
+
470
+ inline bool atomic_set(std::size_t i) noexcept {
471
+ slot_t mask{1ul << (i & 63ul)};
472
+ return __atomic_fetch_or(&u64s_[i / 64], mask, __ATOMIC_ACQUIRE) & mask;
473
+ }
474
+
475
+ inline void atomic_reset(std::size_t i) noexcept {
476
+ slot_t mask{1ul << (i & 63ul)};
477
+ __atomic_fetch_and(&u64s_[i / 64], ~mask, __ATOMIC_RELEASE);
478
+ }
479
+
480
+ #endif
343
481
  };
344
482
 
345
483
  using visits_bitset_t = visits_bitset_gt<>;
@@ -580,57 +718,7 @@ class sorted_buffer_gt {
580
718
  static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
581
719
  };
582
720
 
583
- /**
584
- * @brief Tiny userspace mutex, designed to fit in a single integer.
585
- */
586
- class mutex_t {
587
- #if defined(WINDOWS)
588
- using slot_t = volatile LONG;
589
- #else
590
- using slot_t = std::int32_t;
591
- #endif // WINDOWS
592
-
593
- slot_t flag_;
594
-
595
- public:
596
- inline mutex_t(slot_t flag = 0) noexcept : flag_(flag) {}
597
- inline ~mutex_t() noexcept {}
598
-
599
- inline bool try_lock() noexcept {
600
- slot_t raw = 0;
601
- #if defined(WINDOWS)
602
- return InterlockedCompareExchange(&flag_, 1, raw);
603
- #else
604
- return __atomic_compare_exchange_n(&flag_, &raw, 1, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
605
- #endif // WINDOWS
606
- }
607
-
608
- inline void lock() noexcept {
609
- slot_t raw = 0;
610
- #if defined(WINDOWS)
611
- InterlockedCompareExchange(&flag_, 1, raw);
612
- #else
613
- lock_again:
614
- raw = 0;
615
- if (!__atomic_compare_exchange_n(&flag_, &raw, 1, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
616
- goto lock_again;
617
- #endif // WINDOWS
618
- }
619
-
620
- inline void unlock() noexcept {
621
- #if defined(WINDOWS)
622
- InterlockedExchange(&flag_, 0);
623
- #else
624
- __atomic_store_n(&flag_, 0, __ATOMIC_RELEASE);
625
- #endif
626
- }
627
- };
628
-
629
- static_assert(sizeof(mutex_t) == sizeof(std::int32_t), "Mutex is larger than expected");
630
-
631
- using lock_t = std::unique_lock<mutex_t>;
632
-
633
- #if defined(WINDOWS)
721
+ #if defined(USEARCH_IS_WINDOWS)
634
722
  #pragma pack(push, 1) // Pack struct members on 1-byte alignment
635
723
  #endif
636
724
 
@@ -644,7 +732,7 @@ class usearch_pack_m uint40_t {
644
732
  inline uint40_t() noexcept { std::memset(octets, 0, 5); }
645
733
  inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets + 1, (char*)&n, 4), octets[0] = 0; }
646
734
  inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
647
- #if defined(__clang__)
735
+ #if defined(USEARCH_IS_CLANG)
648
736
  inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
649
737
  #endif
650
738
 
@@ -675,12 +763,15 @@ class usearch_pack_m uint40_t {
675
763
  }
676
764
  };
677
765
 
678
- #if defined(WINDOWS)
766
+ #if defined(USEARCH_IS_WINDOWS)
679
767
  #pragma pack(pop) // Reset alignment to default
680
768
  #endif
681
769
 
682
770
  static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
683
771
 
772
+ /**
773
+ * @brief Non-owning memory range view, similar to `std::span`, but for C++11.
774
+ */
684
775
  template <typename scalar_at> class span_gt {
685
776
  scalar_at* data_;
686
777
  std::size_t size_;
@@ -693,6 +784,10 @@ template <typename scalar_at> class span_gt {
693
784
  operator scalar_at*() const noexcept { return data(); }
694
785
  };
695
786
 
787
+ constexpr std::size_t default_connectivity() { return 16; }
788
+ constexpr std::size_t default_expansion_add() { return 128; }
789
+ constexpr std::size_t default_expansion_search() { return 64; }
790
+
696
791
  /**
697
792
  * @brief Configuration settings for the index construction.
698
793
  * Includes the main `::connectivity` parameter (`M` in the paper)
@@ -703,26 +798,25 @@ struct config_t {
703
798
  /// @brief Number of neighbors per graph node.
704
799
  /// Defaults to 32 in FAISS and 16 in hnswlib.
705
800
  /// > It is called `M` in the paper.
706
- static constexpr std::size_t connectivity_default_k = 16;
801
+ std::size_t connectivity = default_connectivity();
707
802
 
708
803
  /// @brief Hyper-parameter controlling the quality of indexing.
709
804
  /// Defaults to 40 in FAISS and 200 in hnswlib.
710
805
  /// > It is called `efConstruction` in the paper.
711
- static constexpr std::size_t expansion_add_default_k = 128;
806
+ std::size_t expansion_add = default_expansion_add();
712
807
 
713
808
  /// @brief Hyper-parameter controlling the quality of search.
714
809
  /// Defaults to 16 in FAISS and 10 in hnswlib.
715
810
  /// > It is called `ef` in the paper.
716
- static constexpr std::size_t expansion_search_default_k = 64;
717
-
718
- std::size_t connectivity = connectivity_default_k;
719
- std::size_t expansion_add = expansion_add_default_k;
720
- std::size_t expansion_search = expansion_search_default_k;
811
+ std::size_t expansion_search = default_expansion_search();
721
812
 
722
813
  ///
723
814
  std::size_t max_elements = 0;
724
- std::size_t max_threads_add = 0;
725
- std::size_t max_threads_search = 0;
815
+ std::size_t max_threads_add = 1;
816
+ std::size_t max_threads_search = 1;
817
+
818
+ std::size_t max_threads() const noexcept { return (std::max)(max_threads_add, max_threads_search); }
819
+ std::size_t concurrency() const noexcept { return (std::min)(max_threads_add, max_threads_search); }
726
820
  };
727
821
 
728
822
  struct add_config_t {
@@ -732,33 +826,19 @@ struct add_config_t {
732
826
  bool store_vector{true};
733
827
  };
734
828
 
735
- struct add_result_t {
736
- std::size_t new_size{};
737
- std::size_t cycles{};
738
- std::size_t measurements{};
739
- };
740
-
741
829
  struct search_config_t {
742
830
  std::size_t thread{};
743
831
  bool exact{};
744
832
  };
745
833
 
746
- struct search_result_t {
747
- std::size_t count{};
748
- std::size_t cycles{};
749
- std::size_t measurements{};
750
-
751
- inline operator std::size_t() const noexcept { return count; }
752
- };
753
-
754
834
  /**
755
835
  * @brief Approximate Nearest Neighbors Search index using the
756
836
  * Hierarchical Navigable Small World graph algorithm.
757
837
  *
758
838
  * @section Features
759
- * - Search for vectors of different dimensionality.
760
- * - Thread-safe.
761
- * - Bring your threads!
839
+ * - Thread-safe for concurrent construction.
840
+ * - Doesn't allocate new threads, and reuses the ones its called from.
841
+ * - Search for vectors of different dimensionality, if ::`metric_at` supports that.
762
842
  *
763
843
  * @tparam metric_at
764
844
  * A function vector responsible for computing the distance between two vectors.
@@ -774,6 +854,40 @@ struct search_result_t {
774
854
  *
775
855
  * @tparam allocator_at
776
856
  * Dynamic memory allocator.
857
+ *
858
+ *
859
+ * @section Implementation Details
860
+ *
861
+ * Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
862
+ * Every added vector forms a node in one or more levels of the graph.
863
+ * Every node is present in the base level. Every following level contains a smaller
864
+ * fraction of nodes. During search, the operation starts with the smaller levels
865
+ * and zooms-in on every following iteration of larger graph traversals.
866
+ *
867
+ * Just one memory allocation is performed regardless of the number of levels.
868
+ * The adjacency lists across all levels are concatenated into that single buffer.
869
+ * That buffer starts with a "head", that stores the metadata, such as the
870
+ * tallest "level" of the graph that it belongs to, the external "label", and the
871
+ * number of "dimensions" in the vector.
872
+ *
873
+ * @subsection Colocated Nodes
874
+ *
875
+ * Every node contains some graph-related data, as well as the original vector.
876
+ * Assuming the vectors can get large, we allow separating the index from the vectors.
877
+ * They can either be stored in a single continuous block of memory, or in two separate
878
+ * locations. Vector storage can also be detached from `index_gt` and managed by the user.
879
+ * To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
880
+ * overloads.
881
+ *
882
+ * @subsection Missing Functionality
883
+ *
884
+ * To simplify the implementation, the `index_gt` lacks endpoints to remove existing
885
+ * vectors. That, however, is solved by `punned_gt`, which also adds automatic casting.
886
+ *
887
+ * @subsection Ugly Implementation Parts
888
+ *
889
+ * Internally, a number of `std::vector`-s is created, often `mutable`.
890
+ *
777
891
  */
778
892
  template <typename metric_at = ip_gt<float>, //
779
893
  typename label_at = std::size_t, //
@@ -788,25 +902,104 @@ class index_gt {
788
902
  using id_t = id_at;
789
903
  using allocator_t = allocator_at;
790
904
 
791
- using vector_view_t = span_gt<scalar_t const>;
792
-
793
- #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) // C++17 ann newer
905
+ /// C++17 and newer version deprecate the `std::result_of`
794
906
  using distance_t =
907
+ #if defined(USEARCH_IS_CPP17)
795
908
  typename std::invoke_result<metric_t, scalar_t const*, scalar_t const*, std::size_t, std::size_t>::type;
796
909
  #else
797
- using distance_t =
798
910
  typename std::result_of<metric_t(scalar_t const*, scalar_t const*, std::size_t, std::size_t)>::type;
799
911
  #endif
800
912
 
913
+ using vector_view_t = span_gt<scalar_t const>;
914
+
915
+ struct member_ref_t {
916
+ misaligned_ref_gt<label_t> label;
917
+ vector_view_t vector;
918
+ };
919
+
920
+ struct member_cref_t {
921
+ label_t label;
922
+ vector_view_t vector;
923
+ };
924
+
925
+ template <typename ref_at> class member_iterator_gt {
926
+ using ref_t = ref_at;
927
+ index_gt& index_;
928
+ std::size_t offset_;
929
+
930
+ member_iterator_gt() noexcept {}
931
+ member_iterator_gt(index_gt& index, std::size_t offset) noexcept : index_(index), offset_(offset) {}
932
+
933
+ public:
934
+ using iterator_category = std::random_access_iterator_tag;
935
+ using value_type = ref_t;
936
+ using difference_type = std::ptrdiff_t;
937
+ using pointer = void;
938
+ using reference = ref_t;
939
+
940
+ reference operator*() const noexcept { return index_.node_with_id_(offset_); }
941
+ member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, offset_ + 1); }
942
+ member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, offset_ - 1); }
943
+ member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, offset_ + d); }
944
+ member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, offset_ - d); }
945
+
946
+ member_iterator_gt& operator++() noexcept {
947
+ offset_ += 1;
948
+ return *this;
949
+ }
950
+ member_iterator_gt& operator--() noexcept {
951
+ offset_ -= 1;
952
+ return *this;
953
+ }
954
+ member_iterator_gt& operator+=(difference_type d) noexcept {
955
+ offset_ += d;
956
+ return *this;
957
+ }
958
+ member_iterator_gt& operator-=(difference_type d) noexcept {
959
+ offset_ -= d;
960
+ return *this;
961
+ }
962
+ bool operator!=(member_iterator_gt const& other) noexcept {
963
+ return &index_ != &other.index_ || offset_ != other.offset_;
964
+ }
965
+ };
966
+
967
+ using member_iterator_t = member_iterator_gt<member_ref_t>;
968
+ using member_citerator_t = member_iterator_gt<member_cref_t>;
969
+
970
+ // STL compatibility:
971
+ using value_type = std::pair<label_t, distance_t>;
972
+ using allocator_type = allocator_t;
973
+ using size_type = std::size_t;
974
+ using difference_type = std::ptrdiff_t;
975
+ using reference = member_ref_t;
976
+ using const_reference = member_cref_t;
977
+ using pointer = void;
978
+ using const_pointer = void;
979
+ using iterator = member_iterator_t;
980
+ using const_iterator = member_citerator_t;
981
+ using reverse_iterator = std::reverse_iterator<member_iterator_t>;
982
+ using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
983
+
801
984
  private:
802
- using neighbors_count_t = id_t;
985
+ using neighbors_count_t = std::uint32_t;
803
986
  using dim_t = std::uint32_t;
804
987
  using level_t = std::int32_t;
805
988
 
806
989
  using allocator_traits_t = std::allocator_traits<allocator_t>;
807
990
  using byte_t = typename allocator_t::value_type;
808
991
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
809
- static constexpr std::size_t base_level_multiple_k = 2;
992
+
993
+ /**
994
+ * @brief How much larger (number of neighbors per node) will the base level be
995
+ * comapred to other levels.
996
+ */
997
+ static constexpr std::size_t base_level_multiple_() { return 2; }
998
+
999
+ /**
1000
+ * @brief How many bytes of memory are needed to form the "head" of the node.
1001
+ */
1002
+ static constexpr std::size_t node_head_bytes_() { return sizeof(label_t) + sizeof(dim_t) + sizeof(level_t); }
810
1003
 
811
1004
  using visits_bitset_t = visits_bitset_gt<allocator_t>;
812
1005
 
@@ -815,14 +1008,13 @@ class index_gt {
815
1008
  std::size_t connectivity_max_base{};
816
1009
  std::size_t neighbors_bytes{};
817
1010
  std::size_t neighbors_base_bytes{};
818
- std::size_t mutex_bytes{};
819
1011
  };
820
1012
  struct candidate_t {
821
- distance_t first;
822
- id_t second;
1013
+ distance_t distance;
1014
+ id_t id;
823
1015
  };
824
1016
  struct compare_by_distance_t {
825
- inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.first < b.first; }
1017
+ inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
826
1018
  };
827
1019
 
828
1020
  using candidates_view_t = span_gt<candidate_t const>;
@@ -830,65 +1022,72 @@ class index_gt {
830
1022
  using top_candidates_t = sorted_buffer_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
831
1023
  using next_candidates_t = max_heap_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
832
1024
 
833
- struct neighbors_ref_t {
834
- neighbors_count_t& count_;
835
- id_t* neighbors_{};
836
-
837
- inline neighbors_ref_t(byte_t* tape) noexcept
838
- : count_(*(neighbors_count_t*)tape), neighbors_((neighbors_count_t*)tape + 1) {}
839
- inline id_t* begin() noexcept { return neighbors_; }
840
- inline id_t* end() noexcept { return neighbors_ + count_; }
841
- inline id_t const* begin() const noexcept { return neighbors_; }
842
- inline id_t const* end() const noexcept { return neighbors_ + count_; }
843
- inline id_t& operator[](std::size_t i) noexcept { return neighbors_[i]; }
844
- inline id_t operator[](std::size_t i) const noexcept { return neighbors_[i]; }
845
- inline std::size_t size() const noexcept { return count_; }
846
- inline void clear() noexcept { count_ = 0u; }
847
- inline void push_back(id_t id) noexcept { neighbors_[count_] = id, count_++; }
848
- };
849
-
850
- #if defined(WINDOWS)
851
- #pragma pack(push, 1) // Pack struct members on 1-byte alignment
852
- #endif
853
- struct usearch_pack_m node_head_t {
854
- label_t label;
855
- dim_t dim;
856
- level_t level;
857
- // Variable length array, that has multiple similarly structured segments.
858
- // Each starts with a `neighbors_count_t` and is followed by such number of `id_t`s.
859
- byte_t neighbors[1];
860
- };
861
- #if defined(WINDOWS)
862
- #pragma pack(pop) // Reset alignment to default
863
- #endif
864
-
865
- static constexpr std::size_t head_bytes_k = sizeof(label_t) + sizeof(dim_t) + sizeof(level_t);
866
-
867
- struct node_t {
1025
+ /**
1026
+ * @brief A loosely-structured handle for every node. One such node is created for every vector.
1027
+ * To minimize memory usage and maximize the number of entries per cache-line, it only
1028
+ * stores to pointers. To reinterpret those into a structured `node_t` more
1029
+ * information is needed from the parent `index_gt`.
1030
+ */
1031
+ class node_t {
868
1032
  byte_t* tape_{};
869
1033
  scalar_t* vector_{};
870
1034
 
1035
+ public:
871
1036
  explicit node_t(byte_t* tape, scalar_t* vector) noexcept : tape_(tape), vector_(vector) {}
1037
+ byte_t* tape() const noexcept { return tape_; }
1038
+ scalar_t* vector() const noexcept { return vector_; }
1039
+ byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
872
1040
 
873
1041
  node_t() = default;
874
1042
  node_t(node_t const&) = default;
875
1043
  node_t& operator=(node_t const&) = default;
1044
+
1045
+ label_t label() const noexcept { return misaligned_load<label_t>(tape_); }
1046
+ dim_t dim() const noexcept { return misaligned_load<dim_t>(tape_ + sizeof(label_t)); }
1047
+ level_t level() const noexcept { return misaligned_load<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t)); }
1048
+
1049
+ void label(label_t v) noexcept { return misaligned_store<label_t>(tape_, v); }
1050
+ void dim(dim_t v) noexcept { return misaligned_store<dim_t>(tape_ + sizeof(label_t), v); }
1051
+ void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
1052
+
1053
+ operator vector_view_t() const noexcept { return {vector(), dim()}; }
1054
+ vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
1055
+ member_ref_t ref() noexcept { return {label(), vector_view()}; }
1056
+ member_cref_t cref() const noexcept { return {label(), vector_view()}; }
876
1057
  };
877
1058
 
878
- class node_ref_t {
879
- mutex_t* mutex_{};
1059
+ /**
1060
+ * @brief A slice of the node's tape, containing a the list of neighbors
1061
+ * for a node in a single graph level. It's pre-allocated to fit
1062
+ * as many neighbors IDs, as may be needed at the target level,
1063
+ * and starts with a single integer `neighbors_count_t` counter.
1064
+ */
1065
+ class neighbors_ref_t {
1066
+ byte_t* tape_;
1067
+
1068
+ static constexpr std::size_t shift(std::size_t i = 0) { return sizeof(neighbors_count_t) + sizeof(id_t) * i; }
880
1069
 
881
1070
  public:
882
- node_head_t& head{};
883
- scalar_t* vector{};
884
-
885
- inline node_ref_t(mutex_t& m, node_head_t& h, scalar_t* s) noexcept : mutex_(&m), head(h), vector(s) {}
886
- /// @brief Locks the list of neighbors for concurrent updates.
887
- inline lock_t lock() const noexcept { return mutex_ ? lock_t{*mutex_} : lock_t{}; }
888
- inline operator node_t() const noexcept { return node_t{mutex_ ? (byte_t*)mutex_ : (byte_t*)&head, vector}; }
889
- inline operator vector_view_t() const noexcept { return {vector, head.dim}; }
1071
+ neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
1072
+ misaligned_ptr_gt<id_t> begin() noexcept { return tape_ + shift(); }
1073
+ misaligned_ptr_gt<id_t> end() noexcept { return begin() + size(); }
1074
+ misaligned_ptr_gt<id_t const> begin() const noexcept { return tape_ + shift(); }
1075
+ misaligned_ptr_gt<id_t const> end() const noexcept { return begin() + size(); }
1076
+ id_t operator[](std::size_t i) const noexcept { return misaligned_load<id_t>(tape_ + shift(i)); }
1077
+ std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
1078
+ void clear() noexcept { misaligned_store<neighbors_count_t>(tape_, 0); }
1079
+ void push_back(id_t id) noexcept {
1080
+ neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1081
+ misaligned_store<id_t>(tape_ + shift(n), id);
1082
+ misaligned_store<neighbors_count_t>(tape_, n + 1);
1083
+ }
890
1084
  };
891
1085
 
1086
+ /**
1087
+ * @brief A package of all kinds of temporary data-structures, that the threads
1088
+ * would reuse to process requests. Similar to having all of those as
1089
+ * separate `thread_local` global variables.
1090
+ */
892
1091
  struct usearch_align_m thread_context_t {
893
1092
  top_candidates_t top_candidates{};
894
1093
  next_candidates_t next_candidates{};
@@ -913,12 +1112,15 @@ class index_gt {
913
1112
  usearch_align_m mutable std::atomic<std::size_t> capacity_{};
914
1113
  usearch_align_m mutable std::atomic<std::size_t> size_{};
915
1114
 
916
- mutex_t global_mutex_{};
1115
+ /// @brief Controls access to `max_level_` and `entry_id_`.
1116
+ /// If any thread is updating those values, no other threads can `add()` or `search()`.
1117
+ std::mutex global_mutex_{};
917
1118
  level_t max_level_{};
918
1119
  id_t entry_id_{};
919
1120
 
920
1121
  using node_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
921
1122
  std::vector<node_t, node_allocator_t> nodes_{};
1123
+ mutable visits_bitset_t nodes_mutexes_{};
922
1124
 
923
1125
  using thread_context_allocator_t = typename allocator_traits_t::template rebind_alloc<thread_context_t>;
924
1126
  mutable std::vector<thread_context_t, thread_context_allocator_t> thread_contexts_{};
@@ -927,16 +1129,15 @@ class index_gt {
927
1129
  std::size_t connectivity() const noexcept { return config_.connectivity; }
928
1130
  std::size_t capacity() const noexcept { return capacity_; }
929
1131
  std::size_t size() const noexcept { return size_; }
930
- std::size_t max_threads_add() const noexcept { return config_.max_threads_add; }
1132
+ std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
1133
+ config_t const& config() const { return config_; }
931
1134
  bool is_immutable() const noexcept { return viewed_file_descriptor_ != 0; }
932
- bool synchronize() const noexcept { return config_.max_threads_add > 1; }
933
1135
 
934
1136
  index_gt(config_t config = {}, metric_t metric = {}, allocator_t allocator = {}) noexcept(false)
935
- : config_(config), metric_(metric), allocator_(allocator) {
1137
+ : config_(normalize(config)), metric_(metric), allocator_(allocator) {
936
1138
 
937
1139
  // Externally defined hyper-parameters:
938
- config_.expansion_add = (std::max)(config_.expansion_add, config_.connectivity);
939
- pre_ = precompute(config);
1140
+ pre_ = precompute_(config);
940
1141
 
941
1142
  // Configure initial empty state:
942
1143
  size_ = 0u;
@@ -945,7 +1146,7 @@ class index_gt {
945
1146
  viewed_file_descriptor_ = 0;
946
1147
 
947
1148
  // Dynamic memory:
948
- thread_contexts_.resize((std::max)(config.max_threads_search, config.max_threads_add));
1149
+ thread_contexts_.resize(config.max_threads());
949
1150
  for (thread_context_t& context : thread_contexts_)
950
1151
  context.metric = metric;
951
1152
  reserve(config.max_elements);
@@ -974,7 +1175,7 @@ class index_gt {
974
1175
  void clear() noexcept {
975
1176
  std::size_t n = size_;
976
1177
  for (std::size_t i = 0; i != n; ++i)
977
- node_free(i);
1178
+ node_free_(i);
978
1179
  size_ = 0;
979
1180
  max_level_ = -1;
980
1181
  entry_id_ = 0u;
@@ -1010,6 +1211,7 @@ class index_gt {
1010
1211
 
1011
1212
  assert_m(new_capacity >= size_, "Can't drop existing values");
1012
1213
  nodes_.resize(new_capacity);
1214
+ nodes_mutexes_ = visits_bitset_t(new_capacity);
1013
1215
  for (thread_context_t& context : thread_contexts_)
1014
1216
  context.visits = visits_bitset_t(new_capacity);
1015
1217
 
@@ -1020,20 +1222,73 @@ class index_gt {
1020
1222
  * @brief Optimizes configuration options to fit the maximum number
1021
1223
  * of neighbors in CPU-cache-aligned buffers.
1022
1224
  */
1023
- static config_t optimize(config_t const& config) noexcept {
1024
- precomputed_constants_t pre = precompute(config);
1025
- std::size_t bytes_per_node_base = head_bytes_k + pre.neighbors_base_bytes + pre.mutex_bytes;
1225
+ static config_t optimize(config_t config) noexcept {
1226
+ precomputed_constants_t pre = precompute_(config);
1227
+ std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
1026
1228
  std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
1027
1229
  std::size_t added_connections = (rounded_size - rounded_size) / sizeof(id_t);
1028
- config_t result = config;
1029
- result.connectivity = config.connectivity + added_connections / base_level_multiple_k;
1030
- return result;
1230
+ config.connectivity = config.connectivity + added_connections / base_level_multiple_();
1231
+ return config;
1232
+ }
1233
+
1234
+ /**
1235
+ * @brief Validates/normalizes the configuration options and throws an exception
1236
+ * if those are incorrect.
1237
+ */
1238
+ static config_t normalize(config_t config) noexcept {
1239
+ config.expansion_add = (std::max<std::size_t>)(config.expansion_add, config.connectivity);
1240
+ config.max_threads_add = (std::max<std::size_t>)(1u, config.max_threads_add);
1241
+ config.max_threads_search = (std::max<std::size_t>)(1u, config.max_threads_search);
1242
+ return config;
1031
1243
  }
1032
1244
 
1033
1245
  #pragma endregion
1034
1246
 
1035
1247
  #pragma region Construction and Search
1036
1248
 
1249
+ struct add_result_t {
1250
+ std::size_t new_size{};
1251
+ std::size_t cycles{};
1252
+ std::size_t measurements{};
1253
+ id_t id{};
1254
+ };
1255
+
1256
+ struct search_result_t {
1257
+ member_cref_t member;
1258
+ distance_t distance;
1259
+ };
1260
+
1261
+ class search_results_t {
1262
+ index_gt const& index_;
1263
+ top_candidates_t& top_;
1264
+
1265
+ friend class index_gt;
1266
+ inline search_results_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
1267
+
1268
+ public:
1269
+ std::size_t count{};
1270
+ std::size_t cycles{};
1271
+ std::size_t measurements{};
1272
+
1273
+ inline search_results_t(search_results_t&&) = default;
1274
+ inline search_results_t& operator=(search_results_t&&) = default;
1275
+
1276
+ inline operator std::size_t() const noexcept { return count; }
1277
+ inline std::size_t size() const noexcept { return count; }
1278
+ inline search_result_t operator[](std::size_t i) const noexcept {
1279
+ candidate_t const* top_ordered = top_.data();
1280
+ return {index_.node_with_id_(top_ordered[i].id).cref(), top_ordered[i].distance};
1281
+ }
1282
+ inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
1283
+ for (std::size_t i = 0; i != count; ++i) {
1284
+ search_result_t result = operator[](i);
1285
+ labels[i] = result.member.label;
1286
+ distances[i] = result.distance;
1287
+ }
1288
+ return count;
1289
+ }
1290
+ };
1291
+
1037
1292
  /**
1038
1293
  * @brief Inserts a new vector into the index. Thread-safe.
1039
1294
  *
@@ -1046,23 +1301,25 @@ class index_gt {
1046
1301
  assert_m(!is_immutable(), "Can't add to an immutable index");
1047
1302
  add_result_t result;
1048
1303
  result.new_size = size_.fetch_add(1);
1049
- id_t id = static_cast<id_t>(result.new_size);
1304
+ id_t new_id = result.id = static_cast<id_t>(result.new_size);
1050
1305
 
1051
1306
  // Determining how much memory to allocate depends on the target level
1052
- lock_t new_level_lock(global_mutex_);
1053
- level_t max_level = max_level_;
1307
+ std::unique_lock<std::mutex> new_level_lock(global_mutex_);
1308
+ level_t max_level = max_level_; // Copy under lock
1309
+ id_t entry_id = entry_id_; // Copy under lock
1054
1310
  thread_context_t& context = thread_contexts_[config.thread];
1055
- level_t target_level = choose_random_level(context.level_generator);
1311
+ level_t target_level = choose_random_level_(context.level_generator);
1056
1312
  if (target_level <= max_level)
1057
1313
  new_level_lock.unlock();
1058
1314
 
1059
1315
  // Allocate the neighbors
1060
- node_ref_t new_node = node_malloc(label, vector, target_level, config.store_vector);
1061
- lock_t new_lock = new_node.lock();
1062
- nodes_[id] = new_node;
1316
+ nodes_[result.new_size] = node_malloc_(label, vector, target_level, config.store_vector);
1317
+ node_lock_t new_lock = node_lock_(new_id);
1318
+ result.new_size++;
1063
1319
 
1064
1320
  // Do nothing for the first element
1065
- if (!id) {
1321
+ if (!new_id) {
1322
+ entry_id_ = new_id;
1066
1323
  max_level_ = target_level;
1067
1324
  return result;
1068
1325
  }
@@ -1072,12 +1329,12 @@ class index_gt {
1072
1329
  result.cycles = context.iteration_cycles;
1073
1330
 
1074
1331
  // Go down the level, tracking only the closest match
1075
- id_t closest_id = search_for_one(entry_id_, vector, max_level, target_level, context);
1332
+ id_t closest_id = search_for_one_(entry_id, vector, max_level, target_level, context);
1076
1333
 
1077
1334
  // From `target_level` down perform proper extensive search
1078
1335
  for (level_t level = (std::min)(target_level, max_level); level >= 0; level--) {
1079
- search_to_insert(closest_id, vector, level, context);
1080
- closest_id = connect_new_element(id, level, context);
1336
+ search_to_insert_(closest_id, vector, level, context);
1337
+ closest_id = connect_new_node_(new_id, level, context);
1081
1338
  }
1082
1339
 
1083
1340
  // Normalize stats
@@ -1086,7 +1343,7 @@ class index_gt {
1086
1343
 
1087
1344
  // Updating the entry point if needed
1088
1345
  if (target_level > max_level) {
1089
- entry_id_ = id;
1346
+ entry_id_ = new_id;
1090
1347
  max_level_ = target_level;
1091
1348
  }
1092
1349
  return result;
@@ -1097,39 +1354,60 @@ class index_gt {
1097
1354
  *
1098
1355
  * @param[in] query Contiguous range of scalars forming a vector view.
1099
1356
  * @param[in] wanted The upper bound for the number of results to return.
1100
- * @param[in] callback Function receiving the labels and distances to matches in ascending order.
1101
1357
  * @param[in] config Configuration options for this specific operation.
1358
+ * @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
1102
1359
  */
1103
- template <typename callback_at>
1104
- search_result_t search( //
1360
+ search_results_t search( //
1105
1361
  vector_view_t query, std::size_t wanted, //
1106
- callback_at&& callback, search_config_t config = {}) const noexcept(false) {
1362
+ search_config_t config = {}) const noexcept(false) {
1107
1363
 
1108
- search_result_t result;
1364
+ thread_context_t& context = thread_contexts_[config.thread];
1365
+ top_candidates_t& top = context.top_candidates;
1366
+ search_results_t result{*this, top};
1109
1367
  if (!size_)
1110
1368
  return result;
1111
1369
 
1112
1370
  // Go down the level, tracking only the closest match
1113
- thread_context_t& context = thread_contexts_[config.thread];
1114
1371
  result.measurements = context.measurements_count;
1115
1372
  result.cycles = context.iteration_cycles;
1116
1373
 
1117
1374
  if (config.exact) {
1118
- search_exact(query, wanted, context);
1375
+ search_exact_(query, wanted, context);
1119
1376
  } else {
1120
- id_t closest_id = search_for_one(entry_id_, query, max_level_, 0, context);
1377
+ id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
1121
1378
  std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1122
1379
  // For bottom layer we need a more optimized procedure
1123
- search_to_find_in_base(closest_id, query, expansion, context);
1380
+ search_to_find_in_base_(closest_id, query, expansion, context);
1124
1381
  }
1125
1382
 
1126
- top_candidates_t& top = context.top_candidates;
1127
1383
  top.sort_ascending();
1128
1384
  top.shrink(wanted);
1129
1385
 
1130
- candidate_t const* top_ordered = top.data();
1131
- for (std::size_t i = 0; i < top.size(); ++i)
1132
- callback(node(top_ordered[i].second).head.label, top_ordered[i].first);
1386
+ // Normalize stats
1387
+ result.measurements = context.measurements_count - result.measurements;
1388
+ result.cycles = context.iteration_cycles - result.cycles;
1389
+ result.count = top.size();
1390
+ return result;
1391
+ }
1392
+
1393
+ search_results_t search_around( //
1394
+ id_t hint, vector_view_t query, std::size_t wanted, //
1395
+ search_config_t config = {}) const noexcept(false) {
1396
+
1397
+ thread_context_t& context = thread_contexts_[config.thread];
1398
+ top_candidates_t& top = context.top_candidates;
1399
+ search_results_t result{*this, top};
1400
+ if (!size_)
1401
+ return result;
1402
+
1403
+ // Go down the level, tracking only the closest match
1404
+ result.measurements = context.measurements_count;
1405
+ result.cycles = context.iteration_cycles;
1406
+
1407
+ std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1408
+ search_to_find_in_base_(hint, query, expansion, context);
1409
+ top.sort_ascending();
1410
+ top.shrink(wanted);
1133
1411
 
1134
1412
  // Normalize stats
1135
1413
  result.measurements = context.measurements_count - result.measurements;
@@ -1138,28 +1416,39 @@ class index_gt {
1138
1416
  return result;
1139
1417
  }
1140
1418
 
1141
- /**
1142
- * @brief Searches for the closest members to the given ::query. Thread-safe.
1143
- * @note Convenience method for callback-based `search()`.
1144
- *
1145
- * @param[in] query Contiguous range of scalars forming a vector view.
1146
- * @param[in] wanted The upper bound for the number of results to return.
1147
- * @param[out] matches Contiguous output buffer for the labels of closest members.
1148
- * @param[out] distances Contiguous output buffer for the distances to the closest members.
1149
- * @param[in] config Configuration options for this specific operation.
1150
- * @return The number of found approximate matches. Always equal or less then ::wanted.
1151
- */
1152
- search_result_t search( //
1153
- vector_view_t query, std::size_t wanted, //
1154
- label_t* matches, distance_t* distances, search_config_t config = {}) const noexcept(false) {
1419
+ struct stats_t {
1420
+ std::size_t nodes;
1421
+ std::size_t edges;
1422
+ std::size_t max_edged;
1423
+ };
1155
1424
 
1156
- auto callback = [&](label_t label, distance_t distance) noexcept {
1157
- if (matches)
1158
- *(matches++) = label;
1159
- if (distances)
1160
- *(distances++) = distance;
1161
- };
1162
- return search(query, wanted, callback, config);
1425
+ stats_t stats() const noexcept {
1426
+ stats_t result{};
1427
+ result.nodes = nodes_.size();
1428
+ for (std::size_t i = 0; i != result.nodes; ++i) {
1429
+ node_t node = node_with_id_(i);
1430
+ std::size_t max_edges = node.level() * config_.connectivity + base_level_multiple_() * config_.connectivity;
1431
+ std::size_t edges = 0;
1432
+ for (level_t level = 0; level != node.level(); ++level)
1433
+ edges += neighbors_(node, level).size();
1434
+
1435
+ result.edges += edges;
1436
+ result.max_edges += max_edges;
1437
+ }
1438
+ return result;
1439
+ }
1440
+
1441
+ stats_t stats(std::size_t level) const noexcept {
1442
+ stats_t result{};
1443
+ result.nodes = nodes_.size();
1444
+ for (std::size_t i = 0; i != result.nodes; ++i) {
1445
+ node_t node = node_with_id_(i);
1446
+ if (node.level() >= level)
1447
+ result.edges += neighbors_(node, level).size();
1448
+ }
1449
+ std::size_t max_edges_per_node = level ? config_.connectivity : base_level_multiple_() * config_.connectivity;
1450
+ result.max_edges = result.nodes * max_edges_per_node;
1451
+ return result;
1163
1452
  }
1164
1453
 
1165
1454
  #pragma endregion
@@ -1201,32 +1490,25 @@ class index_gt {
1201
1490
  if (!file)
1202
1491
  throw std::runtime_error(std::strerror(errno));
1203
1492
 
1204
- // Write the header
1205
- {
1206
- std::size_t written = std::fwrite(&state, sizeof(state), 1, file);
1493
+ auto write_chunk = [&](void* begin, std::size_t length) {
1494
+ std::size_t written = std::fwrite(begin, length, 1, file);
1207
1495
  if (!written) {
1208
1496
  std::fclose(file);
1209
1497
  throw std::runtime_error(std::strerror(errno));
1210
1498
  }
1211
- }
1499
+ };
1500
+
1501
+ // Write the header
1502
+ write_chunk(&state, sizeof(state));
1212
1503
 
1213
1504
  // Serialize nodes one by one
1214
1505
  for (std::size_t i = 0; i != state.size; ++i) {
1215
- node_ref_t node_ref = node(static_cast<id_t>(i));
1216
- std::size_t bytes_to_dump = node_dump_size(node_ref.head.dim, node_ref.head.level);
1217
- std::size_t bytes_in_vec = node_ref.head.dim * sizeof(scalar_t);
1218
- // Dump just neighbors, as vectors may be in a disjoint location
1219
- std::size_t written = std::fwrite(&node_ref.head, bytes_to_dump - bytes_in_vec, 1, file);
1220
- if (!written) {
1221
- std::fclose(file);
1222
- throw std::runtime_error(std::strerror(errno));
1223
- }
1224
- // Dump the vector
1225
- written = std::fwrite(node_ref.vector, bytes_in_vec, 1, file);
1226
- if (!written) {
1227
- std::fclose(file);
1228
- throw std::runtime_error(std::strerror(errno));
1229
- }
1506
+ node_t node = node_with_id_(i);
1507
+ std::size_t node_bytes = node_bytes_(node);
1508
+ std::size_t node_vector_bytes = node_vector_bytes_(node);
1509
+ // Dump neighbors and vectors, as vectors may be in a disjoint location
1510
+ write_chunk(node.tape(), node_bytes - node_vector_bytes);
1511
+ write_chunk(node.vector(), node_vector_bytes);
1230
1512
  }
1231
1513
 
1232
1514
  std::fclose(file);
@@ -1243,13 +1525,17 @@ class index_gt {
1243
1525
  if (!file)
1244
1526
  throw std::runtime_error(std::strerror(errno));
1245
1527
 
1246
- // Read the header
1247
- {
1248
- std::size_t read = std::fread(&state, sizeof(state), 1, file);
1528
+ auto read_chunk = [&](void* begin, std::size_t length) {
1529
+ std::size_t read = std::fread(begin, length, 1, file);
1249
1530
  if (!read) {
1250
1531
  std::fclose(file);
1251
1532
  throw std::runtime_error(std::strerror(errno));
1252
1533
  }
1534
+ };
1535
+
1536
+ // Read the header
1537
+ {
1538
+ read_chunk(&state, sizeof(state));
1253
1539
  if (state.bytes_per_label != sizeof(label_t)) {
1254
1540
  std::fclose(file);
1255
1541
  throw std::runtime_error("Incompatible label type!");
@@ -1261,7 +1547,7 @@ class index_gt {
1261
1547
 
1262
1548
  config_.connectivity = state.connectivity;
1263
1549
  config_.max_elements = state.size;
1264
- pre_ = precompute(config_);
1550
+ pre_ = precompute_(config_);
1265
1551
  reserve(state.size);
1266
1552
  size_ = state.size;
1267
1553
  max_level_ = static_cast<level_t>(state.max_level);
@@ -1270,22 +1556,17 @@ class index_gt {
1270
1556
 
1271
1557
  // Load nodes one by one
1272
1558
  for (std::size_t i = 0; i != state.size; ++i) {
1273
- node_head_t head;
1274
- std::size_t read = std::fread(&head, head_bytes_k, 1, file);
1275
- if (!read) {
1276
- std::fclose(file);
1277
- throw std::runtime_error(std::strerror(errno));
1278
- }
1279
-
1280
- std::size_t bytes_to_dump = node_dump_size(head.dim, head.level);
1281
- node_ref_t node_ref = node_malloc(head.label, {nullptr, head.dim}, head.level, true);
1282
- read = std::fread((byte_t*)&node_ref.head + head_bytes_k, bytes_to_dump - head_bytes_k, 1, file);
1283
- if (!read) {
1284
- std::fclose(file);
1285
- throw std::runtime_error(std::strerror(errno));
1286
- }
1287
-
1288
- nodes_[i] = node_ref;
1559
+ label_t label;
1560
+ dim_t dim;
1561
+ level_t level;
1562
+ read_chunk(&label, sizeof(label));
1563
+ read_chunk(&dim, sizeof(dim));
1564
+ read_chunk(&level, sizeof(level));
1565
+
1566
+ std::size_t node_bytes = node_bytes_(dim, level);
1567
+ node_t node = node_malloc_(label, {nullptr, dim}, level, true);
1568
+ read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
1569
+ nodes_[i] = node;
1289
1570
  }
1290
1571
 
1291
1572
  std::fclose(file);
@@ -1298,7 +1579,7 @@ class index_gt {
1298
1579
  * Available on Linux, MacOS, but @b not on Windows.
1299
1580
  */
1300
1581
  void view(char const* file_path) noexcept(false) {
1301
- #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
1582
+ #if defined(USEARCH_IS_WINDOWS)
1302
1583
  throw std::logic_error("Memory-mapping is not yet available for Windows");
1303
1584
  #else
1304
1585
  serialized_state_t state;
@@ -1338,7 +1619,7 @@ class index_gt {
1338
1619
  config_.connectivity = state.connectivity;
1339
1620
  config_.max_elements = state.size;
1340
1621
  config_.max_threads_add = 0;
1341
- pre_ = precompute(config_);
1622
+ pre_ = precompute_(config_);
1342
1623
  reserve(state.size);
1343
1624
  size_ = state.size;
1344
1625
  max_level_ = static_cast<level_t>(state.max_level);
@@ -1348,13 +1629,15 @@ class index_gt {
1348
1629
  // Locate every node packed into file
1349
1630
  std::size_t progress = sizeof(state);
1350
1631
  for (std::size_t i = 0; i != state.size; ++i) {
1351
- node_head_t const& head = *(node_head_t const*)(file + progress);
1352
- std::size_t bytes_to_dump = node_dump_size(head.dim, head.level);
1353
- std::size_t bytes_in_vec = head.dim * sizeof(scalar_t);
1354
- nodes_[i].tape_ = (byte_t*)(file + progress);
1355
- nodes_[i].vector_ = (scalar_t*)(file + progress + bytes_to_dump - bytes_in_vec);
1356
- progress += bytes_to_dump;
1357
- max_level_ = (std::max)(max_level_, head.level);
1632
+ byte_t* tape = (byte_t*)(file + progress);
1633
+ dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
1634
+ level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
1635
+
1636
+ std::size_t node_bytes = node_bytes_(dim, level);
1637
+ std::size_t node_vector_bytes = dim * sizeof(scalar_t);
1638
+ nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
1639
+ progress += node_bytes;
1640
+ max_level_ = (std::max)(max_level_, level);
1358
1641
  }
1359
1642
 
1360
1643
  viewed_file_descriptor_ = descriptor;
@@ -1364,123 +1647,114 @@ class index_gt {
1364
1647
  #pragma endregion
1365
1648
 
1366
1649
  private:
1367
- inline static precomputed_constants_t precompute(config_t const& config) noexcept {
1650
+ inline static precomputed_constants_t precompute_(config_t const& config) noexcept {
1368
1651
  precomputed_constants_t pre;
1369
- pre.connectivity_max_base = config.connectivity * base_level_multiple_k;
1652
+ pre.connectivity_max_base = config.connectivity * base_level_multiple_();
1370
1653
  pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
1371
1654
  pre.neighbors_bytes = config.connectivity * sizeof(id_t) + sizeof(neighbors_count_t);
1372
1655
  pre.neighbors_base_bytes = pre.connectivity_max_base * sizeof(id_t) + sizeof(neighbors_count_t);
1373
- pre.mutex_bytes = sizeof(mutex_t) * (config.max_threads_add > 1);
1374
1656
  return pre;
1375
1657
  }
1376
1658
 
1377
- inline std::size_t node_dump_size(dim_t dim, level_t level) const noexcept {
1378
- return head_bytes_k + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
1659
+ inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
1660
+ inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
1661
+ return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
1379
1662
  }
1380
1663
 
1381
- void node_free(std::size_t id) noexcept {
1664
+ inline bool node_stored_(node_t node) const noexcept {
1665
+ std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
1666
+ return (node.tape() + node_head_bytes_() + levels_bytes) == (byte_t*)node.vector();
1667
+ }
1668
+
1669
+ inline std::size_t node_vector_bytes_(dim_t dim) const noexcept { return dim * sizeof(scalar_t); }
1670
+ inline std::size_t node_vector_bytes_(node_t node) const noexcept { return node_vector_bytes_(node.dim()); }
1671
+
1672
+ void node_free_(std::size_t id) noexcept {
1382
1673
 
1383
1674
  if (viewed_file_descriptor_ != 0)
1384
1675
  return;
1385
1676
 
1386
- // This function is rarely called and can be as expensive as needed for higher space-efficiency.
1387
1677
  node_t& node = nodes_[id];
1388
- if (!node.tape_)
1389
- return;
1390
-
1391
- node_head_t const& head = *(node_head_t const*)(node.tape_ + pre_.mutex_bytes);
1392
- std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * head.level;
1393
- bool store_vector = (byte_t*)(node.tape_ + pre_.mutex_bytes + head_bytes_k + levels_bytes) == //
1394
- (byte_t*)(node.vector_);
1395
- std::size_t node_bytes = //
1396
- pre_.mutex_bytes + // Optional concurrency-control
1397
- head_bytes_k + levels_bytes + // Obligatory neighborhood index
1398
- head.dim * store_vector // Optional vector copy
1399
- ;
1400
-
1401
- allocator_t{}.deallocate(node.tape_, node_bytes);
1678
+ std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_stored_(node);
1679
+ allocator_t{}.deallocate(node.tape(), node_bytes);
1402
1680
  node = node_t{};
1403
1681
  }
1404
1682
 
1405
- node_ref_t node_malloc(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept(false) {
1683
+ node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept(false) {
1406
1684
 
1407
- // This function is rarely called and can be as expensive as needed for higher space-efficiency.
1408
1685
  std::size_t dim = vector.size();
1409
- std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
1410
- std::size_t node_bytes = //
1411
- pre_.mutex_bytes + // Optional concurrency-control
1412
- head_bytes_k + levels_bytes + // Obligatory neighborhood index
1413
- sizeof(scalar_t) * dim * store_vector // Optional vector copy
1414
- ;
1686
+ std::size_t stored_vector_bytes = node_vector_bytes_(dim) * store_vector;
1687
+ std::size_t node_bytes = node_bytes_(dim, level) - node_vector_bytes_(dim) * !store_vector;
1415
1688
 
1416
1689
  byte_t* data = (byte_t*)allocator_t{}.allocate(node_bytes);
1417
1690
  assert_m(data, "Not enough memory for links");
1691
+ std::memset(data, 0, node_bytes);
1692
+ if (vector.data())
1693
+ std::memcpy(data + node_bytes - stored_vector_bytes, vector.data(), stored_vector_bytes);
1418
1694
 
1419
- mutex_t* mutex = synchronize() ? (mutex_t*)data : nullptr;
1420
1695
  scalar_t* scalars = store_vector //
1421
- ? (scalar_t*)(data + pre_.mutex_bytes + head_bytes_k + levels_bytes)
1696
+ ? (scalar_t*)(data + node_bytes - stored_vector_bytes)
1422
1697
  : (scalar_t*)(vector.data());
1423
1698
 
1424
- std::memset(data, 0, node_bytes);
1425
- std::memcpy(scalars, vector.data(), sizeof(scalar_t) * dim * (store_vector && vector.data()));
1426
-
1427
- node_head_t& head = *(node_head_t*)(data + pre_.mutex_bytes);
1428
- head.label = label;
1429
- head.dim = static_cast<dim_t>(dim);
1430
- head.level = level;
1431
-
1432
- return {*mutex, head, scalars};
1699
+ node_t node{data, scalars};
1700
+ node.label(label);
1701
+ node.dim(static_cast<dim_t>(dim));
1702
+ node.level(level);
1703
+ return node;
1433
1704
  }
1434
1705
 
1435
- inline node_ref_t node(id_t id) const noexcept { return node(nodes_[id]); }
1706
+ inline node_t node_with_id_(std::size_t idx) const noexcept { return nodes_[idx]; }
1707
+ inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
1436
1708
 
1437
- inline node_ref_t node(node_t node) const noexcept {
1438
- byte_t* data = node.tape_;
1439
- mutex_t* mutex = synchronize() ? (mutex_t*)data : nullptr;
1440
- node_head_t& head = *(node_head_t*)(data + pre_.mutex_bytes);
1441
- scalar_t* scalars = node.vector_;
1709
+ inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
1710
+ return {node.neighbors_tape() + pre_.neighbors_base_bytes + (level - 1) * pre_.neighbors_bytes};
1711
+ }
1442
1712
 
1443
- return {*mutex, head, scalars};
1713
+ inline neighbors_ref_t neighbors_(node_t node, level_t level) const noexcept {
1714
+ return level ? neighbors_non_base_(node, level) : neighbors_base_(node);
1444
1715
  }
1445
1716
 
1446
- inline neighbors_ref_t neighbors_base(node_ref_t node) const noexcept { return {node.head.neighbors}; }
1717
+ struct node_lock_t {
1718
+ visits_bitset_t& bitset;
1719
+ std::size_t idx;
1447
1720
 
1448
- inline neighbors_ref_t neighbors_non_base(node_ref_t node, level_t level) const noexcept {
1449
- return {node.head.neighbors + pre_.neighbors_base_bytes + (level - 1) * pre_.neighbors_bytes};
1450
- }
1721
+ inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
1722
+ };
1451
1723
 
1452
- inline neighbors_ref_t neighbors(node_ref_t node, level_t level) const noexcept {
1453
- return level ? neighbors_non_base(node, level) : neighbors_base(node);
1724
+ inline node_lock_t node_lock_(std::size_t idx) const noexcept {
1725
+ while (nodes_mutexes_.atomic_set(idx))
1726
+ ;
1727
+ return {nodes_mutexes_, idx};
1454
1728
  }
1455
1729
 
1456
- id_t connect_new_element(id_t new_id, level_t level, thread_context_t& context) noexcept(false) {
1730
+ id_t connect_new_node_(id_t new_id, level_t level, thread_context_t& context) noexcept(false) {
1457
1731
 
1458
- node_ref_t new_node = node(new_id);
1732
+ node_t new_node = node_with_id_(new_id);
1459
1733
  top_candidates_t& top = context.top_candidates;
1460
1734
  std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
1461
1735
 
1462
1736
  // Outgoing links from `new_id`:
1463
- neighbors_ref_t new_neighbors = neighbors(new_node, level);
1737
+ neighbors_ref_t new_neighbors = neighbors_(new_node, level);
1464
1738
  {
1465
1739
  assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
1466
- candidates_view_t top_view = refine(top, config_.connectivity, context);
1740
+ candidates_view_t top_view = refine_(top, config_.connectivity, context);
1467
1741
 
1468
1742
  for (std::size_t idx = 0; idx != top_view.size(); idx++) {
1469
1743
  assert_m(!new_neighbors[idx], "Possible memory corruption");
1470
- assert_m(level <= node(top_view[idx].second).head.level, "Linking to missing level");
1471
- new_neighbors.push_back(top_view[idx].second);
1744
+ assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
1745
+ new_neighbors.push_back(top_view[idx].id);
1472
1746
  }
1473
1747
  }
1474
1748
 
1475
1749
  // Reverse links from the neighbors:
1476
1750
  for (id_t close_id : new_neighbors) {
1477
- node_ref_t close_node = node(close_id);
1478
- lock_t close_lock = close_node.lock();
1751
+ node_t close_node = node_with_id_(close_id);
1752
+ node_lock_t close_lock = node_lock_(close_id);
1479
1753
 
1480
- neighbors_ref_t close_header = neighbors(close_node, level);
1754
+ neighbors_ref_t close_header = neighbors_(close_node, level);
1481
1755
  assert_m(close_header.size() <= connectivity_max, "Possible corruption");
1482
1756
  assert_m(close_id != new_id, "Self-loops are impossible");
1483
- assert_m(level <= close_node.head.level, "Linking to missing level");
1757
+ assert_m(level <= close_node.level(), "Linking to missing level");
1484
1758
 
1485
1759
  // If `new_id` is already present in the neighboring connections of `close_id`
1486
1760
  // then no need to modify any connections or run the heuristics.
@@ -1494,39 +1768,39 @@ class index_gt {
1494
1768
  top.reserve(close_header.size() + 1);
1495
1769
  top.emplace_reserved(context.measure(new_node, close_node), new_id);
1496
1770
  for (id_t successor_id : close_header)
1497
- top.emplace_reserved(context.measure(node(successor_id), close_node), successor_id);
1771
+ top.emplace_reserved(context.measure(node_with_id_(successor_id), close_node), successor_id);
1498
1772
 
1499
1773
  // Export the results:
1500
1774
  close_header.clear();
1501
- candidates_view_t top_view = refine(top, connectivity_max, context);
1775
+ candidates_view_t top_view = refine_(top, connectivity_max, context);
1502
1776
  for (std::size_t idx = 0; idx != top_view.size(); idx++)
1503
- close_header.push_back(top_view[idx].second);
1777
+ close_header.push_back(top_view[idx].id);
1504
1778
  }
1505
1779
 
1506
1780
  return new_neighbors[0];
1507
1781
  }
1508
1782
 
1509
- level_t choose_random_level(std::default_random_engine& level_generator) const noexcept {
1783
+ level_t choose_random_level_(std::default_random_engine& level_generator) const noexcept {
1510
1784
  std::uniform_real_distribution<double> distribution(0.0, 1.0);
1511
1785
  double r = -std::log(distribution(level_generator)) * pre_.inverse_log_connectivity;
1512
1786
  return (level_t)r;
1513
1787
  }
1514
1788
 
1515
- id_t search_for_one( //
1789
+ id_t search_for_one_( //
1516
1790
  id_t closest_id, vector_view_t query, //
1517
1791
  level_t begin_level, level_t end_level, //
1518
1792
  thread_context_t& context) const noexcept {
1519
1793
 
1520
- distance_t closest_dist = context.measure(query, node(closest_id));
1794
+ distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
1521
1795
  for (level_t level = begin_level; level > end_level; level--) {
1522
1796
  bool changed;
1523
1797
  do {
1524
1798
  changed = false;
1525
- node_ref_t closest_node = node(closest_id);
1526
- lock_t closest_lock = closest_node.lock();
1527
- neighbors_ref_t closest_header = neighbors_non_base(closest_node, level);
1528
- for (id_t candidate_id : closest_header) {
1529
- distance_t candidate_dist = context.measure(query, node(candidate_id));
1799
+ node_t closest_node = node_with_id_(closest_id);
1800
+ node_lock_t closest_lock = node_lock_(closest_id);
1801
+ neighbors_ref_t closest_neighbors = neighbors_non_base_(closest_node, level);
1802
+ for (id_t candidate_id : closest_neighbors) {
1803
+ distance_t candidate_dist = context.measure(query, node_with_id_(candidate_id));
1530
1804
  if (candidate_dist < closest_dist) {
1531
1805
  closest_dist = candidate_dist;
1532
1806
  closest_id = candidate_id;
@@ -1539,7 +1813,7 @@ class index_gt {
1539
1813
  return closest_id;
1540
1814
  }
1541
1815
 
1542
- void search_to_insert( //
1816
+ void search_to_insert_( //
1543
1817
  id_t start_id, vector_view_t query, level_t level, //
1544
1818
  thread_context_t& context) noexcept(false) {
1545
1819
 
@@ -1553,7 +1827,7 @@ class index_gt {
1553
1827
  top.clear();
1554
1828
  top.reserve(top_limit + 1);
1555
1829
 
1556
- distance_t radius = context.measure(query, node(start_id));
1830
+ distance_t radius = context.measure(query, node_with_id_(start_id));
1557
1831
  candidates.emplace(-radius, start_id);
1558
1832
  top.emplace_reserved(radius, start_id);
1559
1833
  visits.set(start_id);
@@ -1561,37 +1835,37 @@ class index_gt {
1561
1835
  while (!candidates.empty()) {
1562
1836
 
1563
1837
  candidate_t candidacy = candidates.top();
1564
- if ((-candidacy.first) > radius && top.size() == top_limit)
1838
+ if ((-candidacy.distance) > radius && top.size() == top_limit)
1565
1839
  break;
1566
1840
 
1567
1841
  candidates.pop();
1568
1842
  context.iteration_cycles++;
1569
1843
 
1570
- id_t candidate_id = candidacy.second;
1571
- node_ref_t candidate_node = node(candidate_id);
1572
- lock_t candidate_lock = candidate_node.lock();
1573
- neighbors_ref_t candidate_header = neighbors(candidate_node, level);
1844
+ id_t candidate_id = candidacy.id;
1845
+ node_t candidate_ref = node_with_id_(candidate_id);
1846
+ node_lock_t candidate_lock = node_lock_(candidate_id);
1847
+ neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
1574
1848
 
1575
- prefetch_neighbors(candidate_header, visits);
1576
- for (id_t successor_id : candidate_header) {
1849
+ prefetch_neighbors_(candidate_neighbors, visits);
1850
+ for (id_t successor_id : candidate_neighbors) {
1577
1851
  if (visits.test(successor_id))
1578
1852
  continue;
1579
1853
 
1580
1854
  visits.set(successor_id);
1581
- distance_t successor_dist = context.measure(query, node(successor_id));
1855
+ distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
1582
1856
 
1583
1857
  if (top.size() < top_limit || successor_dist < radius) {
1584
1858
  candidates.emplace(-successor_dist, successor_id);
1585
1859
  top.emplace_reserved(successor_dist, successor_id);
1586
1860
  if (top.size() > top_limit)
1587
1861
  top.pop();
1588
- radius = top.top().first;
1862
+ radius = top.top().distance;
1589
1863
  }
1590
1864
  }
1591
1865
  }
1592
1866
  }
1593
1867
 
1594
- void search_to_find_in_base( //
1868
+ void search_to_find_in_base_( //
1595
1869
  id_t start_id, vector_view_t query, std::size_t expansion, //
1596
1870
  thread_context_t& context) const noexcept(false) {
1597
1871
 
@@ -1605,7 +1879,7 @@ class index_gt {
1605
1879
  top.clear();
1606
1880
  top.reserve(top_limit + 1);
1607
1881
 
1608
- distance_t radius = context.measure(query, node(start_id));
1882
+ distance_t radius = context.measure(query, node_with_id_(start_id));
1609
1883
  candidates.emplace(-radius, start_id);
1610
1884
  top.emplace_reserved(radius, start_id);
1611
1885
  visits.set(start_id);
@@ -1613,43 +1887,43 @@ class index_gt {
1613
1887
  while (!candidates.empty()) {
1614
1888
 
1615
1889
  candidate_t candidate = candidates.top();
1616
- if ((-candidate.first) > radius)
1890
+ if ((-candidate.distance) > radius)
1617
1891
  break;
1618
1892
 
1619
1893
  candidates.pop();
1620
1894
  context.iteration_cycles++;
1621
1895
 
1622
- id_t candidate_id = candidate.second;
1623
- neighbors_ref_t candidate_header = neighbors_base(node(candidate_id));
1896
+ id_t candidate_id = candidate.id;
1897
+ neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
1624
1898
 
1625
- prefetch_neighbors(candidate_header, visits);
1626
- for (id_t successor_id : candidate_header) {
1899
+ prefetch_neighbors_(candidate_neighbors, visits);
1900
+ for (id_t successor_id : candidate_neighbors) {
1627
1901
  if (visits.test(successor_id))
1628
1902
  continue;
1629
1903
 
1630
1904
  visits.set(successor_id);
1631
- distance_t successor_dist = context.measure(query, node(successor_id));
1905
+ distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
1632
1906
 
1633
1907
  if (top.size() < top_limit || successor_dist < radius) {
1634
1908
  candidates.emplace(-successor_dist, successor_id);
1635
1909
  top.emplace_reserved(successor_dist, successor_id);
1636
1910
  if (top.size() > top_limit)
1637
1911
  top.pop();
1638
- radius = top.top().first;
1912
+ radius = top.top().distance;
1639
1913
  }
1640
1914
  }
1641
1915
  }
1642
1916
  }
1643
1917
 
1644
- void search_exact(vector_view_t query, std::size_t count, thread_context_t& context) const noexcept(false) {
1918
+ void search_exact_(vector_view_t query, std::size_t count, thread_context_t& context) const noexcept(false) {
1645
1919
  top_candidates_t& top = context.top_candidates;
1646
1920
  top.clear();
1647
1921
  top.reserve(count);
1648
1922
  for (std::size_t i = 0; i != nodes_.size(); ++i)
1649
- top.emplace_reserved(context.measure(query, node(nodes_[i])), static_cast<id_t>(i));
1923
+ top.emplace_reserved(context.measure(query, node_with_id_(i)), static_cast<id_t>(i));
1650
1924
  }
1651
1925
 
1652
- void prefetch_neighbors(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
1926
+ void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
1653
1927
 
1654
1928
  /**
1655
1929
  * @brief This algorithm from the original paper implements a heuristic,
@@ -1657,7 +1931,7 @@ class index_gt {
1657
1931
  * to keep only the neighbors, that are from each other.
1658
1932
  */
1659
1933
 
1660
- candidates_view_t refine( //
1934
+ candidates_view_t refine_( //
1661
1935
  top_candidates_t& top, std::size_t needed, //
1662
1936
  thread_context_t& context) const noexcept {
1663
1937
 
@@ -1671,13 +1945,13 @@ class index_gt {
1671
1945
  std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
1672
1946
  while (submitted_count < needed && consumed_count < top_count) {
1673
1947
  candidate_t candidate = top_data[consumed_count];
1674
- node_ref_t candidate_node = node(candidate.second);
1675
- distance_t candidate_dist = candidate.first;
1948
+ node_t candidate_ref = node_with_id_(candidate.id);
1949
+ distance_t candidate_dist = candidate.distance;
1676
1950
  bool good = true;
1677
1951
  for (std::size_t idx = 0; idx < submitted_count; idx++) {
1678
1952
  candidate_t submitted = top_data[idx];
1679
- node_ref_t submitted_node = node(submitted.second);
1680
- distance_t inter_result_dist = context.measure(submitted_node, candidate_node);
1953
+ node_t submitted_node = node_with_id_(submitted.id);
1954
+ distance_t inter_result_dist = context.measure(submitted_node, candidate_ref);
1681
1955
  if (inter_result_dist < candidate_dist) {
1682
1956
  good = false;
1683
1957
  break;