usearch 0.2.4 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/.clang-tidy +60 -0
  2. package/.github/ISSUE_TEMPLATE/bug_report.yml +90 -0
  3. package/.github/ISSUE_TEMPLATE/config.yml +5 -0
  4. package/.github/ISSUE_TEMPLATE/feature_request.yml +64 -0
  5. package/.github/workflows/release.yml +47 -20
  6. package/.github/workflows/update_version.sh +1 -0
  7. package/.vscode/launch.json +22 -0
  8. package/.vscode/settings.json +9 -1
  9. package/.vscode/tasks.json +4 -4
  10. package/CITATION.cff +11 -0
  11. package/CMakeLists.txt +108 -95
  12. package/CODE_OF_CONDUCT.md +128 -0
  13. package/CONTRIBUTING.md +12 -0
  14. package/Cargo.toml +1 -1
  15. package/Dockerfile +14 -0
  16. package/README.md +21 -12
  17. package/VERSION +1 -1
  18. package/binding.gyp +13 -9
  19. package/build.rs +2 -1
  20. package/cmake/config.cmake.in +15 -0
  21. package/cmake/pkg-config.pc.in +4 -0
  22. package/cmake/usearchConfigVersion.cmake.in +20 -0
  23. package/conanfile.py +1 -1
  24. package/docs/{Benchmarking.md → benchmarks.md} +11 -2
  25. package/docs/benchmarks.rst +5 -0
  26. package/docs/{Compilation.md → compilation.md} +54 -27
  27. package/docs/compilation.rst +5 -0
  28. package/docs/conf.dox +2668 -0
  29. package/docs/index.rst +7 -0
  30. package/include/usearch/usearch.hpp +833 -439
  31. package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +17 -16
  32. package/javascript/lib.cpp +94 -20
  33. package/javascript/test.js +1 -1
  34. package/objc/USearchObjective.mm +47 -30
  35. package/objc/include/USearchObjective.h +18 -26
  36. package/package.json +1 -1
  37. package/pyproject.toml +3 -2
  38. package/python/lib.cpp +216 -130
  39. package/python/{bench.py → scripts/bench.py} +6 -1
  40. package/python/scripts/bench_params.py +112 -0
  41. package/python/{test.py → scripts/test.py} +32 -13
  42. package/python/usearch/client.py +100 -14
  43. package/python/usearch/server.py +104 -23
  44. package/rust/lib.cpp +20 -15
  45. package/rust/lib.hpp +5 -5
  46. package/rust/lib.rs +3 -3
  47. package/setup.py +7 -0
  48. package/src/CMakeLists.txt +107 -0
  49. package/src/bench.cpp +54 -19
  50. package/src/{advanced.hpp → punned.hpp} +235 -203
  51. package/src/test.cpp +62 -0
  52. package/swift/Index+Sugar.swift +1 -0
  53. package/swift/Test.swift +2 -1
  54. package/wolfram/lib.cpp +14 -14
@@ -9,51 +9,84 @@
9
9
  #ifndef UNUM_USEARCH_H
10
10
  #define UNUM_USEARCH_H
11
11
 
12
+ // Inferring C++ version
13
+ #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L)
14
+ #define USEARCH_IS_CPP17
15
+ #endif
16
+
17
+ // Inferring target OS
12
18
  #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
13
- #define _USE_MATH_DEFINES
19
+ #define USEARCH_IS_WINDOWS
20
+ #elif defined(__APPLE__) && defined(__MACH__)
21
+ #define USEARCH_IS_APPLE
22
+ #elif defined(__linux__)
23
+ #define USEARCH_IS_LINUX
24
+ #endif
14
25
 
15
- #include <Windows.h>
26
+ // Inferring the compiler
27
+ #if defined(__clang__)
28
+ #define USEARCH_IS_CLANG
29
+ #elif defined(__GNUC__)
30
+ #define USEARCH_IS_GCC
31
+ #endif
16
32
 
17
- #define usearch_pack_m
18
- #define usearch_align_m __declspec(align(64))
19
- #define WINDOWS
33
+ // Inferring hardware architecture: x86 vs Arm
34
+ #if defined(__x86_64__)
35
+ #define USEARCH_IS_X86
36
+ #elif defined(__aarch64__)
37
+ #define USEARCH_IS_ARM
38
+ #endif
20
39
 
40
+ // OS-specific includes
41
+ #if defined(USEARCH_IS_WINDOWS)
42
+ #define _USE_MATH_DEFINES
43
+ #include <Windows.h>
44
+ #include <sys/stat.h> // `fstat` for file size
21
45
  #else
22
46
  #include <fcntl.h> // `fallocate`
23
47
  #include <stdlib.h> // `posix_memalign`
24
48
  #include <sys/mman.h> // `mmap`
49
+ #include <sys/stat.h> // `fstat` for file size
25
50
  #include <unistd.h> // `open`, `close`
26
-
27
- #define usearch_pack_m __attribute__((packed))
28
- #define usearch_align_m __attribute__((aligned(64)))
29
51
  #endif
30
52
 
31
- #include <sys/stat.h> // `fstat` for file size
32
-
53
+ // STL includes
33
54
  #include <algorithm> // `std::sort_heap`
34
55
  #include <atomic> // `std::atomic`
35
56
  #include <bitset> // `std::bitset`
36
57
  #include <climits> // `CHAR_BIT`
37
58
  #include <cmath> // `std::sqrt`
38
59
  #include <cstring> // `std::memset`
60
+ #include <iterator> // `std::reverse_iterator`
39
61
  #include <mutex> // `std::unique_lock` - replacement candidate
40
62
  #include <random> // `std::default_random_engine` - replacement candidate
41
63
  #include <stdexcept> // `std::runtime_exception`
42
- #include <utility> // `std::exchange`
64
+ #include <utility> // `std::exchange`, `std::pair`
43
65
  #include <vector> // `std::vector`
44
66
 
45
- #if defined(__GNUC__)
67
+ // Prefetching
68
+ #if defined(USEARCH_IS_GCC)
46
69
  // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html
47
70
  // Zero means we are only going to read from that memory.
48
71
  // Three means high temporal locality and suggests to keep
49
72
  // the data in all layers of cache.
50
73
  #define prefetch_m(ptr) __builtin_prefetch((void*)(ptr), 0, 3)
51
- #elif defined(__x86_64__)
74
+ #elif defined(USEARCH_IS_X86)
52
75
  #define prefetch_m(ptr) _mm_prefetch((void*)(ptr), _MM_HINT_T0)
53
76
  #else
54
77
  #define prefetch_m(ptr)
55
78
  #endif
56
79
 
80
+ // Alignment
81
+ #if defined(USEARCH_IS_WINDOWS)
82
+ #define usearch_pack_m
83
+ #define usearch_align_m __declspec(align(64))
84
+ #else
85
+ #define usearch_pack_m __attribute__((packed))
86
+ #define usearch_align_m __attribute__((aligned(64)))
87
+ #endif
88
+
89
+ // Debugging
57
90
  #if defined(NDEBUG)
58
91
  #define assert_m(must_be_true, message)
59
92
  #else
@@ -68,11 +101,16 @@ namespace usearch {
68
101
 
69
102
  using f64_t = double;
70
103
  using f32_t = float;
104
+ using byte_t = char;
71
105
 
72
106
  template <typename at> at angle_to_radians(at angle) noexcept { return angle * M_PI / 180.f; }
73
107
 
74
108
  template <typename at> at square(at value) noexcept { return value * value; }
75
109
 
110
+ template <std::size_t multiple_ak> inline std::size_t divide_round_up(std::size_t num) noexcept {
111
+ return (num + multiple_ak - 1) / multiple_ak;
112
+ }
113
+
76
114
  inline std::size_t ceil2(std::size_t v) noexcept {
77
115
  v--;
78
116
  v |= v >> 1;
@@ -85,6 +123,71 @@ inline std::size_t ceil2(std::size_t v) noexcept {
85
123
  return v;
86
124
  }
87
125
 
126
+ template <typename at> void misaligned_store(void* ptr, at v) noexcept { std::memcpy(ptr, &v, sizeof(v)); }
127
+
128
+ template <typename at> at misaligned_load(void* ptr) noexcept {
129
+ at v;
130
+ std::memcpy(&v, ptr, sizeof(v));
131
+ return v;
132
+ }
133
+
134
+ template <typename at> class misaligned_ref_gt {
135
+ byte_t* ptr_;
136
+
137
+ public:
138
+ misaligned_ref_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
139
+ operator at() const noexcept { return misaligned_load<at>(ptr_); }
140
+ misaligned_ref_gt& operator=(at const& v) noexcept {
141
+ misaligned_store<at>(ptr_, v);
142
+ return *this;
143
+ }
144
+ misaligned_ref_gt& operator=(at v) noexcept {
145
+ misaligned_store<at>(ptr_, v);
146
+ return *this;
147
+ }
148
+ };
149
+
150
+ template <typename at> class misaligned_ptr_gt {
151
+ byte_t* ptr_;
152
+
153
+ public:
154
+ using iterator_category = std::random_access_iterator_tag;
155
+ using value_type = at;
156
+ using difference_type = std::ptrdiff_t;
157
+ using pointer = misaligned_ptr_gt<at>;
158
+ using reference = misaligned_ref_gt<at>;
159
+
160
+ reference operator*() const noexcept { return {ptr_}; }
161
+
162
+ misaligned_ptr_gt(byte_t* ptr) noexcept : ptr_(ptr) {}
163
+ misaligned_ptr_gt operator++(int) noexcept { return misaligned_ptr_gt(ptr_ + sizeof(at)); }
164
+ misaligned_ptr_gt operator--(int) noexcept { return misaligned_ptr_gt(ptr_ - sizeof(at)); }
165
+ misaligned_ptr_gt operator+(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ + d * sizeof(at)); }
166
+ misaligned_ptr_gt operator-(difference_type d) noexcept { return misaligned_ptr_gt(ptr_ - d * sizeof(at)); }
167
+
168
+ misaligned_ptr_gt& operator++() noexcept {
169
+ ptr_ += sizeof(at);
170
+ return *this;
171
+ }
172
+ misaligned_ptr_gt& operator--() noexcept {
173
+ ptr_ -= sizeof(at);
174
+ return *this;
175
+ }
176
+ misaligned_ptr_gt& operator+=(difference_type d) noexcept {
177
+ ptr_ += d * sizeof(at);
178
+ return *this;
179
+ }
180
+ misaligned_ptr_gt& operator-=(difference_type d) noexcept {
181
+ ptr_ -= d * sizeof(at);
182
+ return *this;
183
+ }
184
+ bool operator==(misaligned_ptr_gt const& other) noexcept { return ptr_ == other.ptr_; }
185
+ bool operator!=(misaligned_ptr_gt const& other) noexcept { return ptr_ != other.ptr_; }
186
+ };
187
+
188
+ /**
189
+ * @brief Inner (Dot) Product distance.
190
+ */
88
191
  template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
89
192
  using scalar_t = scalar_at;
90
193
  using result_t = result_at;
@@ -92,12 +195,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
92
195
 
93
196
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
94
197
  result_type ab{};
95
- #if defined(__GNUC__)
96
- #pragma GCC ivdep
97
- #elif defined(__clang__)
98
- #pragma clang loop vectorize(enable)
99
- #elif defined(_OPENMP)
198
+ #if defined(USEARCH_USE_OPENMP)
100
199
  #pragma omp simd reduction(+ : ab)
200
+ #elif defined(USEARCH_IS_CLANG)
201
+ #pragma clang loop vectorize(enable)
202
+ #elif defined(USEARCH_IS_GCC)
203
+ #pragma GCC ivdep
101
204
  #endif
102
205
  for (std::size_t i = 0; i != dim; ++i)
103
206
  ab += result_t(a[i]) * result_t(b[i]);
@@ -105,6 +208,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct ip_gt {
105
208
  }
106
209
  };
107
210
 
211
+ /**
212
+ * @brief Cosine (Angular) distance.
213
+ * Identical to the Inner Product of normalized vectors.
214
+ * Unless you are running on an tiny embedded platform, this metric
215
+ * is recommended over `::ip_gt` for low-precision scalars.
216
+ */
108
217
  template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
109
218
  using scalar_t = scalar_at;
110
219
  using result_t = result_at;
@@ -112,21 +221,25 @@ template <typename scalar_at, typename result_at = scalar_at> struct cos_gt {
112
221
 
113
222
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
114
223
  result_t ab{}, a2{}, b2{};
115
- #if defined(__GNUC__)
116
- #pragma GCC ivdep
117
- #elif defined(__clang__)
118
- #pragma clang loop vectorize(enable)
119
- #elif defined(_OPENMP)
224
+ #if defined(USEARCH_USE_OPENMP)
120
225
  #pragma omp simd reduction(+ : ab, a2, b2)
226
+ #elif defined(USEARCH_IS_CLANG)
227
+ #pragma clang loop vectorize(enable)
228
+ #elif defined(USEARCH_IS_GCC)
229
+ #pragma GCC ivdep
121
230
  #endif
122
231
  for (std::size_t i = 0; i != dim; ++i)
123
232
  ab += result_t(a[i]) * result_t(b[i]), //
124
233
  a2 += square<result_t>(a[i]), //
125
234
  b2 += square<result_t>(b[i]);
126
- return 1 - ab / (std::sqrt(a2) * std::sqrt(b2));
235
+ return ab ? (1 - ab / (std::sqrt(a2) * std::sqrt(b2))) : 0;
127
236
  }
128
237
  };
129
238
 
239
+ /**
240
+ * @brief Squared Euclidean (L2) distance.
241
+ * Square root is avoided at the end, as it won't affect the ordering.
242
+ */
130
243
  template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
131
244
  using scalar_t = scalar_at;
132
245
  using result_t = result_at;
@@ -134,12 +247,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct l2sq_gt {
134
247
 
135
248
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
136
249
  result_t ab_deltas_sq{};
137
- #if defined(__GNUC__)
138
- #pragma GCC ivdep
139
- #elif defined(__clang__)
140
- #pragma clang loop vectorize(enable)
141
- #elif defined(_OPENMP)
250
+ #if defined(USEARCH_USE_OPENMP)
142
251
  #pragma omp simd reduction(+ : ab_deltas_sq)
252
+ #elif defined(USEARCH_IS_CLANG)
253
+ #pragma clang loop vectorize(enable)
254
+ #elif defined(USEARCH_IS_GCC)
255
+ #pragma GCC ivdep
143
256
  #endif
144
257
  for (std::size_t i = 0; i != dim; ++i)
145
258
  ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
@@ -156,17 +269,16 @@ template <typename scalar_at, typename result_at = std::size_t> struct hamming_g
156
269
  using result_t = result_at;
157
270
  using result_type = result_t;
158
271
 
159
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t elements,
160
- std::size_t = 0) const noexcept {
272
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
161
273
  result_t matches{};
162
- #if defined(__GNUC__)
163
- #pragma GCC ivdep
164
- #elif defined(__clang__)
165
- #pragma clang loop vectorize(enable)
166
- #elif defined(_OPENMP)
274
+ #if defined(USEARCH_USE_OPENMP)
167
275
  #pragma omp simd reduction(+ : matches)
276
+ #elif defined(USEARCH_IS_CLANG)
277
+ #pragma clang loop vectorize(enable)
278
+ #elif defined(USEARCH_IS_GCC)
279
+ #pragma GCC ivdep
168
280
  #endif
169
- for (std::size_t i = 0; i != elements; ++i)
281
+ for (std::size_t i = 0; i != dim; ++i)
170
282
  matches += a[i] != b[i];
171
283
  return matches;
172
284
  }
@@ -182,17 +294,17 @@ template <typename scalar_at, typename result_at = std::size_t> struct bit_hammi
182
294
  using result_t = result_at;
183
295
  using result_type = result_t;
184
296
  static_assert(std::is_unsigned<scalar_t>::value, "Hamming distance requires unsigned integral words");
185
- static constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
186
297
 
187
298
  inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words,
188
299
  std::size_t = 0) const noexcept {
300
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
189
301
  result_t matches{};
190
- #if defined(__GNUC__)
191
- #pragma GCC ivdep
192
- #elif defined(__clang__)
193
- #pragma clang loop vectorize(enable)
194
- #elif defined(_OPENMP)
302
+ #if defined(USEARCH_USE_OPENMP)
195
303
  #pragma omp simd reduction(+ : matches)
304
+ #elif defined(USEARCH_IS_CLANG)
305
+ #pragma clang loop vectorize(enable)
306
+ #elif defined(USEARCH_IS_GCC)
307
+ #pragma GCC ivdep
196
308
  #endif
197
309
  for (std::size_t i = 0; i != words; ++i)
198
310
  matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
@@ -239,12 +351,12 @@ template <typename scalar_at, typename result_at = scalar_at> struct pearson_cor
239
351
  scalar_t const* a, scalar_t const* b, std::size_t dim, std::size_t = 0) const noexcept {
240
352
  result_t a_sum{}, b_sum{}, ab_sum{};
241
353
  result_t a_sq_sum{}, b_sq_sum{};
242
- #if defined(__GNUC__)
243
- #pragma GCC ivdep
244
- #elif defined(__clang__)
245
- #pragma clang loop vectorize(enable)
246
- #elif defined(_OPENMP)
354
+ #if defined(USEARCH_USE_OPENMP)
247
355
  #pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
356
+ #elif defined(USEARCH_IS_CLANG)
357
+ #pragma clang loop vectorize(enable)
358
+ #elif defined(USEARCH_IS_GCC)
359
+ #pragma GCC ivdep
248
360
  #endif
249
361
  for (std::size_t i = 0; i != dim; ++i) {
250
362
  a_sum += a[i];
@@ -287,16 +399,21 @@ template <typename scalar_at, typename result_at = scalar_at> struct haversine_g
287
399
  }
288
400
  };
289
401
 
290
- template <std::size_t multiple_ak> inline std::size_t divide_round_up(std::size_t num) noexcept {
291
- return (num + multiple_ak - 1) / multiple_ak;
292
- }
293
-
402
+ /**
403
+ * @brief Light-weight bitset implementation to track visited nodes during graph traversal.
404
+ * Extends basic functionality with atomic operations.
405
+ */
294
406
  template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
295
407
  using allocator_t = allocator_at;
296
408
  using byte_t = typename allocator_t::value_type;
297
409
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
298
410
 
411
+ #if defined(USEARCH_IS_WINDOWS)
412
+ using slot_t = LONG volatile;
413
+ static_assert(sizeof(slot_t) == 8);
414
+ #else
299
415
  using slot_t = std::uint64_t;
416
+ #endif
300
417
 
301
418
  std::uint64_t* u64s_{};
302
419
  std::size_t slots_{};
@@ -306,6 +423,7 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
306
423
  visits_bitset_gt(std::size_t capacity) noexcept {
307
424
  slots_ = divide_round_up<64>(capacity);
308
425
  u64s_ = (slot_t*)allocator_t{}.allocate(slots_ * sizeof(slot_t));
426
+ clear();
309
427
  }
310
428
  ~visits_bitset_gt() noexcept { allocator_t{}.deallocate((byte_t*)u64s_, slots_ * sizeof(slot_t)), u64s_ = nullptr; }
311
429
  void clear() noexcept { std::memset(u64s_, 0, slots_ * sizeof(slot_t)); }
@@ -324,6 +442,42 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
324
442
  std::swap(slots_, other.slots_);
325
443
  return *this;
326
444
  }
445
+
446
+ #if defined(USEARCH_IS_WINDOWS)
447
+
448
+ inline bool atomic_test(std::size_t i) noexcept {
449
+ slot_t mask{1ul << (i & 63ul)};
450
+ return InterlockedOr(&u64s_[i / 64], 0) & mask;
451
+ }
452
+
453
+ inline bool atomic_set(std::size_t i) noexcept {
454
+ slot_t mask{1ul << (i & 63ul)};
455
+ return InterLockedOr(&u64s_[i / 64], mask) & mask;
456
+ }
457
+
458
+ inline void atomic_reset(std::size_t i) noexcept {
459
+ slot_t mask{1ul << (i & 63ul)};
460
+ InterLockedAnd((LONG volatile*)&u64s_[i / 64], ~mask);
461
+ }
462
+
463
+ #else
464
+
465
+ inline bool atomic_test(std::size_t i) noexcept {
466
+ slot_t mask{1ul << (i & 63ul)};
467
+ return __atomic_load_n(&u64s_[i / 64], __ATOMIC_RELAXED) & mask;
468
+ }
469
+
470
+ inline bool atomic_set(std::size_t i) noexcept {
471
+ slot_t mask{1ul << (i & 63ul)};
472
+ return __atomic_fetch_or(&u64s_[i / 64], mask, __ATOMIC_ACQUIRE) & mask;
473
+ }
474
+
475
+ inline void atomic_reset(std::size_t i) noexcept {
476
+ slot_t mask{1ul << (i & 63ul)};
477
+ __atomic_fetch_and(&u64s_[i / 64], ~mask, __ATOMIC_RELEASE);
478
+ }
479
+
480
+ #endif
327
481
  };
328
482
 
329
483
  using visits_bitset_t = visits_bitset_gt<>;
@@ -564,63 +718,13 @@ class sorted_buffer_gt {
564
718
  static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
565
719
  };
566
720
 
567
- /**
568
- *
569
- */
570
- class mutex_t {
571
- #if defined(WINDOWS)
572
- using slot_t = volatile LONG;
573
- #else
574
- using slot_t = std::int32_t;
575
- #endif // WINDOWS
576
-
577
- slot_t flag_;
578
-
579
- public:
580
- inline mutex_t(slot_t flag = 0) noexcept : flag_(flag) {}
581
- inline ~mutex_t() noexcept {}
582
-
583
- inline bool try_lock() noexcept {
584
- slot_t raw = 0;
585
- #if defined(WINDOWS)
586
- return InterlockedCompareExchange(&flag_, 1, raw);
587
- #else
588
- return __atomic_compare_exchange_n(&flag_, &raw, 1, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
589
- #endif // WINDOWS
590
- }
591
-
592
- inline void lock() noexcept {
593
- slot_t raw = 0;
594
- #if defined(WINDOWS)
595
- InterlockedCompareExchange(&flag_, 1, raw);
596
- #else
597
- lock_again:
598
- raw = 0;
599
- if (!__atomic_compare_exchange_n(&flag_, &raw, 1, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
600
- goto lock_again;
601
- #endif // WINDOWS
602
- }
603
-
604
- inline void unlock() noexcept {
605
- #if defined(WINDOWS)
606
- InterlockedExchange(&flag_, 0);
607
- #else
608
- __atomic_store_n(&flag_, 0, __ATOMIC_RELEASE);
721
+ #if defined(USEARCH_IS_WINDOWS)
722
+ #pragma pack(push, 1) // Pack struct members on 1-byte alignment
609
723
  #endif
610
- }
611
- };
612
-
613
- static_assert(sizeof(mutex_t) == sizeof(std::int32_t), "Mutex is larger than expected");
614
-
615
- using lock_t = std::unique_lock<mutex_t>;
616
724
 
617
725
  /**
618
726
  * @brief Five-byte integer type to address node clouds with over 4B entries.
619
727
  */
620
- #if defined(WINDOWS)
621
- #pragma pack(push, 1) // Pack struct members on 1-byte alignment
622
- #endif
623
-
624
728
  class usearch_pack_m uint40_t {
625
729
  unsigned char octets[5];
626
730
 
@@ -628,7 +732,7 @@ class usearch_pack_m uint40_t {
628
732
  inline uint40_t() noexcept { std::memset(octets, 0, 5); }
629
733
  inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets + 1, (char*)&n, 4), octets[0] = 0; }
630
734
  inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
631
- #if defined(__clang__)
735
+ #if defined(USEARCH_IS_CLANG)
632
736
  inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
633
737
  #endif
634
738
 
@@ -658,12 +762,16 @@ class usearch_pack_m uint40_t {
658
762
  return old;
659
763
  }
660
764
  };
661
- #if defined(WINDOWS)
765
+
766
+ #if defined(USEARCH_IS_WINDOWS)
662
767
  #pragma pack(pop) // Reset alignment to default
663
768
  #endif
664
769
 
665
770
  static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
666
771
 
772
+ /**
773
+ * @brief Non-owning memory range view, similar to `std::span`, but for C++11.
774
+ */
667
775
  template <typename scalar_at> class span_gt {
668
776
  scalar_at* data_;
669
777
  std::size_t size_;
@@ -676,48 +784,61 @@ template <typename scalar_at> class span_gt {
676
784
  operator scalar_at*() const noexcept { return data(); }
677
785
  };
678
786
 
787
+ constexpr std::size_t default_connectivity() { return 16; }
788
+ constexpr std::size_t default_expansion_add() { return 128; }
789
+ constexpr std::size_t default_expansion_search() { return 64; }
790
+
679
791
  /**
680
- * @brief
681
- * Configuration settings for the index construction.
682
- * Includes the main `::connectivity` parameter (`M` in the paper)
683
- * and two expansion factors - for construction and search.
792
+ * @brief Configuration settings for the index construction.
793
+ * Includes the main `::connectivity` parameter (`M` in the paper)
794
+ * and two expansion factors - for construction and search.
684
795
  */
685
796
  struct config_t {
686
797
 
687
798
  /// @brief Number of neighbors per graph node.
688
799
  /// Defaults to 32 in FAISS and 16 in hnswlib.
689
800
  /// > It is called `M` in the paper.
690
- static constexpr std::size_t connectivity_default_k = 16;
801
+ std::size_t connectivity = default_connectivity();
691
802
 
692
803
  /// @brief Hyper-parameter controlling the quality of indexing.
693
804
  /// Defaults to 40 in FAISS and 200 in hnswlib.
694
805
  /// > It is called `efConstruction` in the paper.
695
- static constexpr std::size_t expansion_add_default_k = 128;
806
+ std::size_t expansion_add = default_expansion_add();
696
807
 
697
808
  /// @brief Hyper-parameter controlling the quality of search.
698
809
  /// Defaults to 16 in FAISS and 10 in hnswlib.
699
810
  /// > It is called `ef` in the paper.
700
- static constexpr std::size_t expansion_search_default_k = 64;
701
-
702
- std::size_t connectivity = connectivity_default_k;
703
- std::size_t expansion_add = expansion_add_default_k;
704
- std::size_t expansion_search = expansion_search_default_k;
811
+ std::size_t expansion_search = default_expansion_search();
705
812
 
706
813
  ///
707
814
  std::size_t max_elements = 0;
708
- std::size_t max_threads_add = 0;
709
- std::size_t max_threads_search = 0;
815
+ std::size_t max_threads_add = 1;
816
+ std::size_t max_threads_search = 1;
817
+
818
+ std::size_t max_threads() const noexcept { return (std::max)(max_threads_add, max_threads_search); }
819
+ std::size_t concurrency() const noexcept { return (std::min)(max_threads_add, max_threads_search); }
820
+ };
821
+
822
+ struct add_config_t {
823
+ /// @brief Optional thread identifier for multi-threaded construction.
824
+ std::size_t thread{};
825
+ /// @brief Don't copy the ::vector, if it's persisted elsewhere.
826
+ bool store_vector{true};
827
+ };
828
+
829
+ struct search_config_t {
830
+ std::size_t thread{};
831
+ bool exact{};
710
832
  };
711
833
 
712
834
  /**
713
- * @brief
714
- * Approximate Nearest Neighbors Search index using the
715
- * Hierarchical Navigable Small World graph algorithm.
835
+ * @brief Approximate Nearest Neighbors Search index using the
836
+ * Hierarchical Navigable Small World graph algorithm.
716
837
  *
717
838
  * @section Features
718
- * - Search for vectors of different dimensionality.
719
- * - Thread-safe.
720
- * - Bring your threads!
839
+ * - Thread-safe for concurrent construction.
840
+ * - Doesn't allocate new threads, and reuses the ones its called from.
841
+ * - Search for vectors of different dimensionality, if ::`metric_at` supports that.
721
842
  *
722
843
  * @tparam metric_at
723
844
  * A function vector responsible for computing the distance between two vectors.
@@ -733,6 +854,40 @@ struct config_t {
733
854
  *
734
855
  * @tparam allocator_at
735
856
  * Dynamic memory allocator.
857
+ *
858
+ *
859
+ * @section Implementation Details
860
+ *
861
+ * Like every HNSW implementation, USearch builds levels of "Proximity Graphs".
862
+ * Every added vector forms a node in one or more levels of the graph.
863
+ * Every node is present in the base level. Every following level contains a smaller
864
+ * fraction of nodes. During search, the operation starts with the smaller levels
865
+ * and zooms-in on every following iteration of larger graph traversals.
866
+ *
867
+ * Just one memory allocation is performed regardless of the number of levels.
868
+ * The adjacency lists across all levels are concatenated into that single buffer.
869
+ * That buffer starts with a "head", that stores the metadata, such as the
870
+ * tallest "level" of the graph that it belongs to, the external "label", and the
871
+ * number of "dimensions" in the vector.
872
+ *
873
+ * @subsection Colocated Nodes
874
+ *
875
+ * Every node contains some graph-related data, as well as the original vector.
876
+ * Assuming the vectors can get large, we allow separating the index from the vectors.
877
+ * They can either be stored in a single continuous block of memory, or in two separate
878
+ * locations. Vector storage can also be detached from `index_gt` and managed by the user.
879
+ * To properly serialize and deserialize, use the right `save()`, `load()`, and `view()`
880
+ * overloads.
881
+ *
882
+ * @subsection Missing Functionality
883
+ *
884
+ * To simplify the implementation, the `index_gt` lacks endpoints to remove existing
885
+ * vectors. That, however, is solved by `punned_gt`, which also adds automatic casting.
886
+ *
887
+ * @subsection Ugly Implementation Parts
888
+ *
889
+ * Internally, a number of `std::vector`-s is created, often `mutable`.
890
+ *
736
891
  */
737
892
  template <typename metric_at = ip_gt<float>, //
738
893
  typename label_at = std::size_t, //
@@ -747,18 +902,104 @@ class index_gt {
747
902
  using id_t = id_at;
748
903
  using allocator_t = allocator_at;
749
904
 
905
+ /// C++17 and newer version deprecate the `std::result_of`
750
906
  using distance_t =
907
+ #if defined(USEARCH_IS_CPP17)
908
+ typename std::invoke_result<metric_t, scalar_t const*, scalar_t const*, std::size_t, std::size_t>::type;
909
+ #else
751
910
  typename std::result_of<metric_t(scalar_t const*, scalar_t const*, std::size_t, std::size_t)>::type;
911
+ #endif
912
+
913
+ using vector_view_t = span_gt<scalar_t const>;
914
+
915
+ struct member_ref_t {
916
+ misaligned_ref_gt<label_t> label;
917
+ vector_view_t vector;
918
+ };
919
+
920
+ struct member_cref_t {
921
+ label_t label;
922
+ vector_view_t vector;
923
+ };
924
+
925
+ template <typename ref_at> class member_iterator_gt {
926
+ using ref_t = ref_at;
927
+ index_gt& index_;
928
+ std::size_t offset_;
929
+
930
+ member_iterator_gt() noexcept {}
931
+ member_iterator_gt(index_gt& index, std::size_t offset) noexcept : index_(index), offset_(offset) {}
932
+
933
+ public:
934
+ using iterator_category = std::random_access_iterator_tag;
935
+ using value_type = ref_t;
936
+ using difference_type = std::ptrdiff_t;
937
+ using pointer = void;
938
+ using reference = ref_t;
939
+
940
+ reference operator*() const noexcept { return index_.node_with_id_(offset_); }
941
+ member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, offset_ + 1); }
942
+ member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, offset_ - 1); }
943
+ member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, offset_ + d); }
944
+ member_iterator_gt operator-(difference_type d) noexcept { return member_iterator_gt(index_, offset_ - d); }
945
+
946
+ member_iterator_gt& operator++() noexcept {
947
+ offset_ += 1;
948
+ return *this;
949
+ }
950
+ member_iterator_gt& operator--() noexcept {
951
+ offset_ -= 1;
952
+ return *this;
953
+ }
954
+ member_iterator_gt& operator+=(difference_type d) noexcept {
955
+ offset_ += d;
956
+ return *this;
957
+ }
958
+ member_iterator_gt& operator-=(difference_type d) noexcept {
959
+ offset_ -= d;
960
+ return *this;
961
+ }
962
+ bool operator!=(member_iterator_gt const& other) noexcept {
963
+ return &index_ != &other.index_ || offset_ != other.offset_;
964
+ }
965
+ };
966
+
967
+ using member_iterator_t = member_iterator_gt<member_ref_t>;
968
+ using member_citerator_t = member_iterator_gt<member_cref_t>;
969
+
970
+ // STL compatibility:
971
+ using value_type = std::pair<label_t, distance_t>;
972
+ using allocator_type = allocator_t;
973
+ using size_type = std::size_t;
974
+ using difference_type = std::ptrdiff_t;
975
+ using reference = member_ref_t;
976
+ using const_reference = member_cref_t;
977
+ using pointer = void;
978
+ using const_pointer = void;
979
+ using iterator = member_iterator_t;
980
+ using const_iterator = member_citerator_t;
981
+ using reverse_iterator = std::reverse_iterator<member_iterator_t>;
982
+ using reverse_const_iterator = std::reverse_iterator<member_citerator_t>;
752
983
 
753
984
  private:
754
- using neighbors_count_t = id_t;
985
+ using neighbors_count_t = std::uint32_t;
755
986
  using dim_t = std::uint32_t;
756
987
  using level_t = std::int32_t;
757
988
 
758
989
  using allocator_traits_t = std::allocator_traits<allocator_t>;
759
990
  using byte_t = typename allocator_t::value_type;
760
991
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
761
- static constexpr std::size_t base_level_multiple_k = 2;
992
+
993
+ /**
994
+ * @brief How much larger (number of neighbors per node) will the base level be
995
+ * comapred to other levels.
996
+ */
997
+ static constexpr std::size_t base_level_multiple_() { return 2; }
998
+
999
+ /**
1000
+ * @brief How many bytes of memory are needed to form the "head" of the node.
1001
+ */
1002
+ static constexpr std::size_t node_head_bytes_() { return sizeof(label_t) + sizeof(dim_t) + sizeof(level_t); }
762
1003
 
763
1004
  using visits_bitset_t = visits_bitset_gt<allocator_t>;
764
1005
 
@@ -767,14 +1008,13 @@ class index_gt {
767
1008
  std::size_t connectivity_max_base{};
768
1009
  std::size_t neighbors_bytes{};
769
1010
  std::size_t neighbors_base_bytes{};
770
- std::size_t mutex_bytes{};
771
1011
  };
772
1012
  struct candidate_t {
773
- distance_t first;
774
- id_t second;
1013
+ distance_t distance;
1014
+ id_t id;
775
1015
  };
776
1016
  struct compare_by_distance_t {
777
- inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.first < b.first; }
1017
+ inline bool operator()(candidate_t a, candidate_t b) const noexcept { return a.distance < b.distance; }
778
1018
  };
779
1019
 
780
1020
  using candidates_view_t = span_gt<candidate_t const>;
@@ -782,69 +1022,85 @@ class index_gt {
782
1022
  using top_candidates_t = sorted_buffer_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
783
1023
  using next_candidates_t = max_heap_gt<candidate_t, compare_by_distance_t, candidates_allocator_t>;
784
1024
 
785
- struct neighbors_ref_t {
786
- neighbors_count_t& count_;
787
- id_t* neighbors_{};
788
-
789
- inline neighbors_ref_t(byte_t* tape) noexcept
790
- : count_(*(neighbors_count_t*)tape), neighbors_((neighbors_count_t*)tape + 1) {}
791
- inline id_t* begin() noexcept { return neighbors_; }
792
- inline id_t* end() noexcept { return neighbors_ + count_; }
793
- inline id_t const* begin() const noexcept { return neighbors_; }
794
- inline id_t const* end() const noexcept { return neighbors_ + count_; }
795
- inline id_t& operator[](std::size_t i) noexcept { return neighbors_[i]; }
796
- inline id_t operator[](std::size_t i) const noexcept { return neighbors_[i]; }
797
- inline std::size_t size() const noexcept { return count_; }
798
- inline void clear() noexcept { count_ = 0u; }
799
- inline void push_back(id_t id) noexcept { neighbors_[count_] = id, count_++; }
800
- };
801
-
802
- #if defined(WINDOWS)
803
- #pragma pack(push, 1) // Pack struct members on 1-byte alignment
804
- #endif
805
- struct usearch_pack_m node_head_t {
806
- label_t label;
807
- dim_t dim;
808
- level_t level;
809
- // Variable length array, that has multiple similarly structured segments.
810
- // Each starts with a `neighbors_count_t` and is followed by such number of `id_t`s.
811
- byte_t neighbors[1];
812
- };
813
- #if defined(WINDOWS)
814
- #pragma pack(pop) // Reset alignment to default
815
- #endif
816
-
817
- static constexpr std::size_t head_bytes_k = sizeof(label_t) + sizeof(dim_t) + sizeof(level_t);
818
-
819
- struct node_t {
1025
+ /**
1026
+ * @brief A loosely-structured handle for every node. One such node is created for every vector.
1027
+ * To minimize memory usage and maximize the number of entries per cache-line, it only
1028
+ * stores to pointers. To reinterpret those into a structured `node_t` more
1029
+ * information is needed from the parent `index_gt`.
1030
+ */
1031
+ class node_t {
820
1032
  byte_t* tape_{};
821
1033
  scalar_t* vector_{};
822
1034
 
1035
+ public:
823
1036
  explicit node_t(byte_t* tape, scalar_t* vector) noexcept : tape_(tape), vector_(vector) {}
1037
+ byte_t* tape() const noexcept { return tape_; }
1038
+ scalar_t* vector() const noexcept { return vector_; }
1039
+ byte_t* neighbors_tape() const noexcept { return tape_ + node_head_bytes_(); }
824
1040
 
825
1041
  node_t() = default;
826
1042
  node_t(node_t const&) = default;
827
1043
  node_t& operator=(node_t const&) = default;
1044
+
1045
+ label_t label() const noexcept { return misaligned_load<label_t>(tape_); }
1046
+ dim_t dim() const noexcept { return misaligned_load<dim_t>(tape_ + sizeof(label_t)); }
1047
+ level_t level() const noexcept { return misaligned_load<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t)); }
1048
+
1049
+ void label(label_t v) noexcept { return misaligned_store<label_t>(tape_, v); }
1050
+ void dim(dim_t v) noexcept { return misaligned_store<dim_t>(tape_ + sizeof(label_t), v); }
1051
+ void level(level_t v) noexcept { return misaligned_store<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t), v); }
1052
+
1053
+ operator vector_view_t() const noexcept { return {vector(), dim()}; }
1054
+ vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
1055
+ member_ref_t ref() noexcept { return {label(), vector_view()}; }
1056
+ member_cref_t cref() const noexcept { return {label(), vector_view()}; }
828
1057
  };
829
1058
 
830
- class node_ref_t {
831
- mutex_t* mutex_{};
1059
+ /**
1060
+ * @brief A slice of the node's tape, containing a the list of neighbors
1061
+ * for a node in a single graph level. It's pre-allocated to fit
1062
+ * as many neighbors IDs, as may be needed at the target level,
1063
+ * and starts with a single integer `neighbors_count_t` counter.
1064
+ */
1065
+ class neighbors_ref_t {
1066
+ byte_t* tape_;
1067
+
1068
+ static constexpr std::size_t shift(std::size_t i = 0) { return sizeof(neighbors_count_t) + sizeof(id_t) * i; }
832
1069
 
833
1070
  public:
834
- node_head_t& head{};
835
- scalar_t* vector{};
836
-
837
- inline node_ref_t(mutex_t& m, node_head_t& h, scalar_t* s) noexcept : mutex_(&m), head(h), vector(s) {}
838
- inline lock_t lock() const noexcept { return mutex_ ? lock_t{*mutex_} : lock_t{}; }
839
- inline operator node_t() const noexcept { return node_t{mutex_ ? (byte_t*)mutex_ : (byte_t*)&head, vector}; }
1071
+ neighbors_ref_t(byte_t* tape) noexcept : tape_(tape) {}
1072
+ misaligned_ptr_gt<id_t> begin() noexcept { return tape_ + shift(); }
1073
+ misaligned_ptr_gt<id_t> end() noexcept { return begin() + size(); }
1074
+ misaligned_ptr_gt<id_t const> begin() const noexcept { return tape_ + shift(); }
1075
+ misaligned_ptr_gt<id_t const> end() const noexcept { return begin() + size(); }
1076
+ id_t operator[](std::size_t i) const noexcept { return misaligned_load<id_t>(tape_ + shift(i)); }
1077
+ std::size_t size() const noexcept { return misaligned_load<neighbors_count_t>(tape_); }
1078
+ void clear() noexcept { misaligned_store<neighbors_count_t>(tape_, 0); }
1079
+ void push_back(id_t id) noexcept {
1080
+ neighbors_count_t n = misaligned_load<neighbors_count_t>(tape_);
1081
+ misaligned_store<id_t>(tape_ + shift(n), id);
1082
+ misaligned_store<neighbors_count_t>(tape_, n + 1);
1083
+ }
840
1084
  };
841
1085
 
1086
+ /**
1087
+ * @brief A package of all kinds of temporary data-structures, that the threads
1088
+ * would reuse to process requests. Similar to having all of those as
1089
+ * separate `thread_local` global variables.
1090
+ */
842
1091
  struct usearch_align_m thread_context_t {
843
- top_candidates_t top_candidates;
844
- next_candidates_t next_candidates;
845
- visits_bitset_t visits;
846
- std::default_random_engine level_generator;
847
- metric_t metric;
1092
+ top_candidates_t top_candidates{};
1093
+ next_candidates_t next_candidates{};
1094
+ visits_bitset_t visits{};
1095
+ std::default_random_engine level_generator{};
1096
+ metric_t metric{};
1097
+ std::size_t iteration_cycles{};
1098
+ std::size_t measurements_count{};
1099
+
1100
+ inline distance_t measure(vector_view_t a, vector_view_t b) noexcept {
1101
+ measurements_count++;
1102
+ return metric(a.data(), b.data(), a.size(), b.size());
1103
+ }
848
1104
  };
849
1105
 
850
1106
  config_t config_{};
@@ -856,12 +1112,15 @@ class index_gt {
856
1112
  usearch_align_m mutable std::atomic<std::size_t> capacity_{};
857
1113
  usearch_align_m mutable std::atomic<std::size_t> size_{};
858
1114
 
859
- mutex_t global_mutex_{};
1115
+ /// @brief Controls access to `max_level_` and `entry_id_`.
1116
+ /// If any thread is updating those values, no other threads can `add()` or `search()`.
1117
+ std::mutex global_mutex_{};
860
1118
  level_t max_level_{};
861
1119
  id_t entry_id_{};
862
1120
 
863
1121
  using node_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
864
1122
  std::vector<node_t, node_allocator_t> nodes_{};
1123
+ mutable visits_bitset_t nodes_mutexes_{};
865
1124
 
866
1125
  using thread_context_allocator_t = typename allocator_traits_t::template rebind_alloc<thread_context_t>;
867
1126
  mutable std::vector<thread_context_t, thread_context_allocator_t> thread_contexts_{};
@@ -870,16 +1129,15 @@ class index_gt {
870
1129
  std::size_t connectivity() const noexcept { return config_.connectivity; }
871
1130
  std::size_t capacity() const noexcept { return capacity_; }
872
1131
  std::size_t size() const noexcept { return size_; }
873
- std::size_t max_threads_add() const noexcept { return config_.max_threads_add; }
1132
+ std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
1133
+ config_t const& config() const { return config_; }
874
1134
  bool is_immutable() const noexcept { return viewed_file_descriptor_ != 0; }
875
- bool synchronize() const noexcept { return config_.max_threads_add > 1; }
876
1135
 
877
1136
  index_gt(config_t config = {}, metric_t metric = {}, allocator_t allocator = {}) noexcept(false)
878
- : config_(config), metric_(metric), allocator_(allocator) {
1137
+ : config_(normalize(config)), metric_(metric), allocator_(allocator) {
879
1138
 
880
1139
  // Externally defined hyper-parameters:
881
- config_.expansion_add = (std::max)(config_.expansion_add, config_.connectivity);
882
- pre_ = precompute(config);
1140
+ pre_ = precompute_(config);
883
1141
 
884
1142
  // Configure initial empty state:
885
1143
  size_ = 0u;
@@ -888,12 +1146,15 @@ class index_gt {
888
1146
  viewed_file_descriptor_ = 0;
889
1147
 
890
1148
  // Dynamic memory:
891
- thread_contexts_.resize((std::max)(config.max_threads_search, config.max_threads_add));
1149
+ thread_contexts_.resize(config.max_threads());
892
1150
  for (thread_context_t& context : thread_contexts_)
893
1151
  context.metric = metric;
894
1152
  reserve(config.max_elements);
895
1153
  }
896
1154
 
1155
+ /**
1156
+ * @brief Clones the structure with the same hyper-parameters, but without contents.
1157
+ */
897
1158
  index_gt fork() noexcept(false) { return {config_, metric_, allocator_}; }
898
1159
 
899
1160
  ~index_gt() noexcept { clear(); }
@@ -907,15 +1168,22 @@ class index_gt {
907
1168
 
908
1169
  #pragma region Adjusting Configuration
909
1170
 
1171
+ /**
1172
+ * @brief Erases all the vectors from the index.
1173
+ * Will change `size()` to zero, but will keep the same `capacity()`.
1174
+ */
910
1175
  void clear() noexcept {
911
1176
  std::size_t n = size_;
912
1177
  for (std::size_t i = 0; i != n; ++i)
913
- node_free(i);
1178
+ node_free_(i);
914
1179
  size_ = 0;
915
1180
  max_level_ = -1;
916
1181
  entry_id_ = 0u;
917
1182
  }
918
1183
 
1184
+ /**
1185
+ * @brief Swaps the underlying memory buffers and thread contexts.
1186
+ */
919
1187
  void swap(index_gt& other) noexcept {
920
1188
  std::swap(config_, other.config_);
921
1189
  std::swap(metric_, other.metric_);
@@ -936,123 +1204,259 @@ class index_gt {
936
1204
  other.size_ = size;
937
1205
  }
938
1206
 
1207
+ /**
1208
+ * @brief Increases the `capacity()` of the index to allow adding more vectors.
1209
+ */
939
1210
  void reserve(std::size_t new_capacity) noexcept(false) {
940
1211
 
941
1212
  assert_m(new_capacity >= size_, "Can't drop existing values");
942
1213
  nodes_.resize(new_capacity);
1214
+ nodes_mutexes_ = visits_bitset_t(new_capacity);
943
1215
  for (thread_context_t& context : thread_contexts_)
944
1216
  context.visits = visits_bitset_t(new_capacity);
945
1217
 
946
1218
  capacity_ = new_capacity;
947
1219
  }
948
1220
 
949
- static config_t optimize(config_t const& config) noexcept {
950
- precomputed_constants_t pre = precompute(config);
951
- std::size_t bytes_per_node_base = head_bytes_k + pre.neighbors_base_bytes + pre.mutex_bytes;
1221
+ /**
1222
+ * @brief Optimizes configuration options to fit the maximum number
1223
+ * of neighbors in CPU-cache-aligned buffers.
1224
+ */
1225
+ static config_t optimize(config_t config) noexcept {
1226
+ precomputed_constants_t pre = precompute_(config);
1227
+ std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
952
1228
  std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
953
1229
  std::size_t added_connections = (rounded_size - rounded_size) / sizeof(id_t);
954
- config_t result = config;
955
- result.connectivity = config.connectivity + added_connections / base_level_multiple_k;
956
- return result;
1230
+ config.connectivity = config.connectivity + added_connections / base_level_multiple_();
1231
+ return config;
1232
+ }
1233
+
1234
+ /**
1235
+ * @brief Validates/normalizes the configuration options and throws an exception
1236
+ * if those are incorrect.
1237
+ */
1238
+ static config_t normalize(config_t config) noexcept {
1239
+ config.expansion_add = (std::max<std::size_t>)(config.expansion_add, config.connectivity);
1240
+ config.max_threads_add = (std::max<std::size_t>)(1u, config.max_threads_add);
1241
+ config.max_threads_search = (std::max<std::size_t>)(1u, config.max_threads_search);
1242
+ return config;
957
1243
  }
958
1244
 
959
1245
  #pragma endregion
960
1246
 
961
1247
  #pragma region Construction and Search
962
1248
 
963
- id_t add( //
964
- label_t new_label, span_gt<scalar_t const> new_span, //
965
- std::size_t thread_idx = 0, bool store_vector = true) {
1249
+ struct add_result_t {
1250
+ std::size_t new_size{};
1251
+ std::size_t cycles{};
1252
+ std::size_t measurements{};
1253
+ id_t id{};
1254
+ };
1255
+
1256
+ struct search_result_t {
1257
+ member_cref_t member;
1258
+ distance_t distance;
1259
+ };
1260
+
1261
+ class search_results_t {
1262
+ index_gt const& index_;
1263
+ top_candidates_t& top_;
1264
+
1265
+ friend class index_gt;
1266
+ inline search_results_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
1267
+
1268
+ public:
1269
+ std::size_t count{};
1270
+ std::size_t cycles{};
1271
+ std::size_t measurements{};
1272
+
1273
+ inline search_results_t(search_results_t&&) = default;
1274
+ inline search_results_t& operator=(search_results_t&&) = default;
1275
+
1276
+ inline operator std::size_t() const noexcept { return count; }
1277
+ inline std::size_t size() const noexcept { return count; }
1278
+ inline search_result_t operator[](std::size_t i) const noexcept {
1279
+ candidate_t const* top_ordered = top_.data();
1280
+ return {index_.node_with_id_(top_ordered[i].id).cref(), top_ordered[i].distance};
1281
+ }
1282
+ inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
1283
+ for (std::size_t i = 0; i != count; ++i) {
1284
+ search_result_t result = operator[](i);
1285
+ labels[i] = result.member.label;
1286
+ distances[i] = result.distance;
1287
+ }
1288
+ return count;
1289
+ }
1290
+ };
1291
+
1292
+ /**
1293
+ * @brief Inserts a new vector into the index. Thread-safe.
1294
+ *
1295
+ * @param[in] label External identifier/name/descriptor for the vector.
1296
+ * @param[in] vector Contiguous range of scalars forming a vector view.
1297
+ * @param[in] config Configuration options for this specific operation.
1298
+ */
1299
+ add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) noexcept(false) {
966
1300
 
967
1301
  assert_m(!is_immutable(), "Can't add to an immutable index");
968
- id_t new_id = static_cast<id_t>(size_.fetch_add(1));
969
- scalar_t const* new_vector = new_span.data();
970
- std::size_t new_dim = new_span.size();
971
-
972
- // Determining how much memory to allocate depends on the target level.
973
- lock_t new_level_lock(global_mutex_);
974
- level_t max_level = max_level_;
975
- thread_context_t& context = thread_contexts_[thread_idx];
976
- level_t new_target_level = choose_random_level(context.level_generator);
977
- if (new_target_level <= max_level)
1302
+ add_result_t result;
1303
+ result.new_size = size_.fetch_add(1);
1304
+ id_t new_id = result.id = static_cast<id_t>(result.new_size);
1305
+
1306
+ // Determining how much memory to allocate depends on the target level
1307
+ std::unique_lock<std::mutex> new_level_lock(global_mutex_);
1308
+ level_t max_level = max_level_; // Copy under lock
1309
+ id_t entry_id = entry_id_; // Copy under lock
1310
+ thread_context_t& context = thread_contexts_[config.thread];
1311
+ level_t target_level = choose_random_level_(context.level_generator);
1312
+ if (target_level <= max_level)
978
1313
  new_level_lock.unlock();
979
1314
 
980
1315
  // Allocate the neighbors
981
- node_ref_t new_node = node_malloc(new_label, new_vector, new_dim, new_target_level, store_vector);
982
- lock_t new_lock = new_node.lock();
983
- nodes_[new_id] = new_node;
1316
+ nodes_[result.new_size] = node_malloc_(label, vector, target_level, config.store_vector);
1317
+ node_lock_t new_lock = node_lock_(new_id);
1318
+ result.new_size++;
984
1319
 
985
1320
  // Do nothing for the first element
986
1321
  if (!new_id) {
987
- max_level_ = new_target_level;
988
- return new_id;
1322
+ entry_id_ = new_id;
1323
+ max_level_ = target_level;
1324
+ return result;
989
1325
  }
990
1326
 
1327
+ // Pull stats
1328
+ result.measurements = context.measurements_count;
1329
+ result.cycles = context.iteration_cycles;
1330
+
991
1331
  // Go down the level, tracking only the closest match
992
- id_t closest_id = search_for_one(entry_id_, new_vector, new_dim, max_level, new_target_level, context);
1332
+ id_t closest_id = search_for_one_(entry_id, vector, max_level, target_level, context);
993
1333
 
994
- // From `new_target_level` down perform proper extensive search.
995
- for (level_t level = (std::min)(new_target_level, max_level); level >= 0; level--) {
996
- search_to_insert(closest_id, new_vector, new_dim, level, context);
997
- closest_id = connect_new_element(new_id, level, context);
1334
+ // From `target_level` down perform proper extensive search
1335
+ for (level_t level = (std::min)(target_level, max_level); level >= 0; level--) {
1336
+ search_to_insert_(closest_id, vector, level, context);
1337
+ closest_id = connect_new_node_(new_id, level, context);
998
1338
  }
999
1339
 
1000
- // Releasing lock for the maximum level
1001
- if (new_target_level > max_level) {
1340
+ // Normalize stats
1341
+ result.measurements = context.measurements_count - result.measurements;
1342
+ result.cycles = context.iteration_cycles - result.cycles;
1343
+
1344
+ // Updating the entry point if needed
1345
+ if (target_level > max_level) {
1002
1346
  entry_id_ = new_id;
1003
- max_level_ = new_target_level;
1347
+ max_level_ = target_level;
1004
1348
  }
1005
- return new_id;
1349
+ return result;
1006
1350
  }
1007
1351
 
1008
- template <typename label_and_distance_callback_at>
1009
- void search( //
1010
- span_gt<scalar_t const> query_span, std::size_t wanted, //
1011
- label_and_distance_callback_at&& callback, std::size_t thread_idx = 0) const {
1352
+ /**
1353
+ * @brief Searches for the closest members to the given ::query. Thread-safe.
1354
+ *
1355
+ * @param[in] query Contiguous range of scalars forming a vector view.
1356
+ * @param[in] wanted The upper bound for the number of results to return.
1357
+ * @param[in] config Configuration options for this specific operation.
1358
+ * @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
1359
+ */
1360
+ search_results_t search( //
1361
+ vector_view_t query, std::size_t wanted, //
1362
+ search_config_t config = {}) const noexcept(false) {
1012
1363
 
1364
+ thread_context_t& context = thread_contexts_[config.thread];
1365
+ top_candidates_t& top = context.top_candidates;
1366
+ search_results_t result{*this, top};
1013
1367
  if (!size_)
1014
- return;
1368
+ return result;
1015
1369
 
1016
- scalar_t const* query_vec = query_span.data();
1017
- std::size_t query_dim = query_span.size();
1370
+ // Go down the level, tracking only the closest match
1371
+ result.measurements = context.measurements_count;
1372
+ result.cycles = context.iteration_cycles;
1373
+
1374
+ if (config.exact) {
1375
+ search_exact_(query, wanted, context);
1376
+ } else {
1377
+ id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
1378
+ std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1379
+ // For bottom layer we need a more optimized procedure
1380
+ search_to_find_in_base_(closest_id, query, expansion, context);
1381
+ }
1382
+
1383
+ top.sort_ascending();
1384
+ top.shrink(wanted);
1385
+
1386
+ // Normalize stats
1387
+ result.measurements = context.measurements_count - result.measurements;
1388
+ result.cycles = context.iteration_cycles - result.cycles;
1389
+ result.count = top.size();
1390
+ return result;
1391
+ }
1392
+
1393
+ search_results_t search_around( //
1394
+ id_t hint, vector_view_t query, std::size_t wanted, //
1395
+ search_config_t config = {}) const noexcept(false) {
1396
+
1397
+ thread_context_t& context = thread_contexts_[config.thread];
1398
+ top_candidates_t& top = context.top_candidates;
1399
+ search_results_t result{*this, top};
1400
+ if (!size_)
1401
+ return result;
1018
1402
 
1019
1403
  // Go down the level, tracking only the closest match
1020
- thread_context_t& context = thread_contexts_[thread_idx];
1021
- id_t closest_id = search_for_one(entry_id_, query_vec, query_dim, max_level_, 0, context);
1404
+ result.measurements = context.measurements_count;
1405
+ result.cycles = context.iteration_cycles;
1022
1406
 
1023
- // For bottom layer we need a more optimized procedure
1024
1407
  std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1025
- search_to_find_in_base(closest_id, query_vec, query_dim, expansion, context);
1026
- context.top_candidates.sort_ascending();
1027
- context.top_candidates.shrink(wanted);
1028
-
1029
- candidate_t const* top = context.top_candidates.data();
1030
- for (std::size_t i = 0; i < context.top_candidates.size(); ++i)
1031
- callback(node(top[i].second).head.label, top[i].first);
1032
- }
1033
-
1034
- std::size_t search( //
1035
- span_gt<scalar_t const> query_span, std::size_t wanted, //
1036
- label_t* matches, distance_t* distances, //
1037
- std::size_t thread_idx = 0) const {
1038
-
1039
- std::size_t found = 0;
1040
- auto callback = [&](label_t label, distance_t distance) noexcept {
1041
- if (matches)
1042
- matches[found] = label;
1043
- if (distances)
1044
- distances[found] = distance;
1045
- ++found;
1046
- };
1047
- search(query_span, wanted, callback, thread_idx);
1048
- return found;
1408
+ search_to_find_in_base_(hint, query, expansion, context);
1409
+ top.sort_ascending();
1410
+ top.shrink(wanted);
1411
+
1412
+ // Normalize stats
1413
+ result.measurements = context.measurements_count - result.measurements;
1414
+ result.cycles = context.iteration_cycles - result.cycles;
1415
+ result.count = top.size();
1416
+ return result;
1417
+ }
1418
+
1419
+ struct stats_t {
1420
+ std::size_t nodes;
1421
+ std::size_t edges;
1422
+ std::size_t max_edged;
1423
+ };
1424
+
1425
+ stats_t stats() const noexcept {
1426
+ stats_t result{};
1427
+ result.nodes = nodes_.size();
1428
+ for (std::size_t i = 0; i != result.nodes; ++i) {
1429
+ node_t node = node_with_id_(i);
1430
+ std::size_t max_edges = node.level() * config_.connectivity + base_level_multiple_() * config_.connectivity;
1431
+ std::size_t edges = 0;
1432
+ for (level_t level = 0; level != node.level(); ++level)
1433
+ edges += neighbors_(node, level).size();
1434
+
1435
+ result.edges += edges;
1436
+ result.max_edges += max_edges;
1437
+ }
1438
+ return result;
1439
+ }
1440
+
1441
+ stats_t stats(std::size_t level) const noexcept {
1442
+ stats_t result{};
1443
+ result.nodes = nodes_.size();
1444
+ for (std::size_t i = 0; i != result.nodes; ++i) {
1445
+ node_t node = node_with_id_(i);
1446
+ if (node.level() >= level)
1447
+ result.edges += neighbors_(node, level).size();
1448
+ }
1449
+ std::size_t max_edges_per_node = level ? config_.connectivity : base_level_multiple_() * config_.connectivity;
1450
+ result.max_edges = result.nodes * max_edges_per_node;
1451
+ return result;
1049
1452
  }
1050
1453
 
1051
1454
  #pragma endregion
1052
1455
 
1053
1456
  #pragma region Serialization
1054
1457
 
1055
- struct state_t {
1458
+ private:
1459
+ struct serialized_state_t {
1056
1460
  // Check compatibility
1057
1461
  std::uint64_t bytes_per_label{};
1058
1462
  std::uint64_t bytes_per_id{};
@@ -1064,12 +1468,15 @@ class index_gt {
1064
1468
  std::uint64_t max_level{};
1065
1469
  };
1066
1470
 
1471
+ public:
1067
1472
  /**
1068
- * Compatibility: Linux, MacOS, Windows.
1473
+ * @brief Saves serialized binary index representation to disk,
1474
+ * co-locating vectors and neighbors lists.
1475
+ * Available on Linux, MacOS, Windows.
1069
1476
  */
1070
1477
  void save(char const* file_path) const noexcept(false) {
1071
1478
 
1072
- state_t state;
1479
+ serialized_state_t state;
1073
1480
  // Check compatibility
1074
1481
  state.bytes_per_label = sizeof(label_t);
1075
1482
  state.bytes_per_id = sizeof(id_t);
@@ -1083,53 +1490,52 @@ class index_gt {
1083
1490
  if (!file)
1084
1491
  throw std::runtime_error(std::strerror(errno));
1085
1492
 
1086
- // Write the header
1087
- {
1088
- std::size_t written = std::fwrite(&state, sizeof(state), 1, file);
1493
+ auto write_chunk = [&](void* begin, std::size_t length) {
1494
+ std::size_t written = std::fwrite(begin, length, 1, file);
1089
1495
  if (!written) {
1090
1496
  std::fclose(file);
1091
1497
  throw std::runtime_error(std::strerror(errno));
1092
1498
  }
1093
- }
1499
+ };
1500
+
1501
+ // Write the header
1502
+ write_chunk(&state, sizeof(state));
1094
1503
 
1095
1504
  // Serialize nodes one by one
1096
1505
  for (std::size_t i = 0; i != state.size; ++i) {
1097
- node_ref_t node_ref = node(static_cast<id_t>(i));
1098
- std::size_t bytes_to_dump = node_dump_size(node_ref.head.dim, node_ref.head.level);
1099
- std::size_t bytes_in_vec = node_ref.head.dim * sizeof(scalar_t);
1100
- // Dump just neighbors, as vectors may be in a disjoint location
1101
- std::size_t written = std::fwrite(&node_ref.head, bytes_to_dump - bytes_in_vec, 1, file);
1102
- if (!written) {
1103
- std::fclose(file);
1104
- throw std::runtime_error(std::strerror(errno));
1105
- }
1106
- // Dump the vector
1107
- written = std::fwrite(node_ref.vector, bytes_in_vec, 1, file);
1108
- if (!written) {
1109
- std::fclose(file);
1110
- throw std::runtime_error(std::strerror(errno));
1111
- }
1506
+ node_t node = node_with_id_(i);
1507
+ std::size_t node_bytes = node_bytes_(node);
1508
+ std::size_t node_vector_bytes = node_vector_bytes_(node);
1509
+ // Dump neighbors and vectors, as vectors may be in a disjoint location
1510
+ write_chunk(node.tape(), node_bytes - node_vector_bytes);
1511
+ write_chunk(node.vector(), node_vector_bytes);
1112
1512
  }
1113
1513
 
1114
1514
  std::fclose(file);
1115
1515
  }
1116
1516
 
1117
1517
  /**
1118
- * Compatibility: Linux, MacOS, Windows.
1518
+ * @brief Loads the serialized binary index representation from disk,
1519
+ * copying both vectors and neighbors lists into RAM.
1520
+ * Available on Linux, MacOS, Windows.
1119
1521
  */
1120
1522
  void load(char const* file_path) noexcept(false) {
1121
- state_t state;
1523
+ serialized_state_t state;
1122
1524
  std::FILE* file = std::fopen(file_path, "r");
1123
1525
  if (!file)
1124
1526
  throw std::runtime_error(std::strerror(errno));
1125
1527
 
1126
- // Read the header
1127
- {
1128
- std::size_t read = std::fread(&state, sizeof(state), 1, file);
1528
+ auto read_chunk = [&](void* begin, std::size_t length) {
1529
+ std::size_t read = std::fread(begin, length, 1, file);
1129
1530
  if (!read) {
1130
1531
  std::fclose(file);
1131
1532
  throw std::runtime_error(std::strerror(errno));
1132
1533
  }
1534
+ };
1535
+
1536
+ // Read the header
1537
+ {
1538
+ read_chunk(&state, sizeof(state));
1133
1539
  if (state.bytes_per_label != sizeof(label_t)) {
1134
1540
  std::fclose(file);
1135
1541
  throw std::runtime_error("Incompatible label type!");
@@ -1141,7 +1547,7 @@ class index_gt {
1141
1547
 
1142
1548
  config_.connectivity = state.connectivity;
1143
1549
  config_.max_elements = state.size;
1144
- pre_ = precompute(config_);
1550
+ pre_ = precompute_(config_);
1145
1551
  reserve(state.size);
1146
1552
  size_ = state.size;
1147
1553
  max_level_ = static_cast<level_t>(state.max_level);
@@ -1150,22 +1556,17 @@ class index_gt {
1150
1556
 
1151
1557
  // Load nodes one by one
1152
1558
  for (std::size_t i = 0; i != state.size; ++i) {
1153
- node_head_t head;
1154
- std::size_t read = std::fread(&head, head_bytes_k, 1, file);
1155
- if (!read) {
1156
- std::fclose(file);
1157
- throw std::runtime_error(std::strerror(errno));
1158
- }
1159
-
1160
- std::size_t bytes_to_dump = node_dump_size(head.dim, head.level);
1161
- node_ref_t node_ref = node_malloc(head.label, nullptr, head.dim, head.level, true);
1162
- read = std::fread((byte_t*)&node_ref.head + head_bytes_k, bytes_to_dump - head_bytes_k, 1, file);
1163
- if (!read) {
1164
- std::fclose(file);
1165
- throw std::runtime_error(std::strerror(errno));
1166
- }
1167
-
1168
- nodes_[i] = node_ref;
1559
+ label_t label;
1560
+ dim_t dim;
1561
+ level_t level;
1562
+ read_chunk(&label, sizeof(label));
1563
+ read_chunk(&dim, sizeof(dim));
1564
+ read_chunk(&level, sizeof(level));
1565
+
1566
+ std::size_t node_bytes = node_bytes_(dim, level);
1567
+ node_t node = node_malloc_(label, {nullptr, dim}, level, true);
1568
+ read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
1569
+ nodes_[i] = node;
1169
1570
  }
1170
1571
 
1171
1572
  std::fclose(file);
@@ -1173,13 +1574,15 @@ class index_gt {
1173
1574
  }
1174
1575
 
1175
1576
  /**
1176
- * Compatibility: Linux, MacOS.
1577
+ * @brief Memory-maps the serialized binary index representation from disk,
1578
+ * @b without copying the vectors and neighbors lists into RAM.
1579
+ * Available on Linux, MacOS, but @b not on Windows.
1177
1580
  */
1178
1581
  void view(char const* file_path) noexcept(false) {
1179
- #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
1582
+ #if defined(USEARCH_IS_WINDOWS)
1180
1583
  throw std::logic_error("Memory-mapping is not yet available for Windows");
1181
1584
  #else
1182
- state_t state;
1585
+ serialized_state_t state;
1183
1586
  int open_flags = O_RDONLY;
1184
1587
  #if __linux__
1185
1588
  open_flags |= O_NOATIME;
@@ -1216,7 +1619,7 @@ class index_gt {
1216
1619
  config_.connectivity = state.connectivity;
1217
1620
  config_.max_elements = state.size;
1218
1621
  config_.max_threads_add = 0;
1219
- pre_ = precompute(config_);
1622
+ pre_ = precompute_(config_);
1220
1623
  reserve(state.size);
1221
1624
  size_ = state.size;
1222
1625
  max_level_ = static_cast<level_t>(state.max_level);
@@ -1226,13 +1629,15 @@ class index_gt {
1226
1629
  // Locate every node packed into file
1227
1630
  std::size_t progress = sizeof(state);
1228
1631
  for (std::size_t i = 0; i != state.size; ++i) {
1229
- node_head_t const& head = *(node_head_t const*)(file + progress);
1230
- std::size_t bytes_to_dump = node_dump_size(head.dim, head.level);
1231
- std::size_t bytes_in_vec = head.dim * sizeof(scalar_t);
1232
- nodes_[i].tape_ = (byte_t*)(file + progress);
1233
- nodes_[i].vector_ = (scalar_t*)(file + progress + bytes_to_dump - bytes_in_vec);
1234
- progress += bytes_to_dump;
1235
- max_level_ = (std::max)(max_level_, head.level);
1632
+ byte_t* tape = (byte_t*)(file + progress);
1633
+ dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
1634
+ level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
1635
+
1636
+ std::size_t node_bytes = node_bytes_(dim, level);
1637
+ std::size_t node_vector_bytes = dim * sizeof(scalar_t);
1638
+ nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
1639
+ progress += node_bytes;
1640
+ max_level_ = (std::max)(max_level_, level);
1236
1641
  }
1237
1642
 
1238
1643
  viewed_file_descriptor_ = descriptor;
@@ -1242,124 +1647,114 @@ class index_gt {
1242
1647
  #pragma endregion
1243
1648
 
1244
1649
  private:
1245
- inline static precomputed_constants_t precompute(config_t const& config) noexcept {
1650
+ inline static precomputed_constants_t precompute_(config_t const& config) noexcept {
1246
1651
  precomputed_constants_t pre;
1247
- pre.connectivity_max_base = config.connectivity * base_level_multiple_k;
1652
+ pre.connectivity_max_base = config.connectivity * base_level_multiple_();
1248
1653
  pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
1249
1654
  pre.neighbors_bytes = config.connectivity * sizeof(id_t) + sizeof(neighbors_count_t);
1250
1655
  pre.neighbors_base_bytes = pre.connectivity_max_base * sizeof(id_t) + sizeof(neighbors_count_t);
1251
- pre.mutex_bytes = sizeof(mutex_t) * (config.max_threads_add > 1);
1252
1656
  return pre;
1253
1657
  }
1254
1658
 
1255
- inline std::size_t node_dump_size(dim_t dim, level_t level) const noexcept {
1256
- return head_bytes_k + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
1659
+ inline std::size_t node_bytes_(node_t node) const noexcept { return node_bytes_(node.dim(), node.level()); }
1660
+ inline std::size_t node_bytes_(dim_t dim, level_t level) const noexcept {
1661
+ return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
1257
1662
  }
1258
1663
 
1259
- void node_free(std::size_t id) noexcept {
1664
+ inline bool node_stored_(node_t node) const noexcept {
1665
+ std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
1666
+ return (node.tape() + node_head_bytes_() + levels_bytes) == (byte_t*)node.vector();
1667
+ }
1668
+
1669
+ inline std::size_t node_vector_bytes_(dim_t dim) const noexcept { return dim * sizeof(scalar_t); }
1670
+ inline std::size_t node_vector_bytes_(node_t node) const noexcept { return node_vector_bytes_(node.dim()); }
1671
+
1672
+ void node_free_(std::size_t id) noexcept {
1260
1673
 
1261
1674
  if (viewed_file_descriptor_ != 0)
1262
1675
  return;
1263
1676
 
1264
- // This function is rarely called and can be as expensive as needed for higher space-efficiency.
1265
1677
  node_t& node = nodes_[id];
1266
- if (!node.tape_)
1267
- return;
1268
-
1269
- node_head_t const& head = *(node_head_t const*)(node.tape_ + pre_.mutex_bytes);
1270
- std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * head.level;
1271
- bool store_vector = (byte_t*)(node.tape_ + pre_.mutex_bytes + head_bytes_k + levels_bytes) == //
1272
- (byte_t*)(node.vector_);
1273
- std::size_t node_bytes = //
1274
- pre_.mutex_bytes + // Optional concurrency-control
1275
- head_bytes_k + levels_bytes + // Obligatory neighborhood index
1276
- head.dim * store_vector // Optional vector copy
1277
- ;
1278
-
1279
- allocator_t{}.deallocate(node.tape_, node_bytes);
1678
+ std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_stored_(node);
1679
+ allocator_t{}.deallocate(node.tape(), node_bytes);
1280
1680
  node = node_t{};
1281
1681
  }
1282
1682
 
1283
- node_ref_t node_malloc( //
1284
- label_t label, scalar_t const* vector, std::size_t dim, //
1285
- level_t level, bool store_vector = true) noexcept(false) {
1683
+ node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept(false) {
1286
1684
 
1287
- // This function is rarely called and can be as expensive as needed for higher space-efficiency.
1288
- std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * level;
1289
- std::size_t node_bytes = //
1290
- pre_.mutex_bytes + // Optional concurrency-control
1291
- head_bytes_k + levels_bytes + // Obligatory neighborhood index
1292
- sizeof(scalar_t) * dim * store_vector // Optional vector copy
1293
- ;
1685
+ std::size_t dim = vector.size();
1686
+ std::size_t stored_vector_bytes = node_vector_bytes_(dim) * store_vector;
1687
+ std::size_t node_bytes = node_bytes_(dim, level) - node_vector_bytes_(dim) * !store_vector;
1294
1688
 
1295
1689
  byte_t* data = (byte_t*)allocator_t{}.allocate(node_bytes);
1296
1690
  assert_m(data, "Not enough memory for links");
1297
-
1298
- mutex_t* mutex = synchronize() ? (mutex_t*)data : nullptr;
1299
- scalar_t* scalars = store_vector //
1300
- ? (scalar_t*)(data + pre_.mutex_bytes + head_bytes_k + levels_bytes)
1301
- : (scalar_t*)(vector);
1302
-
1303
1691
  std::memset(data, 0, node_bytes);
1304
- std::memcpy(scalars, vector, sizeof(scalar_t) * dim * (store_vector && vector));
1305
-
1306
- node_head_t& head = *(node_head_t*)(data + pre_.mutex_bytes);
1307
- head.label = label;
1308
- head.dim = static_cast<dim_t>(dim);
1309
- head.level = level;
1692
+ if (vector.data())
1693
+ std::memcpy(data + node_bytes - stored_vector_bytes, vector.data(), stored_vector_bytes);
1310
1694
 
1311
- return {*mutex, head, scalars};
1695
+ scalar_t* scalars = store_vector //
1696
+ ? (scalar_t*)(data + node_bytes - stored_vector_bytes)
1697
+ : (scalar_t*)(vector.data());
1698
+
1699
+ node_t node{data, scalars};
1700
+ node.label(label);
1701
+ node.dim(static_cast<dim_t>(dim));
1702
+ node.level(level);
1703
+ return node;
1312
1704
  }
1313
1705
 
1314
- inline node_ref_t node(id_t id) const noexcept { return node(nodes_[id]); }
1706
+ inline node_t node_with_id_(std::size_t idx) const noexcept { return nodes_[idx]; }
1707
+ inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
1315
1708
 
1316
- inline node_ref_t node(node_t node) const noexcept {
1317
- byte_t* data = node.tape_;
1318
- mutex_t* mutex = synchronize() ? (mutex_t*)data : nullptr;
1319
- node_head_t& head = *(node_head_t*)(data + pre_.mutex_bytes);
1320
- scalar_t* scalars = node.vector_;
1709
+ inline neighbors_ref_t neighbors_non_base_(node_t node, level_t level) const noexcept {
1710
+ return {node.neighbors_tape() + pre_.neighbors_base_bytes + (level - 1) * pre_.neighbors_bytes};
1711
+ }
1321
1712
 
1322
- return {*mutex, head, scalars};
1713
+ inline neighbors_ref_t neighbors_(node_t node, level_t level) const noexcept {
1714
+ return level ? neighbors_non_base_(node, level) : neighbors_base_(node);
1323
1715
  }
1324
1716
 
1325
- inline neighbors_ref_t neighbors_base(node_ref_t node) const noexcept { return {node.head.neighbors}; }
1717
+ struct node_lock_t {
1718
+ visits_bitset_t& bitset;
1719
+ std::size_t idx;
1326
1720
 
1327
- inline neighbors_ref_t neighbors_non_base(node_ref_t node, level_t level) const noexcept {
1328
- return {node.head.neighbors + pre_.neighbors_base_bytes + (level - 1) * pre_.neighbors_bytes};
1329
- }
1721
+ inline ~node_lock_t() noexcept { bitset.atomic_reset(idx); }
1722
+ };
1330
1723
 
1331
- inline neighbors_ref_t neighbors(node_ref_t node, level_t level) const noexcept {
1332
- return level ? neighbors_non_base(node, level) : neighbors_base(node);
1724
+ inline node_lock_t node_lock_(std::size_t idx) const noexcept {
1725
+ while (nodes_mutexes_.atomic_set(idx))
1726
+ ;
1727
+ return {nodes_mutexes_, idx};
1333
1728
  }
1334
1729
 
1335
- id_t connect_new_element(id_t new_id, level_t level, thread_context_t& context) noexcept(false) {
1730
+ id_t connect_new_node_(id_t new_id, level_t level, thread_context_t& context) noexcept(false) {
1336
1731
 
1337
- node_ref_t new_node = node(new_id);
1732
+ node_t new_node = node_with_id_(new_id);
1338
1733
  top_candidates_t& top = context.top_candidates;
1339
1734
  std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
1340
1735
 
1341
1736
  // Outgoing links from `new_id`:
1342
- neighbors_ref_t new_neighbors = neighbors(new_node, level);
1737
+ neighbors_ref_t new_neighbors = neighbors_(new_node, level);
1343
1738
  {
1344
1739
  assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
1345
- candidates_view_t top_view = refine(top, config_.connectivity, context);
1740
+ candidates_view_t top_view = refine_(top, config_.connectivity, context);
1346
1741
 
1347
1742
  for (std::size_t idx = 0; idx != top_view.size(); idx++) {
1348
1743
  assert_m(!new_neighbors[idx], "Possible memory corruption");
1349
- assert_m(level <= node(top_view[idx].second).head.level, "Linking to missing level");
1350
- new_neighbors.push_back(top_view[idx].second);
1744
+ assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
1745
+ new_neighbors.push_back(top_view[idx].id);
1351
1746
  }
1352
1747
  }
1353
1748
 
1354
1749
  // Reverse links from the neighbors:
1355
1750
  for (id_t close_id : new_neighbors) {
1356
- node_ref_t close_node = node(close_id);
1357
- lock_t close_lock = close_node.lock();
1751
+ node_t close_node = node_with_id_(close_id);
1752
+ node_lock_t close_lock = node_lock_(close_id);
1358
1753
 
1359
- neighbors_ref_t close_header = neighbors(close_node, level);
1754
+ neighbors_ref_t close_header = neighbors_(close_node, level);
1360
1755
  assert_m(close_header.size() <= connectivity_max, "Possible corruption");
1361
1756
  assert_m(close_id != new_id, "Self-loops are impossible");
1362
- assert_m(level <= close_node.head.level, "Linking to missing level");
1757
+ assert_m(level <= close_node.level(), "Linking to missing level");
1363
1758
 
1364
1759
  // If `new_id` is already present in the neighboring connections of `close_id`
1365
1760
  // then no need to modify any connections or run the heuristics.
@@ -1371,66 +1766,56 @@ class index_gt {
1371
1766
  // To fit a new connection we need to drop an existing one.
1372
1767
  top.clear();
1373
1768
  top.reserve(close_header.size() + 1);
1374
- top.emplace_reserved( //
1375
- context.metric( //
1376
- new_node.vector, close_node.vector, new_node.head.dim, close_node.head.dim),
1377
- new_id);
1378
- for (id_t successor_id : close_header) {
1379
- node_ref_t successor_node = node(successor_id);
1380
- top.emplace_reserved( //
1381
- context.metric( //
1382
- successor_node.vector, close_node.vector, successor_node.head.dim, close_node.head.dim),
1383
- successor_id);
1384
- }
1769
+ top.emplace_reserved(context.measure(new_node, close_node), new_id);
1770
+ for (id_t successor_id : close_header)
1771
+ top.emplace_reserved(context.measure(node_with_id_(successor_id), close_node), successor_id);
1385
1772
 
1386
1773
  // Export the results:
1387
1774
  close_header.clear();
1388
- candidates_view_t top_view = refine(top, connectivity_max, context);
1775
+ candidates_view_t top_view = refine_(top, connectivity_max, context);
1389
1776
  for (std::size_t idx = 0; idx != top_view.size(); idx++)
1390
- close_header.push_back(top_view[idx].second);
1777
+ close_header.push_back(top_view[idx].id);
1391
1778
  }
1392
1779
 
1393
1780
  return new_neighbors[0];
1394
1781
  }
1395
1782
 
1396
- level_t choose_random_level(std::default_random_engine& level_generator) const noexcept {
1783
+ level_t choose_random_level_(std::default_random_engine& level_generator) const noexcept {
1397
1784
  std::uniform_real_distribution<double> distribution(0.0, 1.0);
1398
1785
  double r = -std::log(distribution(level_generator)) * pre_.inverse_log_connectivity;
1399
1786
  return (level_t)r;
1400
1787
  }
1401
1788
 
1402
- id_t search_for_one( //
1403
- id_t entry_id, scalar_t const* query_vec, std::size_t query_dim, //
1404
- level_t begin_level, level_t end_level, thread_context_t& context) const noexcept {
1789
+ id_t search_for_one_( //
1790
+ id_t closest_id, vector_view_t query, //
1791
+ level_t begin_level, level_t end_level, //
1792
+ thread_context_t& context) const noexcept {
1405
1793
 
1406
- id_t closest_id = entry_id;
1407
- distance_t closest_dist =
1408
- context.metric(query_vec, node(closest_id).vector, query_dim, node(closest_id).head.dim);
1794
+ distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
1409
1795
  for (level_t level = begin_level; level > end_level; level--) {
1410
1796
  bool changed;
1411
1797
  do {
1412
1798
  changed = false;
1413
- node_ref_t closest_node = node(closest_id);
1414
- lock_t closest_lock = closest_node.lock();
1415
- neighbors_ref_t closest_header = neighbors_non_base(closest_node, level);
1416
- for (id_t candidate_id : closest_header) {
1417
- node_ref_t candidate_node = node(candidate_id);
1418
- distance_t candidate_dist =
1419
- context.metric(query_vec, candidate_node.vector, query_dim, candidate_node.head.dim);
1799
+ node_t closest_node = node_with_id_(closest_id);
1800
+ node_lock_t closest_lock = node_lock_(closest_id);
1801
+ neighbors_ref_t closest_neighbors = neighbors_non_base_(closest_node, level);
1802
+ for (id_t candidate_id : closest_neighbors) {
1803
+ distance_t candidate_dist = context.measure(query, node_with_id_(candidate_id));
1420
1804
  if (candidate_dist < closest_dist) {
1421
1805
  closest_dist = candidate_dist;
1422
1806
  closest_id = candidate_id;
1423
1807
  changed = true;
1424
1808
  }
1425
1809
  }
1810
+ context.iteration_cycles++;
1426
1811
  } while (changed);
1427
1812
  }
1428
1813
  return closest_id;
1429
1814
  }
1430
1815
 
1431
- void search_to_insert( //
1432
- id_t start_id, scalar_t const* query_vec, std::size_t query_dim, //
1433
- level_t level, thread_context_t& context) noexcept(false) {
1816
+ void search_to_insert_( //
1817
+ id_t start_id, vector_view_t query, level_t level, //
1818
+ thread_context_t& context) noexcept(false) {
1434
1819
 
1435
1820
  visits_bitset_t& visits = context.visits;
1436
1821
  next_candidates_t& candidates = context.next_candidates; // pop min, push
@@ -1442,7 +1827,7 @@ class index_gt {
1442
1827
  top.clear();
1443
1828
  top.reserve(top_limit + 1);
1444
1829
 
1445
- distance_t radius = context.metric(query_vec, node(start_id).vector, query_dim, node(start_id).head.dim);
1830
+ distance_t radius = context.measure(query, node_with_id_(start_id));
1446
1831
  candidates.emplace(-radius, start_id);
1447
1832
  top.emplace_reserved(radius, start_id);
1448
1833
  visits.set(start_id);
@@ -1450,38 +1835,39 @@ class index_gt {
1450
1835
  while (!candidates.empty()) {
1451
1836
 
1452
1837
  candidate_t candidacy = candidates.top();
1453
- if ((-candidacy.first) > radius && top.size() == top_limit)
1838
+ if ((-candidacy.distance) > radius && top.size() == top_limit)
1454
1839
  break;
1455
1840
 
1456
1841
  candidates.pop();
1457
- id_t candidate_id = candidacy.second;
1458
- node_ref_t candidate_node = node(candidate_id);
1459
- lock_t candidate_lock = candidate_node.lock();
1460
- neighbors_ref_t candidate_header = neighbors(candidate_node, level);
1842
+ context.iteration_cycles++;
1461
1843
 
1462
- prefetch_neighbors(candidate_header, visits);
1463
- for (id_t successor_id : candidate_header) {
1844
+ id_t candidate_id = candidacy.id;
1845
+ node_t candidate_ref = node_with_id_(candidate_id);
1846
+ node_lock_t candidate_lock = node_lock_(candidate_id);
1847
+ neighbors_ref_t candidate_neighbors = neighbors_(candidate_ref, level);
1848
+
1849
+ prefetch_neighbors_(candidate_neighbors, visits);
1850
+ for (id_t successor_id : candidate_neighbors) {
1464
1851
  if (visits.test(successor_id))
1465
1852
  continue;
1466
1853
 
1467
1854
  visits.set(successor_id);
1468
- node_ref_t successor_node = node(successor_id);
1469
- distance_t successor_dist =
1470
- context.metric(query_vec, successor_node.vector, query_dim, successor_node.head.dim);
1855
+ distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
1856
+
1471
1857
  if (top.size() < top_limit || successor_dist < radius) {
1472
1858
  candidates.emplace(-successor_dist, successor_id);
1473
1859
  top.emplace_reserved(successor_dist, successor_id);
1474
1860
  if (top.size() > top_limit)
1475
1861
  top.pop();
1476
- radius = top.top().first;
1862
+ radius = top.top().distance;
1477
1863
  }
1478
1864
  }
1479
1865
  }
1480
1866
  }
1481
1867
 
1482
- void search_to_find_in_base( //
1483
- id_t start_id, scalar_t const* query_vec, std::size_t query_dim, //
1484
- std::size_t expansion, thread_context_t& context) const noexcept(false) {
1868
+ void search_to_find_in_base_( //
1869
+ id_t start_id, vector_view_t query, std::size_t expansion, //
1870
+ thread_context_t& context) const noexcept(false) {
1485
1871
 
1486
1872
  visits_bitset_t& visits = context.visits;
1487
1873
  next_candidates_t& candidates = context.next_candidates; // pop min, push
@@ -1493,7 +1879,7 @@ class index_gt {
1493
1879
  top.clear();
1494
1880
  top.reserve(top_limit + 1);
1495
1881
 
1496
- distance_t radius = context.metric(query_vec, node(start_id).vector, query_dim, node(start_id).head.dim);
1882
+ distance_t radius = context.measure(query, node_with_id_(start_id));
1497
1883
  candidates.emplace(-radius, start_id);
1498
1884
  top.emplace_reserved(radius, start_id);
1499
1885
  visits.set(start_id);
@@ -1501,43 +1887,53 @@ class index_gt {
1501
1887
  while (!candidates.empty()) {
1502
1888
 
1503
1889
  candidate_t candidate = candidates.top();
1504
- if ((-candidate.first) > radius)
1890
+ if ((-candidate.distance) > radius)
1505
1891
  break;
1506
1892
 
1507
1893
  candidates.pop();
1894
+ context.iteration_cycles++;
1508
1895
 
1509
- id_t candidate_id = candidate.second;
1510
- neighbors_ref_t candidate_header = neighbors_base(node(candidate_id));
1896
+ id_t candidate_id = candidate.id;
1897
+ neighbors_ref_t candidate_neighbors = neighbors_base_(node_with_id_(candidate_id));
1511
1898
 
1512
- prefetch_neighbors(candidate_header, visits);
1513
- for (id_t successor_id : candidate_header) {
1899
+ prefetch_neighbors_(candidate_neighbors, visits);
1900
+ for (id_t successor_id : candidate_neighbors) {
1514
1901
  if (visits.test(successor_id))
1515
1902
  continue;
1516
1903
 
1517
1904
  visits.set(successor_id);
1518
- node_ref_t successor_node = node(successor_id);
1519
- distance_t successor_dist =
1520
- context.metric(query_vec, successor_node.vector, query_dim, successor_node.head.dim);
1905
+ distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
1521
1906
 
1522
1907
  if (top.size() < top_limit || successor_dist < radius) {
1523
1908
  candidates.emplace(-successor_dist, successor_id);
1524
1909
  top.emplace_reserved(successor_dist, successor_id);
1525
1910
  if (top.size() > top_limit)
1526
1911
  top.pop();
1527
- radius = top.top().first;
1912
+ radius = top.top().distance;
1528
1913
  }
1529
1914
  }
1530
1915
  }
1531
1916
  }
1532
1917
 
1533
- void prefetch_neighbors(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
1918
+ void search_exact_(vector_view_t query, std::size_t count, thread_context_t& context) const noexcept(false) {
1919
+ top_candidates_t& top = context.top_candidates;
1920
+ top.clear();
1921
+ top.reserve(count);
1922
+ for (std::size_t i = 0; i != nodes_.size(); ++i)
1923
+ top.emplace_reserved(context.measure(query, node_with_id_(i)), static_cast<id_t>(i));
1924
+ }
1925
+
1926
+ void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
1534
1927
 
1535
1928
  /**
1536
1929
  * @brief This algorithm from the original paper implements a heuristic,
1537
1930
  * that massively reduces the number of connections a point has,
1538
1931
  * to keep only the neighbors, that are from each other.
1539
1932
  */
1540
- candidates_view_t refine(top_candidates_t& top, std::size_t needed, thread_context_t& context) const noexcept {
1933
+
1934
+ candidates_view_t refine_( //
1935
+ top_candidates_t& top, std::size_t needed, //
1936
+ thread_context_t& context) const noexcept {
1541
1937
 
1542
1938
  top.sort_ascending();
1543
1939
  candidate_t* top_data = top.data();
@@ -1549,15 +1945,13 @@ class index_gt {
1549
1945
  std::size_t consumed_count = 1; /// Always equal or greater than `submitted_count`.
1550
1946
  while (submitted_count < needed && consumed_count < top_count) {
1551
1947
  candidate_t candidate = top_data[consumed_count];
1552
- node_ref_t candidate_node = node(candidate.second);
1553
- distance_t candidate_dist = candidate.first;
1948
+ node_t candidate_ref = node_with_id_(candidate.id);
1949
+ distance_t candidate_dist = candidate.distance;
1554
1950
  bool good = true;
1555
1951
  for (std::size_t idx = 0; idx < submitted_count; idx++) {
1556
1952
  candidate_t submitted = top_data[idx];
1557
- node_ref_t submitted_node = node(submitted.second);
1558
- distance_t inter_result_dist = context.metric( //
1559
- submitted_node.vector, candidate_node.vector, //
1560
- submitted_node.head.dim, candidate_node.head.dim);
1953
+ node_t submitted_node = node_with_id_(submitted.id);
1954
+ distance_t inter_result_dist = context.measure(submitted_node, candidate_ref);
1561
1955
  if (inter_result_dist < candidate_dist) {
1562
1956
  good = false;
1563
1957
  break;