usearch 0.11.1 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,8 +6,8 @@
6
6
  *
7
7
  * @copyright Copyright (c) 2023
8
8
  */
9
- #ifndef UNUM_USEARCH_H
10
- #define UNUM_USEARCH_H
9
+ #ifndef UNUM_USEARCH_HPP
10
+ #define UNUM_USEARCH_HPP
11
11
 
12
12
  #if !defined(USEARCH_VERSION_MAJOR)
13
13
  #define USEARCH_VERSION_MAJOR 0
@@ -100,12 +100,14 @@
100
100
 
101
101
  // Debugging
102
102
  #if defined(NDEBUG)
103
- #define assert_m(must_be_true, message)
103
+ #define usearch_assert_m(must_be_true, message)
104
+ #define usearch_noexcept_m noexcept
104
105
  #else
105
- #define assert_m(must_be_true, message) \
106
+ #define usearch_assert_m(must_be_true, message) \
106
107
  if (!(must_be_true)) { \
107
108
  throw std::runtime_error(message); \
108
109
  }
110
+ #define usearch_noexcept_m
109
111
  #endif
110
112
 
111
113
  namespace unum {
@@ -307,14 +309,14 @@ template <typename scalar_at, typename result_at = std::size_t> struct hamming_g
307
309
  * two arrays of integers. An example would be a textual document,
308
310
  * tokenized and hashed into a fixed-capacity bitset.
309
311
  */
310
- template <typename scalar_at, typename result_at = std::size_t> struct bit_hamming_gt {
312
+ template <typename scalar_at, typename result_at = std::size_t> struct bitwise_hamming_gt {
311
313
  using scalar_t = scalar_at;
312
314
  using result_t = result_at;
313
315
  using result_type = result_t;
314
316
  static_assert(std::is_unsigned<scalar_t>::value, "Hamming distance requires unsigned integral words");
315
317
 
316
- inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words,
317
- std::size_t = 0) const noexcept {
318
+ inline result_t operator()( //
319
+ scalar_t const* a, scalar_t const* b, std::size_t words, std::size_t = 0) const noexcept {
318
320
  constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
319
321
  result_t matches{};
320
322
  #if defined(USEARCH_USE_OPENMP)
@@ -330,12 +332,72 @@ template <typename scalar_at, typename result_at = std::size_t> struct bit_hammi
330
332
  }
331
333
  };
332
334
 
335
+ /**
336
+ * @brief Tanimoto distance is the intersection over bitwise union.
337
+ * Often used in chemistry and biology to compare molecular fingerprints.
338
+ */
339
+ template <typename scalar_at, typename result_at = float> struct bitwise_tanimoto_gt {
340
+ using scalar_t = scalar_at;
341
+ using result_t = result_at;
342
+ using result_type = result_t;
343
+ static_assert(std::is_unsigned<scalar_t>::value, "Tanimoto distance requires unsigned integral words");
344
+ static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
345
+
346
+ inline result_t operator()( //
347
+ scalar_t const* a, scalar_t const* b, std::size_t words, std::size_t = 0) const noexcept {
348
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
349
+ result_t and_count{};
350
+ result_t or_count{};
351
+ #if defined(USEARCH_USE_OPENMP)
352
+ #pragma omp simd reduction(+ : matches)
353
+ #elif defined(USEARCH_IS_CLANG)
354
+ #pragma clang loop vectorize(enable)
355
+ #elif defined(USEARCH_IS_GCC)
356
+ #pragma GCC ivdep
357
+ #endif
358
+ for (std::size_t i = 0; i != words; ++i)
359
+ and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
360
+ or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
361
+ return 1 - result_t(and_count) / or_count;
362
+ }
363
+ };
364
+
365
+ /**
366
+ * @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
367
+ * Often used in chemistry and biology to compare molecular fingerprints.
368
+ */
369
+ template <typename scalar_at, typename result_at = float> struct bitwise_sorensen_gt {
370
+ using scalar_t = scalar_at;
371
+ using result_t = result_at;
372
+ using result_type = result_t;
373
+ static_assert(std::is_unsigned<scalar_t>::value, "Sorensen-Dice distance requires unsigned integral words");
374
+ static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
375
+
376
+ inline result_t operator()( //
377
+ scalar_t const* a, scalar_t const* b, std::size_t words, std::size_t = 0) const noexcept {
378
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
379
+ result_t and_count{};
380
+ result_t any_count{};
381
+ #if defined(USEARCH_USE_OPENMP)
382
+ #pragma omp simd reduction(+ : matches)
383
+ #elif defined(USEARCH_IS_CLANG)
384
+ #pragma clang loop vectorize(enable)
385
+ #elif defined(USEARCH_IS_GCC)
386
+ #pragma GCC ivdep
387
+ #endif
388
+ for (std::size_t i = 0; i != words; ++i)
389
+ and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
390
+ any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
391
+ return 1 - 2 * result_t(and_count) / any_count;
392
+ }
393
+ };
394
+
333
395
  /**
334
396
  * @brief Counts the number of matching elements in two unique sorted sets.
335
397
  * Can be used to compute the similarity between two textual documents
336
398
  * using the IDs of tokens present in them.
337
399
  */
338
- template <typename scalar_at, typename result_at = scalar_at> struct jaccard_gt {
400
+ template <typename scalar_at, typename result_at = float> struct jaccard_gt {
339
401
  using scalar_t = scalar_at;
340
402
  using result_t = result_at;
341
403
  using result_type = result_t;
@@ -351,7 +413,7 @@ template <typename scalar_at, typename result_at = scalar_at> struct jaccard_gt
351
413
  i += a[i] < b[j];
352
414
  j += a[i] >= b[j];
353
415
  }
354
- return 1.f - intersection * 1.f / (a_length + b_length - intersection);
416
+ return 1 - intersection / (a_length + b_length - intersection);
355
417
  }
356
418
  };
357
419
 
@@ -360,7 +422,7 @@ template <typename scalar_at, typename result_at = scalar_at> struct jaccard_gt
360
422
  * Can be used to compute the similarity between two textual documents
361
423
  * using the IDs of tokens present in them.
362
424
  */
363
- template <typename scalar_at, typename result_at = scalar_at> struct pearson_correlation_gt {
425
+ template <typename scalar_at, typename result_at = float> struct pearson_correlation_gt {
364
426
  using scalar_t = scalar_at;
365
427
  using result_t = result_at;
366
428
  using result_type = result_t;
@@ -417,6 +479,60 @@ template <typename scalar_at, typename result_at = scalar_at> struct haversine_g
417
479
  }
418
480
  };
419
481
 
482
+ class error_t {
483
+ char const* message_{};
484
+
485
+ public:
486
+ error_t(char const* message = nullptr) noexcept : message_(message) {}
487
+ error_t& operator=(char const* message) noexcept {
488
+ message_ = message;
489
+ return *this;
490
+ }
491
+ error_t(error_t&& other) noexcept : message_(exchange(other.message_, nullptr)) {}
492
+ error_t& operator=(error_t&& other) noexcept {
493
+ std::swap(message_, other.message_);
494
+ return *this;
495
+ }
496
+ explicit operator bool() const noexcept { return message_ != nullptr; }
497
+
498
+ #if defined(__cpp_exceptions) && (1 == __cpp_exceptions)
499
+ ~error_t() noexcept(false) {
500
+ if (message_)
501
+ if (!std::uncaught_exception())
502
+ raise();
503
+ }
504
+ void raise() noexcept(false) {
505
+ if (message_)
506
+ throw std::runtime_error(exchange(message_, nullptr));
507
+ }
508
+ #else
509
+ ~error_t() noexcept { raise(); }
510
+ void raise() noexcept {
511
+ if (message_)
512
+ std::terminate();
513
+ }
514
+ #endif
515
+ };
516
+
517
+ template <typename result_at> struct expected_gt {
518
+ result_at result;
519
+ error_t error;
520
+
521
+ operator result_at&() & {
522
+ error.raise();
523
+ return result;
524
+ }
525
+ operator result_at&&() && {
526
+ error.raise();
527
+ return std::move(result);
528
+ }
529
+ explicit operator bool() const noexcept { return !error; }
530
+ expected_gt failed(char const* message) noexcept {
531
+ error = message;
532
+ return std::move(*this);
533
+ }
534
+ };
535
+
420
536
  /**
421
537
  * @brief Light-weight bitset implementation to track visited nodes during graph traversal.
422
538
  * Extends basic functionality with atomic operations.
@@ -426,55 +542,68 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
426
542
  using byte_t = typename allocator_t::value_type;
427
543
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
428
544
 
429
- #if defined(USEARCH_IS_WINDOWS)
430
- using slot_t = LONGLONG;
431
- #else
432
545
  using slot_t = unsigned long;
433
- #endif
434
546
 
435
547
  static constexpr std::size_t bits_per_slot() { return sizeof(slot_t) * CHAR_BIT; }
436
548
  static constexpr slot_t bits_mask() { return sizeof(slot_t) * CHAR_BIT - 1; }
437
549
 
438
550
  slot_t* slots_{};
439
- std::size_t slots_count_{};
551
+ /// @brief Number of slots.
552
+ std::size_t count_{};
440
553
 
441
554
  public:
442
555
  visits_bitset_gt() noexcept {}
443
- explicit visits_bitset_gt(std::size_t capacity) noexcept {
444
- slots_count_ = divide_round_up<bits_per_slot()>(capacity);
445
- slots_ = (slot_t*)allocator_t{}.allocate(slots_count_ * sizeof(slot_t));
446
- clear();
556
+ ~visits_bitset_gt() noexcept { reset(); }
557
+ void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(slot_t)); }
558
+
559
+ void reset() noexcept {
560
+ if (slots_)
561
+ allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(slot_t)), slots_ = nullptr, count_ = 0;
447
562
  }
448
- ~visits_bitset_gt() noexcept {
449
- allocator_t{}.deallocate((byte_t*)slots_, slots_count_ * sizeof(slot_t)), slots_ = nullptr;
563
+
564
+ bool resize(std::size_t capacity) noexcept {
565
+
566
+ std::size_t count = divide_round_up<bits_per_slot()>(capacity);
567
+ if (count <= count_)
568
+ return true;
569
+
570
+ slot_t* slots = (slot_t*)allocator_t{}.allocate(count * sizeof(slot_t));
571
+ if (!slots)
572
+ return false;
573
+
574
+ reset();
575
+ count_ = count;
576
+ slots_ = slots;
577
+ clear();
578
+ return true;
450
579
  }
451
- void clear() noexcept { std::memset(slots_, 0, slots_count_ * sizeof(slot_t)); }
452
- inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
453
- inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
454
580
 
455
581
  visits_bitset_gt(visits_bitset_gt&& other) noexcept {
456
582
  slots_ = other.slots_;
457
- slots_count_ = other.slots_count_;
583
+ count_ = other.count_;
458
584
  other.slots_ = nullptr;
459
- other.slots_count_ = 0;
585
+ other.count_ = 0;
460
586
  }
461
587
 
462
588
  visits_bitset_gt& operator=(visits_bitset_gt&& other) noexcept {
463
589
  std::swap(slots_, other.slots_);
464
- std::swap(slots_count_, other.slots_count_);
590
+ std::swap(count_, other.count_);
465
591
  return *this;
466
592
  }
467
593
 
594
+ inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
595
+ inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
596
+
468
597
  #if defined(USEARCH_IS_WINDOWS)
469
598
 
470
599
  inline bool atomic_set(std::size_t i) noexcept {
471
600
  slot_t mask{1ul << (i & bits_mask())};
472
- return InterlockedOr64Acquire((slot_t volatile*)&slots_[i / bits_per_slot()], mask) & mask;
601
+ return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
473
602
  }
474
603
 
475
604
  inline void atomic_reset(std::size_t i) noexcept {
476
605
  slot_t mask{1ul << (i & bits_mask())};
477
- InterlockedAnd64Release((slot_t volatile*)&slots_[i / bits_per_slot()], ~mask);
606
+ InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
478
607
  }
479
608
 
480
609
  #else
@@ -510,6 +639,9 @@ class max_heap_gt {
510
639
 
511
640
  using value_type = element_t;
512
641
 
642
+ static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
643
+ static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
644
+
513
645
  private:
514
646
  element_t* elements_;
515
647
  std::size_t size_;
@@ -518,11 +650,10 @@ class max_heap_gt {
518
650
  public:
519
651
  max_heap_gt() noexcept : elements_(nullptr), size_(0), capacity_(0) {}
520
652
 
521
- max_heap_gt(max_heap_gt&& other) noexcept {
522
- std::swap(elements_, other.elements_);
523
- std::swap(size_, other.size_);
524
- std::swap(capacity_, other.capacity_);
525
- }
653
+ max_heap_gt(max_heap_gt&& other) noexcept
654
+ : elements_(exchange(other.elements_, nullptr)), size_(exchange(other.size_, 0)),
655
+ capacity_(exchange(other.capacity_, 0)) {}
656
+
526
657
  max_heap_gt& operator=(max_heap_gt&& other) noexcept {
527
658
  std::swap(elements_, other.elements_);
528
659
  std::swap(size_, other.size_);
@@ -533,20 +664,24 @@ class max_heap_gt {
533
664
  max_heap_gt(max_heap_gt const&) = delete;
534
665
  max_heap_gt& operator=(max_heap_gt const&) = delete;
535
666
 
536
- bool empty() const noexcept { return !size_; }
537
- std::size_t size() const noexcept { return size_; }
538
- std::size_t capacity() const noexcept { return capacity_; }
539
- /// @brief Selects the largest element in the heap.
540
- /// @return Reference to the stored element.
541
- element_t const& top() const noexcept { return elements_[0]; }
667
+ ~max_heap_gt() noexcept { reset(); }
542
668
 
543
- void clear() noexcept {
544
- while (size_) {
545
- size_--;
546
- elements_[size_].~element_t();
547
- }
669
+ void reset() noexcept {
670
+ if (elements_)
671
+ allocator_t{}.deallocate(elements_, capacity_);
672
+ elements_ = nullptr;
673
+ capacity_ = 0;
674
+ size_ = 0;
548
675
  }
549
676
 
677
+ inline bool empty() const noexcept { return !size_; }
678
+ inline std::size_t size() const noexcept { return size_; }
679
+ inline std::size_t capacity() const noexcept { return capacity_; }
680
+ /// @brief Selects the largest element in the heap.
681
+ /// @return Reference to the stored element.
682
+ inline element_t const& top() const noexcept { return elements_[0]; }
683
+ inline void clear() noexcept { size_ = 0; }
684
+
550
685
  bool reserve(std::size_t new_capacity) noexcept {
551
686
  if (new_capacity < capacity_)
552
687
  return true;
@@ -559,7 +694,7 @@ class max_heap_gt {
559
694
  return false;
560
695
 
561
696
  if (elements_) {
562
- std::uninitialized_copy_n(elements_, size_, new_elements);
697
+ std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
563
698
  allocator.deallocate(elements_, capacity_);
564
699
  }
565
700
  elements_ = new_elements;
@@ -567,23 +702,21 @@ class max_heap_gt {
567
702
  return new_elements;
568
703
  }
569
704
 
570
- template <typename... args_at> //
571
- bool emplace(args_at&&... args) noexcept {
705
+ bool insert(element_t&& element) noexcept {
572
706
  if (!reserve(size_ + 1))
573
707
  return false;
574
708
 
575
- emplace_reserved(std::forward<args_at>(args)...);
709
+ insert_reserved(std::move(element));
576
710
  return true;
577
711
  }
578
712
 
579
- template <typename... args_at> //
580
- void emplace_reserved(args_at&&... args) noexcept {
581
- new (&elements_[size_]) element_t({std::forward<args_at>(args)...});
713
+ inline void insert_reserved(element_t&& element) noexcept {
714
+ new (&elements_[size_]) element_t(element);
582
715
  size_++;
583
716
  shift_up(size_ - 1);
584
717
  }
585
718
 
586
- element_t pop() noexcept {
719
+ inline element_t pop() noexcept {
587
720
  element_t result = top();
588
721
  std::swap(elements_[0], elements_[size_ - 1]);
589
722
  size_--;
@@ -593,11 +726,11 @@ class max_heap_gt {
593
726
  }
594
727
 
595
728
  /** @brief Invalidates the "max-heap" property, transforming into ascending range. */
596
- void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
597
- void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
729
+ inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
730
+ inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
598
731
 
599
- element_t* data() noexcept { return elements_; }
600
- element_t const* data() const noexcept { return elements_; }
732
+ inline element_t* data() noexcept { return elements_; }
733
+ inline element_t const* data() const noexcept { return elements_; }
601
734
 
602
735
  private:
603
736
  inline std::size_t parent_idx(std::size_t i) const noexcept { return (i - 1u) / 2u; }
@@ -642,6 +775,9 @@ class sorted_buffer_gt {
642
775
  using comparator_t = comparator_at;
643
776
  using allocator_t = allocator_at;
644
777
 
778
+ static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
779
+ static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
780
+
645
781
  using value_type = element_t;
646
782
 
647
783
  private:
@@ -652,11 +788,10 @@ class sorted_buffer_gt {
652
788
  public:
653
789
  sorted_buffer_gt() noexcept : elements_(nullptr), size_(0), capacity_(0) {}
654
790
 
655
- sorted_buffer_gt(sorted_buffer_gt&& other) noexcept {
656
- std::swap(elements_, other.elements_);
657
- std::swap(size_, other.size_);
658
- std::swap(capacity_, other.capacity_);
659
- }
791
+ sorted_buffer_gt(sorted_buffer_gt&& other) noexcept
792
+ : elements_(exchange(other.elements_, nullptr)), size_(exchange(other.size_, 0)),
793
+ capacity_(exchange(other.capacity_, 0)) {}
794
+
660
795
  sorted_buffer_gt& operator=(sorted_buffer_gt&& other) noexcept {
661
796
  std::swap(elements_, other.elements_);
662
797
  std::swap(size_, other.size_);
@@ -667,21 +802,22 @@ class sorted_buffer_gt {
667
802
  sorted_buffer_gt(sorted_buffer_gt const&) = delete;
668
803
  sorted_buffer_gt& operator=(sorted_buffer_gt const&) = delete;
669
804
 
670
- ~sorted_buffer_gt() {
671
- clear();
672
- allocator_t{}.deallocate(elements_, capacity_);
673
- }
674
-
675
- bool empty() const noexcept { return !size_; }
676
- std::size_t size() const noexcept { return size_; }
677
- std::size_t capacity() const noexcept { return capacity_; }
678
- element_t const& top() const noexcept { return elements_[size_ - 1]; }
805
+ ~sorted_buffer_gt() noexcept { reset(); }
679
806
 
680
- void clear() noexcept {
681
- while (size_)
682
- size_--, elements_[size_].~element_t();
807
+ void reset() noexcept {
808
+ if (elements_)
809
+ allocator_t{}.deallocate(elements_, capacity_);
810
+ elements_ = nullptr;
811
+ capacity_ = 0;
812
+ size_ = 0;
683
813
  }
684
814
 
815
+ inline bool empty() const noexcept { return !size_; }
816
+ inline std::size_t size() const noexcept { return size_; }
817
+ inline std::size_t capacity() const noexcept { return capacity_; }
818
+ inline element_t const& top() const noexcept { return elements_[size_ - 1]; }
819
+ inline void clear() noexcept { size_ = 0; }
820
+
685
821
  bool reserve(std::size_t new_capacity) noexcept {
686
822
  if (new_capacity < capacity_)
687
823
  return true;
@@ -694,7 +830,7 @@ class sorted_buffer_gt {
694
830
  return false;
695
831
 
696
832
  if (size_) {
697
- std::uninitialized_copy_n(elements_, size_, new_elements);
833
+ std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
698
834
  allocator.deallocate(elements_, capacity_);
699
835
  }
700
836
  elements_ = new_elements;
@@ -702,8 +838,7 @@ class sorted_buffer_gt {
702
838
  return true;
703
839
  }
704
840
 
705
- template <typename... args_at> void emplace_reserved(args_at&&... args) {
706
- element_t element({std::forward<args_at>(args)...});
841
+ inline void insert_reserved(element_t&& element) noexcept {
707
842
  std::size_t slot = std::lower_bound(elements_, elements_ + size_, element, &less) - elements_;
708
843
  std::size_t to_move = size_ - slot;
709
844
  element_t* source = elements_ + size_ - 1;
@@ -713,7 +848,23 @@ class sorted_buffer_gt {
713
848
  size_++;
714
849
  }
715
850
 
716
- element_t pop() noexcept {
851
+ /**
852
+ * @return `true` if the entry was added, `false` if it wasn't relevant enough.
853
+ */
854
+ inline bool insert(element_t&& element, std::size_t limit) noexcept {
855
+ std::size_t slot = std::lower_bound(elements_, elements_ + size_, element, &less) - elements_;
856
+ if (slot == limit)
857
+ return false;
858
+ std::size_t to_move = size_ - slot - (size_ == limit);
859
+ element_t* source = elements_ + size_ - 1 - (size_ == limit);
860
+ for (; to_move; --to_move, --source)
861
+ source[1] = source[0];
862
+ elements_[slot] = element;
863
+ size_ += size_ != limit;
864
+ return true;
865
+ }
866
+
867
+ inline element_t pop() noexcept {
717
868
  size_--;
718
869
  element_t result = elements_[size_];
719
870
  elements_[size_].~element_t();
@@ -721,17 +872,17 @@ class sorted_buffer_gt {
721
872
  }
722
873
 
723
874
  void sort_ascending() noexcept {}
724
- void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
875
+ inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
725
876
 
726
- element_t* data() noexcept { return elements_; }
727
- element_t const* data() const noexcept { return elements_; }
877
+ inline element_t* data() noexcept { return elements_; }
878
+ inline element_t const* data() const noexcept { return elements_; }
728
879
 
729
880
  private:
730
881
  static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
731
882
  };
732
883
 
733
884
  #if defined(USEARCH_IS_WINDOWS)
734
- #pragma pack(push, 1) // Pack struct members on 1-byte alignment
885
+ #pragma pack(push, 1) // Pack struct elements on 1-byte alignment
735
886
  #endif
736
887
 
737
888
  /**
@@ -812,7 +963,7 @@ static constexpr char const* default_magic() { return "unumusearch"; }
812
963
  * Includes the main `::connectivity` parameter (`M` in the paper)
813
964
  * and two expansion factors - for construction and search.
814
965
  */
815
- struct config_t {
966
+ struct index_config_t {
816
967
 
817
968
  /// @brief Number of neighbors per graph node.
818
969
  /// Defaults to 32 in FAISS and 16 in hnswlib.
@@ -829,14 +980,25 @@ struct config_t {
829
980
  /// > It is called `ef` in the paper.
830
981
  std::size_t expansion_search = default_expansion_search();
831
982
 
832
- ///
833
- std::size_t max_elements = 0;
834
- std::size_t max_threads_add = 1;
835
- std::size_t max_threads_search = 1;
983
+ /// @brief Parameter controlling the physical layout of vectors
984
+ /// in memory. When using multi-byte scalar types, like `float`,
985
+ /// dereferencing at mis-aligned locations may cause UB. So you
986
+ /// should set it to at least the size of expected floats.
987
+ /// Moreover, when using SIMD-accelerated metrics, you may prefer
988
+ /// to align to SIMD register size for higher performance with
989
+ /// less split-loads.
836
990
  std::size_t vector_alignment = 1;
991
+ };
992
+
993
+ struct index_limits_t {
994
+ std::size_t elements = 0;
995
+ std::size_t threads_add = std::thread::hardware_concurrency();
996
+ std::size_t threads_search = std::thread::hardware_concurrency();
837
997
 
838
- std::size_t max_threads() const noexcept { return (std::max)(max_threads_add, max_threads_search); }
839
- std::size_t concurrency() const noexcept { return (std::min)(max_threads_add, max_threads_search); }
998
+ index_limits_t(std::size_t n, std::size_t t) noexcept : elements(n), threads_add(t), threads_search(t) {}
999
+ index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
1000
+ std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
1001
+ std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
840
1002
  };
841
1003
 
842
1004
  struct add_config_t {
@@ -851,6 +1013,24 @@ struct search_config_t {
851
1013
  bool exact{};
852
1014
  };
853
1015
 
1016
+ enum class common_metric_kind_t : std::uint8_t {
1017
+ unknown_k = 0,
1018
+ // Classics:
1019
+ ip_k = 'i',
1020
+ cos_k = 'c',
1021
+ l2sq_k = 'e',
1022
+
1023
+ // Custom:
1024
+ pearson_k = 'p',
1025
+ haversine_k = 'h',
1026
+
1027
+ // Sets:
1028
+ jaccard_k = 'j',
1029
+ bitwise_hamming_k = 'b',
1030
+ bitwise_tanimoto_k = 't',
1031
+ bitwise_sorensen_k = 's',
1032
+ };
1033
+
854
1034
  using file_header_t = byte_t[64];
855
1035
 
856
1036
  /**
@@ -865,17 +1045,6 @@ struct file_head_t {
865
1045
  using version_minor_t = std::uint16_t;
866
1046
  using version_patch_t = std::uint16_t;
867
1047
 
868
- enum metric_t : std::uint8_t {
869
- unknown_k = 0,
870
- ip_k = 'i',
871
- cos_k = 'c',
872
- l2sq_k = 'e',
873
- jaccard_k = 'j',
874
- hamming_k = 'h',
875
- pearson_k = 'p',
876
- haversine_k = 'h',
877
- };
878
-
879
1048
  using connectivity_t = std::uint8_t;
880
1049
  using max_level_t = std::uint8_t;
881
1050
  using vector_alignment_t = std::uint8_t;
@@ -888,7 +1057,7 @@ struct file_head_t {
888
1057
  misaligned_ref_gt<version_major_t> version_major;
889
1058
  misaligned_ref_gt<version_minor_t> version_minor;
890
1059
  misaligned_ref_gt<version_patch_t> version_patch;
891
- misaligned_ref_gt<metric_t> metric;
1060
+ misaligned_ref_gt<common_metric_kind_t> metric;
892
1061
  misaligned_ref_gt<connectivity_t> connectivity;
893
1062
  misaligned_ref_gt<max_level_t> max_level;
894
1063
  misaligned_ref_gt<vector_alignment_t> vector_alignment;
@@ -901,7 +1070,8 @@ struct file_head_t {
901
1070
  : magic((char const*)exchange(ptr, ptr + sizeof(magic_t))),
902
1071
  version_major(exchange(ptr, ptr + sizeof(version_major_t))),
903
1072
  version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
904
- version_patch(exchange(ptr, ptr + sizeof(version_patch_t))), metric(exchange(ptr, ptr + sizeof(metric_t))),
1073
+ version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
1074
+ metric(exchange(ptr, ptr + sizeof(common_metric_kind_t))),
905
1075
  connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
906
1076
  max_level(exchange(ptr, ptr + sizeof(max_level_t))),
907
1077
  vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
@@ -936,6 +1106,13 @@ static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 byt
936
1106
  * @tparam allocator_at
937
1107
  * Dynamic memory allocator.
938
1108
  *
1109
+ * @section Usage
1110
+ *
1111
+ * @subsection Exceptions
1112
+ *
1113
+ * None of the methods throw exceptions in the "Release" compilation mode.
1114
+ * It may only `throw` if your memory ::allocator_at or ::metric_at isn't
1115
+ * safe to copy.
939
1116
  *
940
1117
  * @section Implementation Details
941
1118
  *
@@ -1077,8 +1254,8 @@ class index_gt {
1077
1254
  static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
1078
1255
 
1079
1256
  /**
1080
- * @brief How much larger (number of neighbors per node) will the base level be
1081
- * comapred to other levels.
1257
+ * @brief How much larger (number of neighbors per node) will
1258
+ * the base level be compared to other levels.
1082
1259
  */
1083
1260
  static constexpr std::size_t base_level_multiple_() { return 2; }
1084
1261
 
@@ -1140,6 +1317,7 @@ class index_gt {
1140
1317
  vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
1141
1318
  explicit operator member_ref_t() noexcept { return {{tape_}, vector_view()}; }
1142
1319
  explicit operator member_cref_t() const noexcept { return {label(), vector_view()}; }
1320
+ explicit operator bool() const noexcept { return tape_; }
1143
1321
  };
1144
1322
 
1145
1323
  /**
@@ -1174,7 +1352,7 @@ class index_gt {
1174
1352
  * would reuse to process requests. Similar to having all of those as
1175
1353
  * separate `thread_local` global variables.
1176
1354
  */
1177
- struct usearch_align_m thread_context_t {
1355
+ struct usearch_align_m context_t {
1178
1356
  top_candidates_t top_candidates{};
1179
1357
  next_candidates_t next_candidates{};
1180
1358
  visits_bitset_t visits{};
@@ -1189,7 +1367,8 @@ class index_gt {
1189
1367
  }
1190
1368
  };
1191
1369
 
1192
- config_t config_{};
1370
+ index_config_t config_{};
1371
+ index_limits_t limits_{};
1193
1372
  metric_t metric_{};
1194
1373
  allocator_t allocator_{};
1195
1374
  precomputed_constants_t pre_{};
@@ -1205,36 +1384,36 @@ class index_gt {
1205
1384
  id_t entry_id_{};
1206
1385
 
1207
1386
  using node_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
1208
- std::vector<node_t, node_allocator_t> nodes_{};
1387
+ node_t* nodes_{};
1209
1388
  mutable visits_bitset_t nodes_mutexes_{};
1210
1389
 
1211
- using thread_context_allocator_t = typename allocator_traits_t::template rebind_alloc<thread_context_t>;
1212
- mutable std::vector<thread_context_t, thread_context_allocator_t> thread_contexts_{};
1390
+ using context_allocator_t = typename allocator_traits_t::template rebind_alloc<context_t>;
1391
+ context_t* contexts_{};
1213
1392
 
1214
1393
  public:
1215
1394
  std::size_t connectivity() const noexcept { return config_.connectivity; }
1216
1395
  std::size_t capacity() const noexcept { return capacity_; }
1217
1396
  std::size_t size() const noexcept { return size_; }
1218
1397
  std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
1219
- config_t const& config() const { return config_; }
1398
+ index_config_t const& config() const noexcept { return config_; }
1399
+ index_limits_t const& limits() const noexcept { return limits_; }
1220
1400
  bool is_immutable() const noexcept { return viewed_file_descriptor_ != 0; }
1221
1401
 
1222
- explicit index_gt(config_t config = {}, metric_t metric = {}, allocator_t allocator = {}) noexcept(false)
1223
- : config_(normalize(config)), metric_(metric), allocator_(allocator), pre_(precompute_(config)),
1224
- viewed_file_descriptor_(0), size_(0u), max_level_(-1), entry_id_(0u) {
1225
-
1226
- thread_contexts_.resize(config.max_threads());
1227
- for (thread_context_t& context : thread_contexts_)
1228
- context.metric = metric;
1229
- reserve(config.max_elements);
1230
- }
1402
+ /**
1403
+ * @section Exceptions
1404
+ * Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
1405
+ */
1406
+ explicit index_gt(index_config_t config = {}, metric_t metric = {}, allocator_t allocator = {}) noexcept
1407
+ : config_(normalize(config)), limits_(0, 0), metric_(metric), allocator_(allocator), pre_(precompute_(config)),
1408
+ viewed_file_descriptor_(0), size_(0u), max_level_(-1), entry_id_(0u), nodes_(nullptr), nodes_mutexes_(),
1409
+ contexts_(nullptr) {}
1231
1410
 
1232
1411
  /**
1233
1412
  * @brief Clones the structure with the same hyper-parameters, but without contents.
1234
1413
  */
1235
- index_gt fork() noexcept(false) { return index_gt{config_, metric_, allocator_}; }
1414
+ index_gt fork() noexcept { return index_gt{config_, metric_, allocator_}; }
1236
1415
 
1237
- ~index_gt() noexcept { clear(); }
1416
+ ~index_gt() noexcept { reset(); }
1238
1417
 
1239
1418
  index_gt(index_gt&& other) noexcept { swap(other); }
1240
1419
 
@@ -1265,11 +1444,27 @@ class index_gt {
1265
1444
  entry_id_ = 0u;
1266
1445
  }
1267
1446
 
1447
+ /**
1448
+ * @brief Erases all the vectors from the index, also deallocating the registry.
1449
+ * Will change both `size()` and `capacity()` to zero.
1450
+ */
1451
+ void reset() noexcept {
1452
+ clear();
1453
+
1454
+ if (nodes_)
1455
+ node_allocator_t{}.deallocate(exchange(nodes_, nullptr), limits_.elements);
1456
+ if (contexts_)
1457
+ context_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
1458
+ limits_ = index_limits_t{0, 0};
1459
+ capacity_ = 0;
1460
+ }
1461
+
1268
1462
  /**
1269
1463
  * @brief Swaps the underlying memory buffers and thread contexts.
1270
1464
  */
1271
1465
  void swap(index_gt& other) noexcept {
1272
1466
  std::swap(config_, other.config_);
1467
+ std::swap(limits_, other.limits_);
1273
1468
  std::swap(metric_, other.metric_);
1274
1469
  std::swap(allocator_, other.allocator_);
1275
1470
  std::swap(pre_, other.pre_);
@@ -1277,7 +1472,7 @@ class index_gt {
1277
1472
  std::swap(max_level_, other.max_level_);
1278
1473
  std::swap(entry_id_, other.entry_id_);
1279
1474
  std::swap(nodes_, other.nodes_);
1280
- std::swap(thread_contexts_, other.thread_contexts_);
1475
+ std::swap(contexts_, other.contexts_);
1281
1476
 
1282
1477
  // Non-atomic parts.
1283
1478
  std::size_t capacity_copy = capacity_;
@@ -1290,23 +1485,71 @@ class index_gt {
1290
1485
 
1291
1486
  /**
1292
1487
  * @brief Increases the `capacity()` of the index to allow adding more vectors.
1488
+ * @return `true` on success, `false` on memory allocation errors.
1293
1489
  */
1294
- void reserve(std::size_t new_capacity) noexcept(false) {
1490
+ bool reserve(index_limits_t limits) usearch_noexcept_m {
1295
1491
 
1296
- assert_m(new_capacity >= size_, "Can't drop existing values");
1297
- nodes_.resize(new_capacity);
1298
- nodes_mutexes_ = visits_bitset_t(new_capacity);
1299
- for (thread_context_t& context : thread_contexts_)
1300
- context.visits = visits_bitset_t(new_capacity);
1492
+ usearch_assert_m(limits.elements >= size_, "Can't drop existing values");
1301
1493
 
1302
- capacity_ = new_capacity;
1494
+ if (!nodes_mutexes_.resize(limits.elements))
1495
+ return false;
1496
+
1497
+ node_allocator_t node_allocator;
1498
+ node_t* new_nodes = node_allocator.allocate(limits.elements);
1499
+ if (!new_nodes)
1500
+ return false;
1501
+
1502
+ std::size_t limits_threads = limits.threads();
1503
+ context_allocator_t context_allocator;
1504
+ context_t* new_contexts = context_allocator.allocate(limits_threads);
1505
+ if (!new_contexts) {
1506
+ node_allocator.deallocate(new_nodes, limits.elements);
1507
+ return false;
1508
+ }
1509
+ for (std::size_t i = 0; i != limits_threads; ++i) {
1510
+ context_t& context = new_contexts[i];
1511
+ new (&context) context_t();
1512
+ context.metric = metric_;
1513
+ if (!context.visits.resize(limits.elements)) {
1514
+ for (std::size_t j = 0; j != i; ++j)
1515
+ context.visits.reset();
1516
+ node_allocator.deallocate(new_nodes, limits.elements);
1517
+ context_allocator.deallocate(new_contexts, limits_threads);
1518
+ return false;
1519
+ }
1520
+ }
1521
+
1522
+ // We have passed all the require memory allocations.
1523
+ // The remaining code can't fail. Let's just reuse some of our existing buffers.
1524
+ for (std::size_t i = 0; i != limits_.threads(); ++i) {
1525
+ context_t& old_context = contexts_[i];
1526
+ context_t& context = new_contexts[i];
1527
+ std::swap(old_context.top_candidates, context.top_candidates);
1528
+ std::swap(old_context.next_candidates, context.next_candidates);
1529
+ std::swap(old_context.iteration_cycles, context.iteration_cycles);
1530
+ std::swap(old_context.measurements_count, context.measurements_count);
1531
+ old_context.visits.reset();
1532
+ }
1533
+
1534
+ // Move the nodes info, and deallocate previous buffers.
1535
+ if (nodes_)
1536
+ std::memcpy(new_nodes, nodes_, sizeof(node_t) * size()),
1537
+ node_allocator.deallocate(nodes_, limits_.elements);
1538
+ if (contexts_)
1539
+ context_allocator.deallocate(contexts_, limits_.threads());
1540
+
1541
+ limits_ = limits;
1542
+ capacity_ = limits.elements;
1543
+ nodes_ = new_nodes;
1544
+ contexts_ = new_contexts;
1545
+ return true;
1303
1546
  }
1304
1547
 
1305
1548
  /**
1306
1549
  * @brief Optimizes configuration options to fit the maximum number
1307
1550
  * of neighbors in CPU-cache-aligned buffers.
1308
1551
  */
1309
- static config_t optimize(config_t config) noexcept {
1552
+ static index_config_t optimize(index_config_t config) noexcept {
1310
1553
  precomputed_constants_t pre = precompute_(config);
1311
1554
  std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
1312
1555
  std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
@@ -1316,13 +1559,12 @@ class index_gt {
1316
1559
  }
1317
1560
 
1318
1561
  /**
1319
- * @brief Validates/normalizes the configuration options and throws an exception
1320
- * if those are incorrect.
1562
+ * @brief Validates/normalizes the configuration options.
1321
1563
  */
1322
- static config_t normalize(config_t config) noexcept {
1564
+ static index_config_t normalize(index_config_t config) noexcept {
1323
1565
  config.expansion_add = (std::max<std::size_t>)(config.expansion_add, config.connectivity);
1324
- config.max_threads_add = (std::max<std::size_t>)(1u, config.max_threads_add);
1325
- config.max_threads_search = (std::max<std::size_t>)(1u, config.max_threads_search);
1566
+ // config.threads_add = (std::max<std::size_t>)(1u, config.threads_add);
1567
+ // config.threads_search = (std::max<std::size_t>)(1u, config.threads_search);
1326
1568
  return config;
1327
1569
  }
1328
1570
 
@@ -1331,46 +1573,67 @@ class index_gt {
1331
1573
  #pragma region Construction and Search
1332
1574
 
1333
1575
  struct add_result_t {
1576
+ error_t error{};
1334
1577
  std::size_t new_size{};
1335
1578
  std::size_t cycles{};
1336
1579
  std::size_t measurements{};
1337
1580
  id_t id{};
1581
+
1582
+ explicit operator bool() const noexcept { return !error; }
1583
+ add_result_t failed(char const* message) noexcept {
1584
+ error = message;
1585
+ return std::move(*this);
1586
+ }
1338
1587
  };
1339
1588
 
1340
- struct search_result_t {
1341
- member_cref_t member;
1589
+ struct match_t {
1590
+ member_cref_t element;
1342
1591
  distance_t distance;
1343
1592
  };
1344
1593
 
1345
- class search_results_t {
1594
+ class search_result_t {
1346
1595
  index_gt const& index_;
1347
1596
  top_candidates_t& top_;
1348
1597
 
1349
1598
  friend class index_gt;
1350
- inline search_results_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
1599
+ inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
1351
1600
 
1352
1601
  public:
1353
1602
  std::size_t count{};
1354
1603
  std::size_t cycles{};
1355
1604
  std::size_t measurements{};
1605
+ error_t error{};
1606
+
1607
+ inline search_result_t(search_result_t&&) = default;
1608
+ inline search_result_t& operator=(search_result_t&&) = default;
1356
1609
 
1357
- inline search_results_t(search_results_t&&) = default;
1358
- inline search_results_t& operator=(search_results_t&&) = default;
1610
+ explicit operator bool() const noexcept { return !error; }
1611
+ search_result_t failed(char const* message) noexcept {
1612
+ error = message;
1613
+ return std::move(*this);
1614
+ }
1359
1615
 
1360
1616
  inline operator std::size_t() const noexcept { return count; }
1361
1617
  inline std::size_t size() const noexcept { return count; }
1362
- inline search_result_t operator[](std::size_t i) const noexcept {
1618
+ inline match_t operator[](std::size_t i) const noexcept {
1363
1619
  candidate_t const* top_ordered = top_.data();
1364
1620
  return {member_cref_t(index_.node_with_id_(top_ordered[i].id)), top_ordered[i].distance};
1365
1621
  }
1366
1622
  inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
1367
1623
  for (std::size_t i = 0; i != count; ++i) {
1368
- search_result_t result = operator[](i);
1369
- labels[i] = result.member.label;
1624
+ match_t result = operator[](i);
1625
+ labels[i] = result.element.label;
1370
1626
  distances[i] = result.distance;
1371
1627
  }
1372
1628
  return count;
1373
1629
  }
1630
+ inline std::size_t dump_to(label_t* labels) const noexcept {
1631
+ for (std::size_t i = 0; i != count; ++i) {
1632
+ match_t result = operator[](i);
1633
+ labels[i] = result.element.label;
1634
+ }
1635
+ return count;
1636
+ }
1374
1637
  };
1375
1638
 
1376
1639
  /**
@@ -1380,26 +1643,44 @@ class index_gt {
1380
1643
  * @param[in] vector Contiguous range of scalars forming a vector view.
1381
1644
  * @param[in] config Configuration options for this specific operation.
1382
1645
  */
1383
- add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) noexcept(false) {
1646
+ add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) usearch_noexcept_m {
1384
1647
 
1385
- assert_m(!is_immutable(), "Can't add to an immutable index");
1648
+ usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
1386
1649
  add_result_t result;
1387
- result.new_size = size_.fetch_add(1);
1388
- id_t new_id = result.id = static_cast<id_t>(result.new_size);
1389
1650
 
1390
- // Determining how much memory to allocate depends on the target level
1651
+ // Make sure we have enough local memory to perform this request
1652
+ context_t& context = contexts_[config.thread];
1653
+ top_candidates_t& top = context.top_candidates;
1654
+ next_candidates_t& next = context.next_candidates;
1655
+ top.clear();
1656
+ next.clear();
1657
+
1658
+ // The top list needs one more slot than the connectivity of the base level
1659
+ // for the heuristic, that tries to squeeze one more element into saturated list.
1660
+ std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config_.expansion_add);
1661
+ if (!top.reserve(top_limit))
1662
+ return result.failed("Out of memory!");
1663
+ if (!next.reserve(config_.expansion_add))
1664
+ return result.failed("Out of memory!");
1665
+
1666
+ // Determining how much memory to allocate for the node depends on the target level
1391
1667
  std::unique_lock<std::mutex> new_level_lock(global_mutex_);
1392
1668
  level_t max_level_copy = max_level_; // Copy under lock
1393
1669
  id_t entry_id_copy = entry_id_; // Copy under lock
1394
- thread_context_t& context = thread_contexts_[config.thread];
1395
1670
  level_t target_level = choose_random_level_(context.level_generator);
1396
1671
  if (target_level <= max_level_copy)
1397
1672
  new_level_lock.unlock();
1398
1673
 
1399
1674
  // Allocate the neighbors
1400
- nodes_[result.new_size] = node_malloc_(label, vector, target_level, config.store_vector);
1401
- node_lock_t new_lock = node_lock_(new_id);
1402
- result.new_size++;
1675
+ node_t node = node_malloc_(label, vector, target_level, config.store_vector);
1676
+ if (!node)
1677
+ return result.failed("Out of memory!");
1678
+ std::size_t old_size = size_.fetch_add(1);
1679
+ id_t new_id = static_cast<id_t>(old_size);
1680
+ nodes_[old_size] = node;
1681
+ result.new_size = old_size + 1;
1682
+ result.id = new_id;
1683
+ node_lock_t new_lock = node_lock_(old_size);
1403
1684
 
1404
1685
  // Do nothing for the first element
1405
1686
  if (!new_id) {
@@ -1417,7 +1698,7 @@ class index_gt {
1417
1698
 
1418
1699
  // From `target_level` down perform proper extensive search
1419
1700
  for (level_t level = (std::min)(target_level, max_level_copy); level >= 0; --level) {
1420
- search_to_insert_(closest_id, vector, level, context);
1701
+ search_to_insert_(closest_id, vector, level, context); // TODO: Handle out of memory conditions
1421
1702
  closest_id = connect_new_node_(new_id, level, context);
1422
1703
  }
1423
1704
 
@@ -1434,20 +1715,18 @@ class index_gt {
1434
1715
  }
1435
1716
 
1436
1717
  /**
1437
- * @brief Searches for the closest members to the given ::query. Thread-safe.
1718
+ * @brief Searches for the closest elements to the given ::query. Thread-safe.
1438
1719
  *
1439
1720
  * @param[in] query Contiguous range of scalars forming a vector view.
1440
1721
  * @param[in] wanted The upper bound for the number of results to return.
1441
1722
  * @param[in] config Configuration options for this specific operation.
1442
1723
  * @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
1443
1724
  */
1444
- search_results_t search( //
1445
- vector_view_t query, std::size_t wanted, //
1446
- search_config_t config = {}) const noexcept(false) {
1725
+ search_result_t search(vector_view_t query, std::size_t wanted, search_config_t config = {}) const noexcept {
1447
1726
 
1448
- thread_context_t& context = thread_contexts_[config.thread];
1727
+ context_t& context = contexts_[config.thread];
1449
1728
  top_candidates_t& top = context.top_candidates;
1450
- search_results_t result{*this, top};
1729
+ search_result_t result{*this, top};
1451
1730
  if (!size_)
1452
1731
  return result;
1453
1732
 
@@ -1456,12 +1735,21 @@ class index_gt {
1456
1735
  result.cycles = context.iteration_cycles;
1457
1736
 
1458
1737
  if (config.exact) {
1738
+ if (!top.reserve(wanted))
1739
+ return result.failed("Out of memory!");
1459
1740
  search_exact_(query, wanted, context);
1460
1741
  } else {
1461
- id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
1742
+ next_candidates_t& next = context.next_candidates;
1462
1743
  std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1744
+ if (!next.reserve(expansion))
1745
+ return result.failed("Out of memory!");
1746
+ if (!top.reserve(expansion))
1747
+ return result.failed("Out of memory!");
1748
+
1749
+ id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
1463
1750
  // For bottom layer we need a more optimized procedure
1464
- search_to_find_in_base_(closest_id, query, expansion, context);
1751
+ if (!search_to_find_in_base_(closest_id, query, expansion, context))
1752
+ return result.failed("Out of memory!");
1465
1753
  }
1466
1754
 
1467
1755
  top.sort_ascending();
@@ -1474,21 +1762,28 @@ class index_gt {
1474
1762
  return result;
1475
1763
  }
1476
1764
 
1477
- search_results_t search_around( //
1765
+ search_result_t search_around( //
1478
1766
  id_t hint, vector_view_t query, std::size_t wanted, //
1479
- search_config_t config = {}) const noexcept(false) {
1767
+ search_config_t config = {}) const noexcept {
1480
1768
 
1481
- thread_context_t& context = thread_contexts_[config.thread];
1769
+ context_t& context = contexts_[config.thread];
1482
1770
  top_candidates_t& top = context.top_candidates;
1483
- search_results_t result{*this, top};
1771
+ next_candidates_t& next = context.next_candidates;
1772
+ search_result_t result{*this, top};
1773
+
1484
1774
  if (!size_)
1485
1775
  return result;
1486
1776
 
1777
+ std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1778
+ if (!next.reserve(expansion))
1779
+ return result.failed("Out of memory!");
1780
+ if (!top.reserve(expansion))
1781
+ return result.failed("Out of memory!");
1782
+
1487
1783
  // Go down the level, tracking only the closest match
1488
1784
  result.measurements = context.measurements_count;
1489
1785
  result.cycles = context.iteration_cycles;
1490
1786
 
1491
- std::size_t expansion = (std::max)(config_.expansion_search, wanted);
1492
1787
  search_to_find_in_base_(hint, query, expansion, context);
1493
1788
  top.sort_ascending();
1494
1789
  top.shrink(wanted);
@@ -1531,7 +1826,7 @@ class index_gt {
1531
1826
  std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
1532
1827
  for (std::size_t i = 0; i != result.nodes; ++i) {
1533
1828
  node_t node = node_with_id_(i);
1534
- if (node.level() < level)
1829
+ if (static_cast<std::size_t>(node.level()) < level)
1535
1830
  continue;
1536
1831
 
1537
1832
  result.edges += neighbors_(node, level).size();
@@ -1556,33 +1851,44 @@ class index_gt {
1556
1851
  }
1557
1852
 
1558
1853
  // Temporary data-structures, proportional to the number of nodes:
1559
- total += nodes_.capacity() * sizeof(node_t) + allocator_entry_bytes;
1854
+ total += limits_.elements * sizeof(node_t) + allocator_entry_bytes;
1560
1855
 
1561
1856
  // Temporary data-structures, proportional to the number of threads:
1562
1857
  std::size_t per_thread = (std::max)(config_.expansion_search, config_.expansion_add) * sizeof(candidate_t);
1563
- total += thread_contexts_.capacity() * (sizeof(thread_context_t) + per_thread) + allocator_entry_bytes * 3;
1858
+ total += limits_.threads() * (sizeof(context_t) + per_thread) + allocator_entry_bytes * 3;
1564
1859
  return total;
1565
1860
  }
1566
1861
 
1567
1862
  void change_expansion_add(std::size_t n) noexcept { config_.expansion_add = n; }
1568
1863
  void change_expansion_search(std::size_t n) noexcept { config_.expansion_search = n; }
1569
- void change_metric(metric_t const& m) noexcept(false) {
1864
+ void change_metric(metric_t const& m) noexcept {
1570
1865
  metric_ = m;
1571
- for (thread_context_t& context : thread_contexts_)
1572
- context.metric = metric_;
1866
+ for (std::size_t i = 0; i != limits_.threads(); ++i)
1867
+ contexts_[i].metric = metric_;
1573
1868
  }
1574
1869
 
1575
1870
  #pragma endregion
1576
1871
 
1577
1872
  #pragma region Serialization
1578
1873
 
1874
+ struct serialization_result_t {
1875
+ error_t error;
1876
+
1877
+ explicit operator bool() const noexcept { return !error; }
1878
+ serialization_result_t failed(char const* message) noexcept {
1879
+ error = message;
1880
+ return std::move(*this);
1881
+ }
1882
+ };
1883
+
1579
1884
  /**
1580
1885
  * @brief Saves serialized binary index representation to disk,
1581
1886
  * co-locating vectors and neighbors lists.
1582
1887
  * Available on Linux, MacOS, Windows.
1583
1888
  */
1584
- void save(char const* file_path) const noexcept(false) {
1889
+ serialization_result_t save(char const* file_path) const noexcept {
1585
1890
 
1891
+ serialization_result_t result;
1586
1892
  file_header_t state_buffer{};
1587
1893
  std::memset(state_buffer, 0, sizeof(file_header_t));
1588
1894
  file_head_t state{state_buffer};
@@ -1592,7 +1898,7 @@ class index_gt {
1592
1898
  state.version_major = USEARCH_VERSION_MAJOR;
1593
1899
  state.version_minor = USEARCH_VERSION_MINOR;
1594
1900
  state.version_patch = USEARCH_VERSION_PATCH;
1595
- state.metric = file_head_t::unknown_k;
1901
+ state.metric = common_metric_kind_t::unknown_k;
1596
1902
 
1597
1903
  // Describe state
1598
1904
  state.connectivity = config_.connectivity;
@@ -1603,20 +1909,22 @@ class index_gt {
1603
1909
  state.size = size_;
1604
1910
  state.entry_idx = entry_id_;
1605
1911
 
1606
- std::FILE* file = std::fopen(file_path, "w");
1912
+ std::FILE* file = std::fopen(file_path, "wb");
1607
1913
  if (!file)
1608
- throw std::runtime_error(std::strerror(errno));
1914
+ return result.failed(std::strerror(errno));
1609
1915
 
1610
1916
  auto write_chunk = [&](void* begin, std::size_t length) {
1611
1917
  std::size_t written = std::fwrite(begin, length, 1, file);
1612
1918
  if (!written) {
1613
1919
  std::fclose(file);
1614
- throw std::runtime_error(std::strerror(errno));
1920
+ result.failed(std::strerror(errno));
1615
1921
  }
1616
1922
  };
1617
1923
 
1618
1924
  // Write the header
1619
1925
  write_chunk(&state_buffer[0], sizeof(file_header_t));
1926
+ if (result.error)
1927
+ return result;
1620
1928
 
1621
1929
  // Serialize nodes one by one
1622
1930
  for (std::size_t i = 0; i != state.size; ++i) {
@@ -1625,10 +1933,15 @@ class index_gt {
1625
1933
  std::size_t node_vector_bytes = node_vector_bytes_(node);
1626
1934
  // Dump neighbors and vectors, as vectors may be in a disjoint location
1627
1935
  write_chunk(node.tape(), node_bytes - node_vector_bytes);
1936
+ if (result.error)
1937
+ return result;
1628
1938
  write_chunk(node.vector(), node_vector_bytes);
1939
+ if (result.error)
1940
+ return result;
1629
1941
  }
1630
1942
 
1631
1943
  std::fclose(file);
1944
+ return {};
1632
1945
  }
1633
1946
 
1634
1947
  /**
@@ -1636,38 +1949,48 @@ class index_gt {
1636
1949
  * copying both vectors and neighbors lists into RAM.
1637
1950
  * Available on Linux, MacOS, Windows.
1638
1951
  */
1639
- void load(char const* file_path) noexcept(false) {
1952
+ serialization_result_t load(char const* file_path) noexcept {
1953
+
1954
+ serialization_result_t result;
1640
1955
  file_header_t state_buffer{};
1641
- std::FILE* file = std::fopen(file_path, "r");
1956
+ std::FILE* file = std::fopen(file_path, "rb");
1642
1957
  if (!file)
1643
- throw std::runtime_error(std::strerror(errno));
1958
+ return result.failed(std::strerror(errno));
1644
1959
 
1645
1960
  auto read_chunk = [&](void* begin, std::size_t length) {
1646
1961
  std::size_t read = std::fread(begin, length, 1, file);
1647
1962
  if (!read) {
1648
1963
  std::fclose(file);
1649
- throw std::runtime_error(std::strerror(errno));
1964
+ result.failed(std::strerror(errno));
1650
1965
  }
1651
1966
  };
1652
1967
 
1653
1968
  // Read the header
1654
1969
  {
1655
1970
  read_chunk(&state_buffer[0], sizeof(file_header_t));
1971
+ if (result.error)
1972
+ return result;
1973
+
1656
1974
  file_head_t state{state_buffer};
1657
1975
  if (state.bytes_per_label != sizeof(label_t)) {
1658
1976
  std::fclose(file);
1659
- throw std::runtime_error("Incompatible label type!");
1977
+ return result.failed("Incompatible label type!");
1660
1978
  }
1661
1979
  if (state.bytes_per_id != sizeof(id_t)) {
1662
1980
  std::fclose(file);
1663
- throw std::runtime_error("Incompatible ID type!");
1981
+ return result.failed("Incompatible ID type!");
1664
1982
  }
1665
1983
 
1666
1984
  config_.connectivity = state.connectivity;
1667
- config_.max_elements = state.size;
1668
1985
  config_.vector_alignment = state.vector_alignment;
1669
1986
  pre_ = precompute_(config_);
1670
- reserve(state.size);
1987
+
1988
+ index_limits_t limits;
1989
+ limits.elements = state.size;
1990
+ if (!reserve(limits)) {
1991
+ std::fclose(file);
1992
+ return result.failed("Out of memory");
1993
+ }
1671
1994
  size_ = state.size;
1672
1995
  max_level_ = static_cast<level_t>(state.max_level);
1673
1996
  entry_id_ = static_cast<id_t>(state.entry_idx);
@@ -1679,17 +2002,26 @@ class index_gt {
1679
2002
  dim_t dim;
1680
2003
  level_t level;
1681
2004
  read_chunk(&label, sizeof(label));
2005
+ if (result.error)
2006
+ return result;
1682
2007
  read_chunk(&dim, sizeof(dim));
2008
+ if (result.error)
2009
+ return result;
1683
2010
  read_chunk(&level, sizeof(level));
2011
+ if (result.error)
2012
+ return result;
1684
2013
 
1685
2014
  std::size_t node_bytes = node_bytes_(dim, level);
1686
2015
  node_t node = node_malloc_(label, {nullptr, dim}, level, true);
1687
2016
  read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
2017
+ if (result.error)
2018
+ return result;
1688
2019
  nodes_[i] = node;
1689
2020
  }
1690
2021
 
1691
2022
  std::fclose(file);
1692
2023
  viewed_file_descriptor_ = 0;
2024
+ return {};
1693
2025
  }
1694
2026
 
1695
2027
  /**
@@ -1697,9 +2029,10 @@ class index_gt {
1697
2029
  * @b without copying the vectors and neighbors lists into RAM.
1698
2030
  * Available on Linux, MacOS, but @b not on Windows.
1699
2031
  */
1700
- void view(char const* file_path) noexcept(false) {
2032
+ serialization_result_t view(char const* file_path) noexcept {
2033
+ serialization_result_t result;
1701
2034
  #if defined(USEARCH_IS_WINDOWS)
1702
- throw std::logic_error("Memory-mapping is not yet available for Windows");
2035
+ return result.failed("Memory-mapping is not yet available for Windows");
1703
2036
  #else
1704
2037
  int open_flags = O_RDONLY;
1705
2038
  #if __linux__
@@ -1712,14 +2045,14 @@ class index_gt {
1712
2045
  int fstat_status = fstat(descriptor, &file_stat);
1713
2046
  if (fstat_status < 0) {
1714
2047
  close(descriptor);
1715
- throw std::runtime_error(std::strerror(errno));
2048
+ return result.failed(std::strerror(errno));
1716
2049
  }
1717
2050
 
1718
2051
  // Map the entire file
1719
2052
  byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_PRIVATE, descriptor, 0);
1720
2053
  if (file == MAP_FAILED) {
1721
2054
  close(descriptor);
1722
- throw std::runtime_error(std::strerror(errno));
2055
+ return result.failed(std::strerror(errno));
1723
2056
  }
1724
2057
 
1725
2058
  // Read the header
@@ -1727,19 +2060,23 @@ class index_gt {
1727
2060
  file_head_t state{file};
1728
2061
  if (state.bytes_per_label != sizeof(label_t)) {
1729
2062
  close(descriptor);
1730
- throw std::runtime_error("Incompatible label type!");
2063
+ return result.failed("Incompatible label type!");
1731
2064
  }
1732
2065
  if (state.bytes_per_id != sizeof(id_t)) {
1733
2066
  close(descriptor);
1734
- throw std::runtime_error("Incompatible ID type!");
2067
+ return result.failed("Incompatible ID type!");
1735
2068
  }
1736
2069
 
1737
2070
  config_.connectivity = state.connectivity;
1738
- config_.max_elements = state.size;
1739
2071
  config_.vector_alignment = state.vector_alignment;
1740
- config_.max_threads_add = 0;
1741
2072
  pre_ = precompute_(config_);
1742
- reserve(state.size);
2073
+
2074
+ index_limits_t limits;
2075
+ limits.elements = state.size;
2076
+ limits.threads_add = 0;
2077
+ if (!reserve(limits))
2078
+ return result.failed("Out of memory!");
2079
+
1743
2080
  size_ = state.size;
1744
2081
  max_level_ = static_cast<level_t>(state.max_level);
1745
2082
  entry_id_ = static_cast<id_t>(state.entry_idx);
@@ -1760,13 +2097,14 @@ class index_gt {
1760
2097
  }
1761
2098
 
1762
2099
  viewed_file_descriptor_ = descriptor;
2100
+ return {};
1763
2101
  #endif // POSIX
1764
2102
  }
1765
2103
 
1766
2104
  #pragma endregion
1767
2105
 
1768
2106
  private:
1769
- inline static precomputed_constants_t precompute_(config_t const& config) noexcept {
2107
+ inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
1770
2108
  precomputed_constants_t pre;
1771
2109
  pre.connectivity_max_base = config.connectivity * base_level_multiple_();
1772
2110
  pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
@@ -1799,14 +2137,16 @@ class index_gt {
1799
2137
  node = node_t{};
1800
2138
  }
1801
2139
 
1802
- node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept(false) {
2140
+ node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
1803
2141
 
1804
2142
  std::size_t dim = vector.size();
1805
2143
  std::size_t stored_vector_bytes = node_vector_bytes_(dim) * store_vector;
1806
2144
  std::size_t node_bytes = node_bytes_(dim, level) - node_vector_bytes_(dim) * !store_vector;
1807
2145
 
1808
2146
  byte_t* data = (byte_t*)allocator_t{}.allocate(node_bytes);
1809
- assert_m(data, "Not enough memory for links");
2147
+ if (!data)
2148
+ return {};
2149
+
1810
2150
  std::memset(data, 0, node_bytes);
1811
2151
  if (vector.data())
1812
2152
  std::memcpy(data + node_bytes - stored_vector_bytes, vector.data(), stored_vector_bytes);
@@ -1846,7 +2186,7 @@ class index_gt {
1846
2186
  return {nodes_mutexes_, idx};
1847
2187
  }
1848
2188
 
1849
- id_t connect_new_node_(id_t new_id, level_t level, thread_context_t& context) noexcept(false) {
2189
+ id_t connect_new_node_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
1850
2190
 
1851
2191
  node_t new_node = node_with_id_(new_id);
1852
2192
  top_candidates_t& top = context.top_candidates;
@@ -1855,12 +2195,12 @@ class index_gt {
1855
2195
  // Outgoing links from `new_id`:
1856
2196
  neighbors_ref_t new_neighbors = neighbors_(new_node, level);
1857
2197
  {
1858
- assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
2198
+ usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
1859
2199
  candidates_view_t top_view = refine_(top, config_.connectivity, context);
1860
2200
 
1861
2201
  for (std::size_t idx = 0; idx != top_view.size(); idx++) {
1862
- assert_m(!new_neighbors[idx], "Possible memory corruption");
1863
- assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
2202
+ usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
2203
+ usearch_assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
1864
2204
  new_neighbors.push_back(top_view[idx].id);
1865
2205
  }
1866
2206
  }
@@ -1871,9 +2211,9 @@ class index_gt {
1871
2211
  node_lock_t close_lock = node_lock_(close_id);
1872
2212
 
1873
2213
  neighbors_ref_t close_header = neighbors_(close_node, level);
1874
- assert_m(close_header.size() <= connectivity_max, "Possible corruption");
1875
- assert_m(close_id != new_id, "Self-loops are impossible");
1876
- assert_m(level <= close_node.level(), "Linking to missing level");
2214
+ usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
2215
+ usearch_assert_m(close_id != new_id, "Self-loops are impossible");
2216
+ usearch_assert_m(level <= close_node.level(), "Linking to missing level");
1877
2217
 
1878
2218
  // If `new_id` is already present in the neighboring connections of `close_id`
1879
2219
  // then no need to modify any connections or run the heuristics.
@@ -1884,10 +2224,10 @@ class index_gt {
1884
2224
 
1885
2225
  // To fit a new connection we need to drop an existing one.
1886
2226
  top.clear();
1887
- top.reserve(close_header.size() + 1);
1888
- top.emplace_reserved(context.measure(new_node, close_node), new_id);
2227
+ usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
2228
+ top.insert_reserved({context.measure(new_node, close_node), new_id});
1889
2229
  for (id_t successor_id : close_header)
1890
- top.emplace_reserved(context.measure(node_with_id_(successor_id), close_node), successor_id);
2230
+ top.insert_reserved({context.measure(node_with_id_(successor_id), close_node), successor_id});
1891
2231
 
1892
2232
  // Export the results:
1893
2233
  close_header.clear();
@@ -1908,7 +2248,7 @@ class index_gt {
1908
2248
  id_t search_for_one_( //
1909
2249
  id_t closest_id, vector_view_t query, //
1910
2250
  level_t begin_level, level_t end_level, //
1911
- thread_context_t& context) const noexcept {
2251
+ context_t& context) const noexcept {
1912
2252
 
1913
2253
  distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
1914
2254
  for (level_t level = begin_level; level > end_level; --level) {
@@ -1932,32 +2272,34 @@ class index_gt {
1932
2272
  return closest_id;
1933
2273
  }
1934
2274
 
1935
- void search_to_insert_( //
1936
- id_t start_id, vector_view_t query, level_t level, //
1937
- thread_context_t& context) noexcept(false) {
2275
+ /**
2276
+ * @brief Traverses a layer of a graph, to find the best place to insert a new node.
2277
+ * Locks the nodes in the process, assuming other threads are updating neighbors lists.
2278
+ * @return `true` if procedure succeeded, `false` if run out of memory.
2279
+ */
2280
+ bool search_to_insert_(id_t start_id, vector_view_t query, level_t level, context_t& context) noexcept {
1938
2281
 
1939
2282
  visits_bitset_t& visits = context.visits;
1940
- next_candidates_t& candidates = context.next_candidates; // pop min, push
1941
- top_candidates_t& top = context.top_candidates; // pop max, push
2283
+ next_candidates_t& next = context.next_candidates; // pop min, push
2284
+ top_candidates_t& top = context.top_candidates; // pop max, push
1942
2285
  std::size_t const top_limit = config_.expansion_add;
1943
2286
 
1944
2287
  visits.clear();
1945
- candidates.clear();
2288
+ next.clear();
1946
2289
  top.clear();
1947
- top.reserve(top_limit + 1);
1948
2290
 
1949
2291
  distance_t radius = context.measure(query, node_with_id_(start_id));
1950
- candidates.emplace(-radius, start_id);
1951
- top.emplace_reserved(radius, start_id);
2292
+ next.insert_reserved({-radius, start_id});
2293
+ top.insert_reserved({radius, start_id});
1952
2294
  visits.set(start_id);
1953
2295
 
1954
- while (!candidates.empty()) {
2296
+ while (!next.empty()) {
1955
2297
 
1956
- candidate_t candidacy = candidates.top();
2298
+ candidate_t candidacy = next.top();
1957
2299
  if ((-candidacy.distance) > radius && top.size() == top_limit)
1958
2300
  break;
1959
2301
 
1960
- candidates.pop();
2302
+ next.pop();
1961
2303
  context.iteration_cycles++;
1962
2304
 
1963
2305
  id_t candidate_id = candidacy.id;
@@ -1974,42 +2316,46 @@ class index_gt {
1974
2316
  distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
1975
2317
 
1976
2318
  if (top.size() < top_limit || successor_dist < radius) {
1977
- candidates.emplace(-successor_dist, successor_id);
1978
- top.emplace_reserved(successor_dist, successor_id);
1979
- if (top.size() > top_limit)
1980
- top.pop();
2319
+ // This can substantially grow our priority queue:
2320
+ next.insert({-successor_dist, successor_id});
2321
+ // This will automatically evict poor matches:
2322
+ top.insert({successor_dist, successor_id}, top_limit);
1981
2323
  radius = top.top().distance;
1982
2324
  }
1983
2325
  }
1984
2326
  }
2327
+ return true;
1985
2328
  }
1986
2329
 
1987
- void search_to_find_in_base_( //
1988
- id_t start_id, vector_view_t query, std::size_t expansion, //
1989
- thread_context_t& context) const noexcept(false) {
2330
+ /**
2331
+ * @brief Traverses the @b base layer of a graph, to find a close match.
2332
+ * Doesn't lock any nodes, assuming read-only simultaneous access.
2333
+ * @return `true` if procedure succeeded, `false` if run out of memory.
2334
+ */
2335
+ bool search_to_find_in_base_( //
2336
+ id_t start_id, vector_view_t query, std::size_t expansion, context_t& context) const noexcept {
1990
2337
 
1991
2338
  visits_bitset_t& visits = context.visits;
1992
- next_candidates_t& candidates = context.next_candidates; // pop min, push
1993
- top_candidates_t& top = context.top_candidates; // pop max, push
2339
+ next_candidates_t& next = context.next_candidates; // pop min, push
2340
+ top_candidates_t& top = context.top_candidates; // pop max, push
1994
2341
  std::size_t const top_limit = expansion;
1995
2342
 
1996
2343
  visits.clear();
1997
- candidates.clear();
2344
+ next.clear();
1998
2345
  top.clear();
1999
- top.reserve(top_limit + 1);
2000
2346
 
2001
2347
  distance_t radius = context.measure(query, node_with_id_(start_id));
2002
- candidates.emplace(-radius, start_id);
2003
- top.emplace_reserved(radius, start_id);
2348
+ next.insert_reserved({-radius, start_id});
2349
+ top.insert_reserved({radius, start_id});
2004
2350
  visits.set(start_id);
2005
2351
 
2006
- while (!candidates.empty()) {
2352
+ while (!next.empty()) {
2007
2353
 
2008
- candidate_t candidate = candidates.top();
2354
+ candidate_t candidate = next.top();
2009
2355
  if ((-candidate.distance) > radius)
2010
2356
  break;
2011
2357
 
2012
- candidates.pop();
2358
+ next.pop();
2013
2359
  context.iteration_cycles++;
2014
2360
 
2015
2361
  id_t candidate_id = candidate.id;
@@ -2024,22 +2370,26 @@ class index_gt {
2024
2370
  distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
2025
2371
 
2026
2372
  if (top.size() < top_limit || successor_dist < radius) {
2027
- candidates.emplace(-successor_dist, successor_id);
2028
- top.emplace_reserved(successor_dist, successor_id);
2029
- if (top.size() > top_limit)
2030
- top.pop();
2373
+ // This can substantially grow our priority queue:
2374
+ next.insert({-successor_dist, successor_id});
2375
+ // This will automatically evict poor matches:
2376
+ top.insert({successor_dist, successor_id}, top_limit);
2031
2377
  radius = top.top().distance;
2032
2378
  }
2033
2379
  }
2034
2380
  }
2381
+
2382
+ return true;
2035
2383
  }
2036
2384
 
2037
- void search_exact_(vector_view_t query, std::size_t count, thread_context_t& context) const noexcept(false) {
2385
+ /**
2386
+ * @brief Iterates through all managed vectors, without actually touching the index.
2387
+ */
2388
+ void search_exact_(vector_view_t query, std::size_t count, context_t& context) const noexcept {
2038
2389
  top_candidates_t& top = context.top_candidates;
2039
2390
  top.clear();
2040
- top.reserve(count);
2041
2391
  for (std::size_t i = 0; i != size(); ++i)
2042
- top.emplace_reserved(context.measure(query, node_with_id_(i)), static_cast<id_t>(i));
2392
+ top.insert({context.measure(query, node_with_id_(i)), static_cast<id_t>(i)}, count);
2043
2393
  }
2044
2394
 
2045
2395
  void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
@@ -2049,10 +2399,7 @@ class index_gt {
2049
2399
  * that massively reduces the number of connections a point has,
2050
2400
  * to keep only the neighbors, that are from each other.
2051
2401
  */
2052
-
2053
- candidates_view_t refine_( //
2054
- top_candidates_t& top, std::size_t needed, //
2055
- thread_context_t& context) const noexcept {
2402
+ candidates_view_t refine_(top_candidates_t& top, std::size_t needed, context_t& context) const noexcept {
2056
2403
 
2057
2404
  top.sort_ascending();
2058
2405
  candidate_t* top_data = top.data();