usearch 0.11.1 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +2 -5
- package/.github/workflows/release.yml +22 -2
- package/.gitignore +84 -0
- package/.vscode/launch.json +7 -1
- package/.vscode/settings.json +15 -1
- package/Cargo.toml +1 -1
- package/README.md +43 -8
- package/binding.gyp +2 -2
- package/c/lib.cpp +141 -0
- package/c/usearch.h +43 -0
- package/conanfile.py +1 -1
- package/docs/compilation.md +14 -3
- package/golang/go.mod +3 -0
- package/golang/lib.go +238 -0
- package/golang/usearch.h +9 -0
- package/golang/usearch_par_test.go +40 -0
- package/golang/usearch_test.go +82 -0
- package/include/usearch/usearch.hpp +595 -248
- package/java/cloud/unum/usearch/Index.java +1 -1
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +36 -44
- package/javascript/lib.cpp +10 -6
- package/javascript/usearch.js +3 -0
- package/objc/USearchObjective.mm +3 -3
- package/package.json +4 -3
- package/python/lib.cpp +354 -181
- package/python/scripts/bench.py +2 -2
- package/python/scripts/test.py +88 -29
- package/python/usearch/index.py +132 -57
- package/rust/lib.cpp +33 -24
- package/rust/lib.hpp +2 -2
- package/setup.py +4 -1
- package/src/CMakeLists.txt +4 -4
- package/src/bench.cpp +22 -19
- package/src/punned.hpp +108 -79
- package/src/test.cpp +77 -27
- package/wolfram/lib.cpp +7 -8
- package/Dockerfile +0 -16
- package/VERSION +0 -1
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
*
|
|
7
7
|
* @copyright Copyright (c) 2023
|
|
8
8
|
*/
|
|
9
|
-
#ifndef
|
|
10
|
-
#define
|
|
9
|
+
#ifndef UNUM_USEARCH_HPP
|
|
10
|
+
#define UNUM_USEARCH_HPP
|
|
11
11
|
|
|
12
12
|
#if !defined(USEARCH_VERSION_MAJOR)
|
|
13
13
|
#define USEARCH_VERSION_MAJOR 0
|
|
@@ -100,12 +100,14 @@
|
|
|
100
100
|
|
|
101
101
|
// Debugging
|
|
102
102
|
#if defined(NDEBUG)
|
|
103
|
-
#define
|
|
103
|
+
#define usearch_assert_m(must_be_true, message)
|
|
104
|
+
#define usearch_noexcept_m noexcept
|
|
104
105
|
#else
|
|
105
|
-
#define
|
|
106
|
+
#define usearch_assert_m(must_be_true, message) \
|
|
106
107
|
if (!(must_be_true)) { \
|
|
107
108
|
throw std::runtime_error(message); \
|
|
108
109
|
}
|
|
110
|
+
#define usearch_noexcept_m
|
|
109
111
|
#endif
|
|
110
112
|
|
|
111
113
|
namespace unum {
|
|
@@ -307,14 +309,14 @@ template <typename scalar_at, typename result_at = std::size_t> struct hamming_g
|
|
|
307
309
|
* two arrays of integers. An example would be a textual document,
|
|
308
310
|
* tokenized and hashed into a fixed-capacity bitset.
|
|
309
311
|
*/
|
|
310
|
-
template <typename scalar_at, typename result_at = std::size_t> struct
|
|
312
|
+
template <typename scalar_at, typename result_at = std::size_t> struct bitwise_hamming_gt {
|
|
311
313
|
using scalar_t = scalar_at;
|
|
312
314
|
using result_t = result_at;
|
|
313
315
|
using result_type = result_t;
|
|
314
316
|
static_assert(std::is_unsigned<scalar_t>::value, "Hamming distance requires unsigned integral words");
|
|
315
317
|
|
|
316
|
-
inline result_t operator()(
|
|
317
|
-
|
|
318
|
+
inline result_t operator()( //
|
|
319
|
+
scalar_t const* a, scalar_t const* b, std::size_t words, std::size_t = 0) const noexcept {
|
|
318
320
|
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
319
321
|
result_t matches{};
|
|
320
322
|
#if defined(USEARCH_USE_OPENMP)
|
|
@@ -330,12 +332,72 @@ template <typename scalar_at, typename result_at = std::size_t> struct bit_hammi
|
|
|
330
332
|
}
|
|
331
333
|
};
|
|
332
334
|
|
|
335
|
+
/**
|
|
336
|
+
* @brief Tanimoto distance is the intersection over bitwise union.
|
|
337
|
+
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
338
|
+
*/
|
|
339
|
+
template <typename scalar_at, typename result_at = float> struct bitwise_tanimoto_gt {
|
|
340
|
+
using scalar_t = scalar_at;
|
|
341
|
+
using result_t = result_at;
|
|
342
|
+
using result_type = result_t;
|
|
343
|
+
static_assert(std::is_unsigned<scalar_t>::value, "Tanimoto distance requires unsigned integral words");
|
|
344
|
+
static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
|
|
345
|
+
|
|
346
|
+
inline result_t operator()( //
|
|
347
|
+
scalar_t const* a, scalar_t const* b, std::size_t words, std::size_t = 0) const noexcept {
|
|
348
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
349
|
+
result_t and_count{};
|
|
350
|
+
result_t or_count{};
|
|
351
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
352
|
+
#pragma omp simd reduction(+ : matches)
|
|
353
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
354
|
+
#pragma clang loop vectorize(enable)
|
|
355
|
+
#elif defined(USEARCH_IS_GCC)
|
|
356
|
+
#pragma GCC ivdep
|
|
357
|
+
#endif
|
|
358
|
+
for (std::size_t i = 0; i != words; ++i)
|
|
359
|
+
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
360
|
+
or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
|
|
361
|
+
return 1 - result_t(and_count) / or_count;
|
|
362
|
+
}
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
|
|
367
|
+
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
368
|
+
*/
|
|
369
|
+
template <typename scalar_at, typename result_at = float> struct bitwise_sorensen_gt {
|
|
370
|
+
using scalar_t = scalar_at;
|
|
371
|
+
using result_t = result_at;
|
|
372
|
+
using result_type = result_t;
|
|
373
|
+
static_assert(std::is_unsigned<scalar_t>::value, "Sorensen-Dice distance requires unsigned integral words");
|
|
374
|
+
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
375
|
+
|
|
376
|
+
inline result_t operator()( //
|
|
377
|
+
scalar_t const* a, scalar_t const* b, std::size_t words, std::size_t = 0) const noexcept {
|
|
378
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
379
|
+
result_t and_count{};
|
|
380
|
+
result_t any_count{};
|
|
381
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
382
|
+
#pragma omp simd reduction(+ : matches)
|
|
383
|
+
#elif defined(USEARCH_IS_CLANG)
|
|
384
|
+
#pragma clang loop vectorize(enable)
|
|
385
|
+
#elif defined(USEARCH_IS_GCC)
|
|
386
|
+
#pragma GCC ivdep
|
|
387
|
+
#endif
|
|
388
|
+
for (std::size_t i = 0; i != words; ++i)
|
|
389
|
+
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
390
|
+
any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
|
|
391
|
+
return 1 - 2 * result_t(and_count) / any_count;
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
|
|
333
395
|
/**
|
|
334
396
|
* @brief Counts the number of matching elements in two unique sorted sets.
|
|
335
397
|
* Can be used to compute the similarity between two textual documents
|
|
336
398
|
* using the IDs of tokens present in them.
|
|
337
399
|
*/
|
|
338
|
-
template <typename scalar_at, typename result_at =
|
|
400
|
+
template <typename scalar_at, typename result_at = float> struct jaccard_gt {
|
|
339
401
|
using scalar_t = scalar_at;
|
|
340
402
|
using result_t = result_at;
|
|
341
403
|
using result_type = result_t;
|
|
@@ -351,7 +413,7 @@ template <typename scalar_at, typename result_at = scalar_at> struct jaccard_gt
|
|
|
351
413
|
i += a[i] < b[j];
|
|
352
414
|
j += a[i] >= b[j];
|
|
353
415
|
}
|
|
354
|
-
return 1
|
|
416
|
+
return 1 - intersection / (a_length + b_length - intersection);
|
|
355
417
|
}
|
|
356
418
|
};
|
|
357
419
|
|
|
@@ -360,7 +422,7 @@ template <typename scalar_at, typename result_at = scalar_at> struct jaccard_gt
|
|
|
360
422
|
* Can be used to compute the similarity between two textual documents
|
|
361
423
|
* using the IDs of tokens present in them.
|
|
362
424
|
*/
|
|
363
|
-
template <typename scalar_at, typename result_at =
|
|
425
|
+
template <typename scalar_at, typename result_at = float> struct pearson_correlation_gt {
|
|
364
426
|
using scalar_t = scalar_at;
|
|
365
427
|
using result_t = result_at;
|
|
366
428
|
using result_type = result_t;
|
|
@@ -417,6 +479,60 @@ template <typename scalar_at, typename result_at = scalar_at> struct haversine_g
|
|
|
417
479
|
}
|
|
418
480
|
};
|
|
419
481
|
|
|
482
|
+
class error_t {
|
|
483
|
+
char const* message_{};
|
|
484
|
+
|
|
485
|
+
public:
|
|
486
|
+
error_t(char const* message = nullptr) noexcept : message_(message) {}
|
|
487
|
+
error_t& operator=(char const* message) noexcept {
|
|
488
|
+
message_ = message;
|
|
489
|
+
return *this;
|
|
490
|
+
}
|
|
491
|
+
error_t(error_t&& other) noexcept : message_(exchange(other.message_, nullptr)) {}
|
|
492
|
+
error_t& operator=(error_t&& other) noexcept {
|
|
493
|
+
std::swap(message_, other.message_);
|
|
494
|
+
return *this;
|
|
495
|
+
}
|
|
496
|
+
explicit operator bool() const noexcept { return message_ != nullptr; }
|
|
497
|
+
|
|
498
|
+
#if defined(__cpp_exceptions) && (1 == __cpp_exceptions)
|
|
499
|
+
~error_t() noexcept(false) {
|
|
500
|
+
if (message_)
|
|
501
|
+
if (!std::uncaught_exception())
|
|
502
|
+
raise();
|
|
503
|
+
}
|
|
504
|
+
void raise() noexcept(false) {
|
|
505
|
+
if (message_)
|
|
506
|
+
throw std::runtime_error(exchange(message_, nullptr));
|
|
507
|
+
}
|
|
508
|
+
#else
|
|
509
|
+
~error_t() noexcept { raise(); }
|
|
510
|
+
void raise() noexcept {
|
|
511
|
+
if (message_)
|
|
512
|
+
std::terminate();
|
|
513
|
+
}
|
|
514
|
+
#endif
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
template <typename result_at> struct expected_gt {
|
|
518
|
+
result_at result;
|
|
519
|
+
error_t error;
|
|
520
|
+
|
|
521
|
+
operator result_at&() & {
|
|
522
|
+
error.raise();
|
|
523
|
+
return result;
|
|
524
|
+
}
|
|
525
|
+
operator result_at&&() && {
|
|
526
|
+
error.raise();
|
|
527
|
+
return std::move(result);
|
|
528
|
+
}
|
|
529
|
+
explicit operator bool() const noexcept { return !error; }
|
|
530
|
+
expected_gt failed(char const* message) noexcept {
|
|
531
|
+
error = message;
|
|
532
|
+
return std::move(*this);
|
|
533
|
+
}
|
|
534
|
+
};
|
|
535
|
+
|
|
420
536
|
/**
|
|
421
537
|
* @brief Light-weight bitset implementation to track visited nodes during graph traversal.
|
|
422
538
|
* Extends basic functionality with atomic operations.
|
|
@@ -426,55 +542,68 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
|
|
|
426
542
|
using byte_t = typename allocator_t::value_type;
|
|
427
543
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
428
544
|
|
|
429
|
-
#if defined(USEARCH_IS_WINDOWS)
|
|
430
|
-
using slot_t = LONGLONG;
|
|
431
|
-
#else
|
|
432
545
|
using slot_t = unsigned long;
|
|
433
|
-
#endif
|
|
434
546
|
|
|
435
547
|
static constexpr std::size_t bits_per_slot() { return sizeof(slot_t) * CHAR_BIT; }
|
|
436
548
|
static constexpr slot_t bits_mask() { return sizeof(slot_t) * CHAR_BIT - 1; }
|
|
437
549
|
|
|
438
550
|
slot_t* slots_{};
|
|
439
|
-
|
|
551
|
+
/// @brief Number of slots.
|
|
552
|
+
std::size_t count_{};
|
|
440
553
|
|
|
441
554
|
public:
|
|
442
555
|
visits_bitset_gt() noexcept {}
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
556
|
+
~visits_bitset_gt() noexcept { reset(); }
|
|
557
|
+
void clear() noexcept { std::memset(slots_, 0, count_ * sizeof(slot_t)); }
|
|
558
|
+
|
|
559
|
+
void reset() noexcept {
|
|
560
|
+
if (slots_)
|
|
561
|
+
allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(slot_t)), slots_ = nullptr, count_ = 0;
|
|
447
562
|
}
|
|
448
|
-
|
|
449
|
-
|
|
563
|
+
|
|
564
|
+
bool resize(std::size_t capacity) noexcept {
|
|
565
|
+
|
|
566
|
+
std::size_t count = divide_round_up<bits_per_slot()>(capacity);
|
|
567
|
+
if (count <= count_)
|
|
568
|
+
return true;
|
|
569
|
+
|
|
570
|
+
slot_t* slots = (slot_t*)allocator_t{}.allocate(count * sizeof(slot_t));
|
|
571
|
+
if (!slots)
|
|
572
|
+
return false;
|
|
573
|
+
|
|
574
|
+
reset();
|
|
575
|
+
count_ = count;
|
|
576
|
+
slots_ = slots;
|
|
577
|
+
clear();
|
|
578
|
+
return true;
|
|
450
579
|
}
|
|
451
|
-
void clear() noexcept { std::memset(slots_, 0, slots_count_ * sizeof(slot_t)); }
|
|
452
|
-
inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
|
|
453
|
-
inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
|
|
454
580
|
|
|
455
581
|
visits_bitset_gt(visits_bitset_gt&& other) noexcept {
|
|
456
582
|
slots_ = other.slots_;
|
|
457
|
-
|
|
583
|
+
count_ = other.count_;
|
|
458
584
|
other.slots_ = nullptr;
|
|
459
|
-
other.
|
|
585
|
+
other.count_ = 0;
|
|
460
586
|
}
|
|
461
587
|
|
|
462
588
|
visits_bitset_gt& operator=(visits_bitset_gt&& other) noexcept {
|
|
463
589
|
std::swap(slots_, other.slots_);
|
|
464
|
-
std::swap(
|
|
590
|
+
std::swap(count_, other.count_);
|
|
465
591
|
return *this;
|
|
466
592
|
}
|
|
467
593
|
|
|
594
|
+
inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
|
|
595
|
+
inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
|
|
596
|
+
|
|
468
597
|
#if defined(USEARCH_IS_WINDOWS)
|
|
469
598
|
|
|
470
599
|
inline bool atomic_set(std::size_t i) noexcept {
|
|
471
600
|
slot_t mask{1ul << (i & bits_mask())};
|
|
472
|
-
return
|
|
601
|
+
return InterlockedOr((long volatile*)&slots_[i / bits_per_slot()], mask) & mask;
|
|
473
602
|
}
|
|
474
603
|
|
|
475
604
|
inline void atomic_reset(std::size_t i) noexcept {
|
|
476
605
|
slot_t mask{1ul << (i & bits_mask())};
|
|
477
|
-
|
|
606
|
+
InterlockedAnd((long volatile*)&slots_[i / bits_per_slot()], ~mask);
|
|
478
607
|
}
|
|
479
608
|
|
|
480
609
|
#else
|
|
@@ -510,6 +639,9 @@ class max_heap_gt {
|
|
|
510
639
|
|
|
511
640
|
using value_type = element_t;
|
|
512
641
|
|
|
642
|
+
static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
|
|
643
|
+
static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
|
|
644
|
+
|
|
513
645
|
private:
|
|
514
646
|
element_t* elements_;
|
|
515
647
|
std::size_t size_;
|
|
@@ -518,11 +650,10 @@ class max_heap_gt {
|
|
|
518
650
|
public:
|
|
519
651
|
max_heap_gt() noexcept : elements_(nullptr), size_(0), capacity_(0) {}
|
|
520
652
|
|
|
521
|
-
max_heap_gt(max_heap_gt&& other) noexcept
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
}
|
|
653
|
+
max_heap_gt(max_heap_gt&& other) noexcept
|
|
654
|
+
: elements_(exchange(other.elements_, nullptr)), size_(exchange(other.size_, 0)),
|
|
655
|
+
capacity_(exchange(other.capacity_, 0)) {}
|
|
656
|
+
|
|
526
657
|
max_heap_gt& operator=(max_heap_gt&& other) noexcept {
|
|
527
658
|
std::swap(elements_, other.elements_);
|
|
528
659
|
std::swap(size_, other.size_);
|
|
@@ -533,20 +664,24 @@ class max_heap_gt {
|
|
|
533
664
|
max_heap_gt(max_heap_gt const&) = delete;
|
|
534
665
|
max_heap_gt& operator=(max_heap_gt const&) = delete;
|
|
535
666
|
|
|
536
|
-
|
|
537
|
-
std::size_t size() const noexcept { return size_; }
|
|
538
|
-
std::size_t capacity() const noexcept { return capacity_; }
|
|
539
|
-
/// @brief Selects the largest element in the heap.
|
|
540
|
-
/// @return Reference to the stored element.
|
|
541
|
-
element_t const& top() const noexcept { return elements_[0]; }
|
|
667
|
+
~max_heap_gt() noexcept { reset(); }
|
|
542
668
|
|
|
543
|
-
void
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
669
|
+
void reset() noexcept {
|
|
670
|
+
if (elements_)
|
|
671
|
+
allocator_t{}.deallocate(elements_, capacity_);
|
|
672
|
+
elements_ = nullptr;
|
|
673
|
+
capacity_ = 0;
|
|
674
|
+
size_ = 0;
|
|
548
675
|
}
|
|
549
676
|
|
|
677
|
+
inline bool empty() const noexcept { return !size_; }
|
|
678
|
+
inline std::size_t size() const noexcept { return size_; }
|
|
679
|
+
inline std::size_t capacity() const noexcept { return capacity_; }
|
|
680
|
+
/// @brief Selects the largest element in the heap.
|
|
681
|
+
/// @return Reference to the stored element.
|
|
682
|
+
inline element_t const& top() const noexcept { return elements_[0]; }
|
|
683
|
+
inline void clear() noexcept { size_ = 0; }
|
|
684
|
+
|
|
550
685
|
bool reserve(std::size_t new_capacity) noexcept {
|
|
551
686
|
if (new_capacity < capacity_)
|
|
552
687
|
return true;
|
|
@@ -559,7 +694,7 @@ class max_heap_gt {
|
|
|
559
694
|
return false;
|
|
560
695
|
|
|
561
696
|
if (elements_) {
|
|
562
|
-
std::
|
|
697
|
+
std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
|
|
563
698
|
allocator.deallocate(elements_, capacity_);
|
|
564
699
|
}
|
|
565
700
|
elements_ = new_elements;
|
|
@@ -567,23 +702,21 @@ class max_heap_gt {
|
|
|
567
702
|
return new_elements;
|
|
568
703
|
}
|
|
569
704
|
|
|
570
|
-
|
|
571
|
-
bool emplace(args_at&&... args) noexcept {
|
|
705
|
+
bool insert(element_t&& element) noexcept {
|
|
572
706
|
if (!reserve(size_ + 1))
|
|
573
707
|
return false;
|
|
574
708
|
|
|
575
|
-
|
|
709
|
+
insert_reserved(std::move(element));
|
|
576
710
|
return true;
|
|
577
711
|
}
|
|
578
712
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
new (&elements_[size_]) element_t({std::forward<args_at>(args)...});
|
|
713
|
+
inline void insert_reserved(element_t&& element) noexcept {
|
|
714
|
+
new (&elements_[size_]) element_t(element);
|
|
582
715
|
size_++;
|
|
583
716
|
shift_up(size_ - 1);
|
|
584
717
|
}
|
|
585
718
|
|
|
586
|
-
element_t pop() noexcept {
|
|
719
|
+
inline element_t pop() noexcept {
|
|
587
720
|
element_t result = top();
|
|
588
721
|
std::swap(elements_[0], elements_[size_ - 1]);
|
|
589
722
|
size_--;
|
|
@@ -593,11 +726,11 @@ class max_heap_gt {
|
|
|
593
726
|
}
|
|
594
727
|
|
|
595
728
|
/** @brief Invalidates the "max-heap" property, transforming into ascending range. */
|
|
596
|
-
void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
|
|
597
|
-
void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
729
|
+
inline void sort_ascending() noexcept { std::sort_heap(elements_, elements_ + size_, &less); }
|
|
730
|
+
inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
598
731
|
|
|
599
|
-
element_t* data() noexcept { return elements_; }
|
|
600
|
-
element_t const* data() const noexcept { return elements_; }
|
|
732
|
+
inline element_t* data() noexcept { return elements_; }
|
|
733
|
+
inline element_t const* data() const noexcept { return elements_; }
|
|
601
734
|
|
|
602
735
|
private:
|
|
603
736
|
inline std::size_t parent_idx(std::size_t i) const noexcept { return (i - 1u) / 2u; }
|
|
@@ -642,6 +775,9 @@ class sorted_buffer_gt {
|
|
|
642
775
|
using comparator_t = comparator_at;
|
|
643
776
|
using allocator_t = allocator_at;
|
|
644
777
|
|
|
778
|
+
static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
|
|
779
|
+
static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
|
|
780
|
+
|
|
645
781
|
using value_type = element_t;
|
|
646
782
|
|
|
647
783
|
private:
|
|
@@ -652,11 +788,10 @@ class sorted_buffer_gt {
|
|
|
652
788
|
public:
|
|
653
789
|
sorted_buffer_gt() noexcept : elements_(nullptr), size_(0), capacity_(0) {}
|
|
654
790
|
|
|
655
|
-
sorted_buffer_gt(sorted_buffer_gt&& other) noexcept
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
}
|
|
791
|
+
sorted_buffer_gt(sorted_buffer_gt&& other) noexcept
|
|
792
|
+
: elements_(exchange(other.elements_, nullptr)), size_(exchange(other.size_, 0)),
|
|
793
|
+
capacity_(exchange(other.capacity_, 0)) {}
|
|
794
|
+
|
|
660
795
|
sorted_buffer_gt& operator=(sorted_buffer_gt&& other) noexcept {
|
|
661
796
|
std::swap(elements_, other.elements_);
|
|
662
797
|
std::swap(size_, other.size_);
|
|
@@ -667,21 +802,22 @@ class sorted_buffer_gt {
|
|
|
667
802
|
sorted_buffer_gt(sorted_buffer_gt const&) = delete;
|
|
668
803
|
sorted_buffer_gt& operator=(sorted_buffer_gt const&) = delete;
|
|
669
804
|
|
|
670
|
-
~sorted_buffer_gt() {
|
|
671
|
-
clear();
|
|
672
|
-
allocator_t{}.deallocate(elements_, capacity_);
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
bool empty() const noexcept { return !size_; }
|
|
676
|
-
std::size_t size() const noexcept { return size_; }
|
|
677
|
-
std::size_t capacity() const noexcept { return capacity_; }
|
|
678
|
-
element_t const& top() const noexcept { return elements_[size_ - 1]; }
|
|
805
|
+
~sorted_buffer_gt() noexcept { reset(); }
|
|
679
806
|
|
|
680
|
-
void
|
|
681
|
-
|
|
682
|
-
|
|
807
|
+
void reset() noexcept {
|
|
808
|
+
if (elements_)
|
|
809
|
+
allocator_t{}.deallocate(elements_, capacity_);
|
|
810
|
+
elements_ = nullptr;
|
|
811
|
+
capacity_ = 0;
|
|
812
|
+
size_ = 0;
|
|
683
813
|
}
|
|
684
814
|
|
|
815
|
+
inline bool empty() const noexcept { return !size_; }
|
|
816
|
+
inline std::size_t size() const noexcept { return size_; }
|
|
817
|
+
inline std::size_t capacity() const noexcept { return capacity_; }
|
|
818
|
+
inline element_t const& top() const noexcept { return elements_[size_ - 1]; }
|
|
819
|
+
inline void clear() noexcept { size_ = 0; }
|
|
820
|
+
|
|
685
821
|
bool reserve(std::size_t new_capacity) noexcept {
|
|
686
822
|
if (new_capacity < capacity_)
|
|
687
823
|
return true;
|
|
@@ -694,7 +830,7 @@ class sorted_buffer_gt {
|
|
|
694
830
|
return false;
|
|
695
831
|
|
|
696
832
|
if (size_) {
|
|
697
|
-
std::
|
|
833
|
+
std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
|
|
698
834
|
allocator.deallocate(elements_, capacity_);
|
|
699
835
|
}
|
|
700
836
|
elements_ = new_elements;
|
|
@@ -702,8 +838,7 @@ class sorted_buffer_gt {
|
|
|
702
838
|
return true;
|
|
703
839
|
}
|
|
704
840
|
|
|
705
|
-
|
|
706
|
-
element_t element({std::forward<args_at>(args)...});
|
|
841
|
+
inline void insert_reserved(element_t&& element) noexcept {
|
|
707
842
|
std::size_t slot = std::lower_bound(elements_, elements_ + size_, element, &less) - elements_;
|
|
708
843
|
std::size_t to_move = size_ - slot;
|
|
709
844
|
element_t* source = elements_ + size_ - 1;
|
|
@@ -713,7 +848,23 @@ class sorted_buffer_gt {
|
|
|
713
848
|
size_++;
|
|
714
849
|
}
|
|
715
850
|
|
|
716
|
-
|
|
851
|
+
/**
|
|
852
|
+
* @return `true` if the entry was added, `false` if it wasn't relevant enough.
|
|
853
|
+
*/
|
|
854
|
+
inline bool insert(element_t&& element, std::size_t limit) noexcept {
|
|
855
|
+
std::size_t slot = std::lower_bound(elements_, elements_ + size_, element, &less) - elements_;
|
|
856
|
+
if (slot == limit)
|
|
857
|
+
return false;
|
|
858
|
+
std::size_t to_move = size_ - slot - (size_ == limit);
|
|
859
|
+
element_t* source = elements_ + size_ - 1 - (size_ == limit);
|
|
860
|
+
for (; to_move; --to_move, --source)
|
|
861
|
+
source[1] = source[0];
|
|
862
|
+
elements_[slot] = element;
|
|
863
|
+
size_ += size_ != limit;
|
|
864
|
+
return true;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
inline element_t pop() noexcept {
|
|
717
868
|
size_--;
|
|
718
869
|
element_t result = elements_[size_];
|
|
719
870
|
elements_[size_].~element_t();
|
|
@@ -721,17 +872,17 @@ class sorted_buffer_gt {
|
|
|
721
872
|
}
|
|
722
873
|
|
|
723
874
|
void sort_ascending() noexcept {}
|
|
724
|
-
void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
875
|
+
inline void shrink(std::size_t n) noexcept { size_ = (std::min<std::size_t>)(n, size_); }
|
|
725
876
|
|
|
726
|
-
element_t* data() noexcept { return elements_; }
|
|
727
|
-
element_t const* data() const noexcept { return elements_; }
|
|
877
|
+
inline element_t* data() noexcept { return elements_; }
|
|
878
|
+
inline element_t const* data() const noexcept { return elements_; }
|
|
728
879
|
|
|
729
880
|
private:
|
|
730
881
|
static bool less(element_t const& a, element_t const& b) noexcept { return comparator_t{}(a, b); }
|
|
731
882
|
};
|
|
732
883
|
|
|
733
884
|
#if defined(USEARCH_IS_WINDOWS)
|
|
734
|
-
#pragma pack(push, 1) // Pack struct
|
|
885
|
+
#pragma pack(push, 1) // Pack struct elements on 1-byte alignment
|
|
735
886
|
#endif
|
|
736
887
|
|
|
737
888
|
/**
|
|
@@ -812,7 +963,7 @@ static constexpr char const* default_magic() { return "unumusearch"; }
|
|
|
812
963
|
* Includes the main `::connectivity` parameter (`M` in the paper)
|
|
813
964
|
* and two expansion factors - for construction and search.
|
|
814
965
|
*/
|
|
815
|
-
struct
|
|
966
|
+
struct index_config_t {
|
|
816
967
|
|
|
817
968
|
/// @brief Number of neighbors per graph node.
|
|
818
969
|
/// Defaults to 32 in FAISS and 16 in hnswlib.
|
|
@@ -829,14 +980,25 @@ struct config_t {
|
|
|
829
980
|
/// > It is called `ef` in the paper.
|
|
830
981
|
std::size_t expansion_search = default_expansion_search();
|
|
831
982
|
|
|
832
|
-
///
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
983
|
+
/// @brief Parameter controlling the physical layout of vectors
|
|
984
|
+
/// in memory. When using multi-byte scalar types, like `float`,
|
|
985
|
+
/// dereferencing at mis-aligned locations may cause UB. So you
|
|
986
|
+
/// should set it to at least the size of expected floats.
|
|
987
|
+
/// Moreover, when using SIMD-accelerated metrics, you may prefer
|
|
988
|
+
/// to align to SIMD register size for higher performance with
|
|
989
|
+
/// less split-loads.
|
|
836
990
|
std::size_t vector_alignment = 1;
|
|
991
|
+
};
|
|
992
|
+
|
|
993
|
+
struct index_limits_t {
|
|
994
|
+
std::size_t elements = 0;
|
|
995
|
+
std::size_t threads_add = std::thread::hardware_concurrency();
|
|
996
|
+
std::size_t threads_search = std::thread::hardware_concurrency();
|
|
837
997
|
|
|
838
|
-
std::size_t
|
|
839
|
-
std::size_t
|
|
998
|
+
index_limits_t(std::size_t n, std::size_t t) noexcept : elements(n), threads_add(t), threads_search(t) {}
|
|
999
|
+
index_limits_t(std::size_t n = 0) noexcept : index_limits_t(n, std::thread::hardware_concurrency()) {}
|
|
1000
|
+
std::size_t threads() const noexcept { return (std::max)(threads_add, threads_search); }
|
|
1001
|
+
std::size_t concurrency() const noexcept { return (std::min)(threads_add, threads_search); }
|
|
840
1002
|
};
|
|
841
1003
|
|
|
842
1004
|
struct add_config_t {
|
|
@@ -851,6 +1013,24 @@ struct search_config_t {
|
|
|
851
1013
|
bool exact{};
|
|
852
1014
|
};
|
|
853
1015
|
|
|
1016
|
+
enum class common_metric_kind_t : std::uint8_t {
|
|
1017
|
+
unknown_k = 0,
|
|
1018
|
+
// Classics:
|
|
1019
|
+
ip_k = 'i',
|
|
1020
|
+
cos_k = 'c',
|
|
1021
|
+
l2sq_k = 'e',
|
|
1022
|
+
|
|
1023
|
+
// Custom:
|
|
1024
|
+
pearson_k = 'p',
|
|
1025
|
+
haversine_k = 'h',
|
|
1026
|
+
|
|
1027
|
+
// Sets:
|
|
1028
|
+
jaccard_k = 'j',
|
|
1029
|
+
bitwise_hamming_k = 'b',
|
|
1030
|
+
bitwise_tanimoto_k = 't',
|
|
1031
|
+
bitwise_sorensen_k = 's',
|
|
1032
|
+
};
|
|
1033
|
+
|
|
854
1034
|
using file_header_t = byte_t[64];
|
|
855
1035
|
|
|
856
1036
|
/**
|
|
@@ -865,17 +1045,6 @@ struct file_head_t {
|
|
|
865
1045
|
using version_minor_t = std::uint16_t;
|
|
866
1046
|
using version_patch_t = std::uint16_t;
|
|
867
1047
|
|
|
868
|
-
enum metric_t : std::uint8_t {
|
|
869
|
-
unknown_k = 0,
|
|
870
|
-
ip_k = 'i',
|
|
871
|
-
cos_k = 'c',
|
|
872
|
-
l2sq_k = 'e',
|
|
873
|
-
jaccard_k = 'j',
|
|
874
|
-
hamming_k = 'h',
|
|
875
|
-
pearson_k = 'p',
|
|
876
|
-
haversine_k = 'h',
|
|
877
|
-
};
|
|
878
|
-
|
|
879
1048
|
using connectivity_t = std::uint8_t;
|
|
880
1049
|
using max_level_t = std::uint8_t;
|
|
881
1050
|
using vector_alignment_t = std::uint8_t;
|
|
@@ -888,7 +1057,7 @@ struct file_head_t {
|
|
|
888
1057
|
misaligned_ref_gt<version_major_t> version_major;
|
|
889
1058
|
misaligned_ref_gt<version_minor_t> version_minor;
|
|
890
1059
|
misaligned_ref_gt<version_patch_t> version_patch;
|
|
891
|
-
misaligned_ref_gt<
|
|
1060
|
+
misaligned_ref_gt<common_metric_kind_t> metric;
|
|
892
1061
|
misaligned_ref_gt<connectivity_t> connectivity;
|
|
893
1062
|
misaligned_ref_gt<max_level_t> max_level;
|
|
894
1063
|
misaligned_ref_gt<vector_alignment_t> vector_alignment;
|
|
@@ -901,7 +1070,8 @@ struct file_head_t {
|
|
|
901
1070
|
: magic((char const*)exchange(ptr, ptr + sizeof(magic_t))),
|
|
902
1071
|
version_major(exchange(ptr, ptr + sizeof(version_major_t))),
|
|
903
1072
|
version_minor(exchange(ptr, ptr + sizeof(version_minor_t))),
|
|
904
|
-
version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
|
|
1073
|
+
version_patch(exchange(ptr, ptr + sizeof(version_patch_t))),
|
|
1074
|
+
metric(exchange(ptr, ptr + sizeof(common_metric_kind_t))),
|
|
905
1075
|
connectivity(exchange(ptr, ptr + sizeof(connectivity_t))),
|
|
906
1076
|
max_level(exchange(ptr, ptr + sizeof(max_level_t))),
|
|
907
1077
|
vector_alignment(exchange(ptr, ptr + sizeof(vector_alignment_t))),
|
|
@@ -936,6 +1106,13 @@ static_assert(sizeof(file_header_t) == 64, "File header should be exactly 64 byt
|
|
|
936
1106
|
* @tparam allocator_at
|
|
937
1107
|
* Dynamic memory allocator.
|
|
938
1108
|
*
|
|
1109
|
+
* @section Usage
|
|
1110
|
+
*
|
|
1111
|
+
* @subsection Exceptions
|
|
1112
|
+
*
|
|
1113
|
+
* None of the methods throw exceptions in the "Release" compilation mode.
|
|
1114
|
+
* It may only `throw` if your memory ::allocator_at or ::metric_at isn't
|
|
1115
|
+
* safe to copy.
|
|
939
1116
|
*
|
|
940
1117
|
* @section Implementation Details
|
|
941
1118
|
*
|
|
@@ -1077,8 +1254,8 @@ class index_gt {
|
|
|
1077
1254
|
static_assert(sizeof(byte_t) == 1, "Allocator must allocate separate addressable bytes");
|
|
1078
1255
|
|
|
1079
1256
|
/**
|
|
1080
|
-
* @brief How much larger (number of neighbors per node) will
|
|
1081
|
-
*
|
|
1257
|
+
* @brief How much larger (number of neighbors per node) will
|
|
1258
|
+
* the base level be compared to other levels.
|
|
1082
1259
|
*/
|
|
1083
1260
|
static constexpr std::size_t base_level_multiple_() { return 2; }
|
|
1084
1261
|
|
|
@@ -1140,6 +1317,7 @@ class index_gt {
|
|
|
1140
1317
|
vector_view_t vector_view() const noexcept { return {vector(), dim()}; }
|
|
1141
1318
|
explicit operator member_ref_t() noexcept { return {{tape_}, vector_view()}; }
|
|
1142
1319
|
explicit operator member_cref_t() const noexcept { return {label(), vector_view()}; }
|
|
1320
|
+
explicit operator bool() const noexcept { return tape_; }
|
|
1143
1321
|
};
|
|
1144
1322
|
|
|
1145
1323
|
/**
|
|
@@ -1174,7 +1352,7 @@ class index_gt {
|
|
|
1174
1352
|
* would reuse to process requests. Similar to having all of those as
|
|
1175
1353
|
* separate `thread_local` global variables.
|
|
1176
1354
|
*/
|
|
1177
|
-
struct usearch_align_m
|
|
1355
|
+
struct usearch_align_m context_t {
|
|
1178
1356
|
top_candidates_t top_candidates{};
|
|
1179
1357
|
next_candidates_t next_candidates{};
|
|
1180
1358
|
visits_bitset_t visits{};
|
|
@@ -1189,7 +1367,8 @@ class index_gt {
|
|
|
1189
1367
|
}
|
|
1190
1368
|
};
|
|
1191
1369
|
|
|
1192
|
-
|
|
1370
|
+
index_config_t config_{};
|
|
1371
|
+
index_limits_t limits_{};
|
|
1193
1372
|
metric_t metric_{};
|
|
1194
1373
|
allocator_t allocator_{};
|
|
1195
1374
|
precomputed_constants_t pre_{};
|
|
@@ -1205,36 +1384,36 @@ class index_gt {
|
|
|
1205
1384
|
id_t entry_id_{};
|
|
1206
1385
|
|
|
1207
1386
|
using node_allocator_t = typename allocator_traits_t::template rebind_alloc<node_t>;
|
|
1208
|
-
|
|
1387
|
+
node_t* nodes_{};
|
|
1209
1388
|
mutable visits_bitset_t nodes_mutexes_{};
|
|
1210
1389
|
|
|
1211
|
-
using
|
|
1212
|
-
|
|
1390
|
+
using context_allocator_t = typename allocator_traits_t::template rebind_alloc<context_t>;
|
|
1391
|
+
context_t* contexts_{};
|
|
1213
1392
|
|
|
1214
1393
|
public:
|
|
1215
1394
|
std::size_t connectivity() const noexcept { return config_.connectivity; }
|
|
1216
1395
|
std::size_t capacity() const noexcept { return capacity_; }
|
|
1217
1396
|
std::size_t size() const noexcept { return size_; }
|
|
1218
1397
|
std::size_t max_level() const noexcept { return static_cast<std::size_t>(max_level_); }
|
|
1219
|
-
|
|
1398
|
+
index_config_t const& config() const noexcept { return config_; }
|
|
1399
|
+
index_limits_t const& limits() const noexcept { return limits_; }
|
|
1220
1400
|
bool is_immutable() const noexcept { return viewed_file_descriptor_ != 0; }
|
|
1221
1401
|
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
}
|
|
1402
|
+
/**
|
|
1403
|
+
* @section Exceptions
|
|
1404
|
+
* Doesn't throw, unless the ::metric's and ::allocators's throw on copy-construction.
|
|
1405
|
+
*/
|
|
1406
|
+
explicit index_gt(index_config_t config = {}, metric_t metric = {}, allocator_t allocator = {}) noexcept
|
|
1407
|
+
: config_(normalize(config)), limits_(0, 0), metric_(metric), allocator_(allocator), pre_(precompute_(config)),
|
|
1408
|
+
viewed_file_descriptor_(0), size_(0u), max_level_(-1), entry_id_(0u), nodes_(nullptr), nodes_mutexes_(),
|
|
1409
|
+
contexts_(nullptr) {}
|
|
1231
1410
|
|
|
1232
1411
|
/**
|
|
1233
1412
|
* @brief Clones the structure with the same hyper-parameters, but without contents.
|
|
1234
1413
|
*/
|
|
1235
|
-
index_gt fork() noexcept
|
|
1414
|
+
index_gt fork() noexcept { return index_gt{config_, metric_, allocator_}; }
|
|
1236
1415
|
|
|
1237
|
-
~index_gt() noexcept {
|
|
1416
|
+
~index_gt() noexcept { reset(); }
|
|
1238
1417
|
|
|
1239
1418
|
index_gt(index_gt&& other) noexcept { swap(other); }
|
|
1240
1419
|
|
|
@@ -1265,11 +1444,27 @@ class index_gt {
|
|
|
1265
1444
|
entry_id_ = 0u;
|
|
1266
1445
|
}
|
|
1267
1446
|
|
|
1447
|
+
/**
|
|
1448
|
+
* @brief Erases all the vectors from the index, also deallocating the registry.
|
|
1449
|
+
* Will change both `size()` and `capacity()` to zero.
|
|
1450
|
+
*/
|
|
1451
|
+
void reset() noexcept {
|
|
1452
|
+
clear();
|
|
1453
|
+
|
|
1454
|
+
if (nodes_)
|
|
1455
|
+
node_allocator_t{}.deallocate(exchange(nodes_, nullptr), limits_.elements);
|
|
1456
|
+
if (contexts_)
|
|
1457
|
+
context_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
|
|
1458
|
+
limits_ = index_limits_t{0, 0};
|
|
1459
|
+
capacity_ = 0;
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1268
1462
|
/**
|
|
1269
1463
|
* @brief Swaps the underlying memory buffers and thread contexts.
|
|
1270
1464
|
*/
|
|
1271
1465
|
void swap(index_gt& other) noexcept {
|
|
1272
1466
|
std::swap(config_, other.config_);
|
|
1467
|
+
std::swap(limits_, other.limits_);
|
|
1273
1468
|
std::swap(metric_, other.metric_);
|
|
1274
1469
|
std::swap(allocator_, other.allocator_);
|
|
1275
1470
|
std::swap(pre_, other.pre_);
|
|
@@ -1277,7 +1472,7 @@ class index_gt {
|
|
|
1277
1472
|
std::swap(max_level_, other.max_level_);
|
|
1278
1473
|
std::swap(entry_id_, other.entry_id_);
|
|
1279
1474
|
std::swap(nodes_, other.nodes_);
|
|
1280
|
-
std::swap(
|
|
1475
|
+
std::swap(contexts_, other.contexts_);
|
|
1281
1476
|
|
|
1282
1477
|
// Non-atomic parts.
|
|
1283
1478
|
std::size_t capacity_copy = capacity_;
|
|
@@ -1290,23 +1485,71 @@ class index_gt {
|
|
|
1290
1485
|
|
|
1291
1486
|
/**
|
|
1292
1487
|
* @brief Increases the `capacity()` of the index to allow adding more vectors.
|
|
1488
|
+
* @return `true` on success, `false` on memory allocation errors.
|
|
1293
1489
|
*/
|
|
1294
|
-
|
|
1490
|
+
bool reserve(index_limits_t limits) usearch_noexcept_m {
|
|
1295
1491
|
|
|
1296
|
-
|
|
1297
|
-
nodes_.resize(new_capacity);
|
|
1298
|
-
nodes_mutexes_ = visits_bitset_t(new_capacity);
|
|
1299
|
-
for (thread_context_t& context : thread_contexts_)
|
|
1300
|
-
context.visits = visits_bitset_t(new_capacity);
|
|
1492
|
+
usearch_assert_m(limits.elements >= size_, "Can't drop existing values");
|
|
1301
1493
|
|
|
1302
|
-
|
|
1494
|
+
if (!nodes_mutexes_.resize(limits.elements))
|
|
1495
|
+
return false;
|
|
1496
|
+
|
|
1497
|
+
node_allocator_t node_allocator;
|
|
1498
|
+
node_t* new_nodes = node_allocator.allocate(limits.elements);
|
|
1499
|
+
if (!new_nodes)
|
|
1500
|
+
return false;
|
|
1501
|
+
|
|
1502
|
+
std::size_t limits_threads = limits.threads();
|
|
1503
|
+
context_allocator_t context_allocator;
|
|
1504
|
+
context_t* new_contexts = context_allocator.allocate(limits_threads);
|
|
1505
|
+
if (!new_contexts) {
|
|
1506
|
+
node_allocator.deallocate(new_nodes, limits.elements);
|
|
1507
|
+
return false;
|
|
1508
|
+
}
|
|
1509
|
+
for (std::size_t i = 0; i != limits_threads; ++i) {
|
|
1510
|
+
context_t& context = new_contexts[i];
|
|
1511
|
+
new (&context) context_t();
|
|
1512
|
+
context.metric = metric_;
|
|
1513
|
+
if (!context.visits.resize(limits.elements)) {
|
|
1514
|
+
for (std::size_t j = 0; j != i; ++j)
|
|
1515
|
+
context.visits.reset();
|
|
1516
|
+
node_allocator.deallocate(new_nodes, limits.elements);
|
|
1517
|
+
context_allocator.deallocate(new_contexts, limits_threads);
|
|
1518
|
+
return false;
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
// We have passed all the require memory allocations.
|
|
1523
|
+
// The remaining code can't fail. Let's just reuse some of our existing buffers.
|
|
1524
|
+
for (std::size_t i = 0; i != limits_.threads(); ++i) {
|
|
1525
|
+
context_t& old_context = contexts_[i];
|
|
1526
|
+
context_t& context = new_contexts[i];
|
|
1527
|
+
std::swap(old_context.top_candidates, context.top_candidates);
|
|
1528
|
+
std::swap(old_context.next_candidates, context.next_candidates);
|
|
1529
|
+
std::swap(old_context.iteration_cycles, context.iteration_cycles);
|
|
1530
|
+
std::swap(old_context.measurements_count, context.measurements_count);
|
|
1531
|
+
old_context.visits.reset();
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
// Move the nodes info, and deallocate previous buffers.
|
|
1535
|
+
if (nodes_)
|
|
1536
|
+
std::memcpy(new_nodes, nodes_, sizeof(node_t) * size()),
|
|
1537
|
+
node_allocator.deallocate(nodes_, limits_.elements);
|
|
1538
|
+
if (contexts_)
|
|
1539
|
+
context_allocator.deallocate(contexts_, limits_.threads());
|
|
1540
|
+
|
|
1541
|
+
limits_ = limits;
|
|
1542
|
+
capacity_ = limits.elements;
|
|
1543
|
+
nodes_ = new_nodes;
|
|
1544
|
+
contexts_ = new_contexts;
|
|
1545
|
+
return true;
|
|
1303
1546
|
}
|
|
1304
1547
|
|
|
1305
1548
|
/**
|
|
1306
1549
|
* @brief Optimizes configuration options to fit the maximum number
|
|
1307
1550
|
* of neighbors in CPU-cache-aligned buffers.
|
|
1308
1551
|
*/
|
|
1309
|
-
static
|
|
1552
|
+
static index_config_t optimize(index_config_t config) noexcept {
|
|
1310
1553
|
precomputed_constants_t pre = precompute_(config);
|
|
1311
1554
|
std::size_t bytes_per_node_base = node_head_bytes_() + pre.neighbors_base_bytes;
|
|
1312
1555
|
std::size_t rounded_size = divide_round_up<64>(bytes_per_node_base) * 64;
|
|
@@ -1316,13 +1559,12 @@ class index_gt {
|
|
|
1316
1559
|
}
|
|
1317
1560
|
|
|
1318
1561
|
/**
|
|
1319
|
-
* @brief Validates/normalizes the configuration options
|
|
1320
|
-
* if those are incorrect.
|
|
1562
|
+
* @brief Validates/normalizes the configuration options.
|
|
1321
1563
|
*/
|
|
1322
|
-
static
|
|
1564
|
+
static index_config_t normalize(index_config_t config) noexcept {
|
|
1323
1565
|
config.expansion_add = (std::max<std::size_t>)(config.expansion_add, config.connectivity);
|
|
1324
|
-
config.
|
|
1325
|
-
config.
|
|
1566
|
+
// config.threads_add = (std::max<std::size_t>)(1u, config.threads_add);
|
|
1567
|
+
// config.threads_search = (std::max<std::size_t>)(1u, config.threads_search);
|
|
1326
1568
|
return config;
|
|
1327
1569
|
}
|
|
1328
1570
|
|
|
@@ -1331,46 +1573,67 @@ class index_gt {
|
|
|
1331
1573
|
#pragma region Construction and Search
|
|
1332
1574
|
|
|
1333
1575
|
struct add_result_t {
|
|
1576
|
+
error_t error{};
|
|
1334
1577
|
std::size_t new_size{};
|
|
1335
1578
|
std::size_t cycles{};
|
|
1336
1579
|
std::size_t measurements{};
|
|
1337
1580
|
id_t id{};
|
|
1581
|
+
|
|
1582
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1583
|
+
add_result_t failed(char const* message) noexcept {
|
|
1584
|
+
error = message;
|
|
1585
|
+
return std::move(*this);
|
|
1586
|
+
}
|
|
1338
1587
|
};
|
|
1339
1588
|
|
|
1340
|
-
struct
|
|
1341
|
-
member_cref_t
|
|
1589
|
+
struct match_t {
|
|
1590
|
+
member_cref_t element;
|
|
1342
1591
|
distance_t distance;
|
|
1343
1592
|
};
|
|
1344
1593
|
|
|
1345
|
-
class
|
|
1594
|
+
class search_result_t {
|
|
1346
1595
|
index_gt const& index_;
|
|
1347
1596
|
top_candidates_t& top_;
|
|
1348
1597
|
|
|
1349
1598
|
friend class index_gt;
|
|
1350
|
-
inline
|
|
1599
|
+
inline search_result_t(index_gt const& index, top_candidates_t& top) noexcept : index_(index), top_(top) {}
|
|
1351
1600
|
|
|
1352
1601
|
public:
|
|
1353
1602
|
std::size_t count{};
|
|
1354
1603
|
std::size_t cycles{};
|
|
1355
1604
|
std::size_t measurements{};
|
|
1605
|
+
error_t error{};
|
|
1606
|
+
|
|
1607
|
+
inline search_result_t(search_result_t&&) = default;
|
|
1608
|
+
inline search_result_t& operator=(search_result_t&&) = default;
|
|
1356
1609
|
|
|
1357
|
-
|
|
1358
|
-
|
|
1610
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1611
|
+
search_result_t failed(char const* message) noexcept {
|
|
1612
|
+
error = message;
|
|
1613
|
+
return std::move(*this);
|
|
1614
|
+
}
|
|
1359
1615
|
|
|
1360
1616
|
inline operator std::size_t() const noexcept { return count; }
|
|
1361
1617
|
inline std::size_t size() const noexcept { return count; }
|
|
1362
|
-
inline
|
|
1618
|
+
inline match_t operator[](std::size_t i) const noexcept {
|
|
1363
1619
|
candidate_t const* top_ordered = top_.data();
|
|
1364
1620
|
return {member_cref_t(index_.node_with_id_(top_ordered[i].id)), top_ordered[i].distance};
|
|
1365
1621
|
}
|
|
1366
1622
|
inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
|
|
1367
1623
|
for (std::size_t i = 0; i != count; ++i) {
|
|
1368
|
-
|
|
1369
|
-
labels[i] = result.
|
|
1624
|
+
match_t result = operator[](i);
|
|
1625
|
+
labels[i] = result.element.label;
|
|
1370
1626
|
distances[i] = result.distance;
|
|
1371
1627
|
}
|
|
1372
1628
|
return count;
|
|
1373
1629
|
}
|
|
1630
|
+
inline std::size_t dump_to(label_t* labels) const noexcept {
|
|
1631
|
+
for (std::size_t i = 0; i != count; ++i) {
|
|
1632
|
+
match_t result = operator[](i);
|
|
1633
|
+
labels[i] = result.element.label;
|
|
1634
|
+
}
|
|
1635
|
+
return count;
|
|
1636
|
+
}
|
|
1374
1637
|
};
|
|
1375
1638
|
|
|
1376
1639
|
/**
|
|
@@ -1380,26 +1643,44 @@ class index_gt {
|
|
|
1380
1643
|
* @param[in] vector Contiguous range of scalars forming a vector view.
|
|
1381
1644
|
* @param[in] config Configuration options for this specific operation.
|
|
1382
1645
|
*/
|
|
1383
|
-
add_result_t add(label_t label, vector_view_t vector, add_config_t config = {})
|
|
1646
|
+
add_result_t add(label_t label, vector_view_t vector, add_config_t config = {}) usearch_noexcept_m {
|
|
1384
1647
|
|
|
1385
|
-
|
|
1648
|
+
usearch_assert_m(!is_immutable(), "Can't add to an immutable index");
|
|
1386
1649
|
add_result_t result;
|
|
1387
|
-
result.new_size = size_.fetch_add(1);
|
|
1388
|
-
id_t new_id = result.id = static_cast<id_t>(result.new_size);
|
|
1389
1650
|
|
|
1390
|
-
//
|
|
1651
|
+
// Make sure we have enough local memory to perform this request
|
|
1652
|
+
context_t& context = contexts_[config.thread];
|
|
1653
|
+
top_candidates_t& top = context.top_candidates;
|
|
1654
|
+
next_candidates_t& next = context.next_candidates;
|
|
1655
|
+
top.clear();
|
|
1656
|
+
next.clear();
|
|
1657
|
+
|
|
1658
|
+
// The top list needs one more slot than the connectivity of the base level
|
|
1659
|
+
// for the heuristic, that tries to squeeze one more element into saturated list.
|
|
1660
|
+
std::size_t top_limit = (std::max)(base_level_multiple_() * config_.connectivity + 1, config_.expansion_add);
|
|
1661
|
+
if (!top.reserve(top_limit))
|
|
1662
|
+
return result.failed("Out of memory!");
|
|
1663
|
+
if (!next.reserve(config_.expansion_add))
|
|
1664
|
+
return result.failed("Out of memory!");
|
|
1665
|
+
|
|
1666
|
+
// Determining how much memory to allocate for the node depends on the target level
|
|
1391
1667
|
std::unique_lock<std::mutex> new_level_lock(global_mutex_);
|
|
1392
1668
|
level_t max_level_copy = max_level_; // Copy under lock
|
|
1393
1669
|
id_t entry_id_copy = entry_id_; // Copy under lock
|
|
1394
|
-
thread_context_t& context = thread_contexts_[config.thread];
|
|
1395
1670
|
level_t target_level = choose_random_level_(context.level_generator);
|
|
1396
1671
|
if (target_level <= max_level_copy)
|
|
1397
1672
|
new_level_lock.unlock();
|
|
1398
1673
|
|
|
1399
1674
|
// Allocate the neighbors
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1675
|
+
node_t node = node_malloc_(label, vector, target_level, config.store_vector);
|
|
1676
|
+
if (!node)
|
|
1677
|
+
return result.failed("Out of memory!");
|
|
1678
|
+
std::size_t old_size = size_.fetch_add(1);
|
|
1679
|
+
id_t new_id = static_cast<id_t>(old_size);
|
|
1680
|
+
nodes_[old_size] = node;
|
|
1681
|
+
result.new_size = old_size + 1;
|
|
1682
|
+
result.id = new_id;
|
|
1683
|
+
node_lock_t new_lock = node_lock_(old_size);
|
|
1403
1684
|
|
|
1404
1685
|
// Do nothing for the first element
|
|
1405
1686
|
if (!new_id) {
|
|
@@ -1417,7 +1698,7 @@ class index_gt {
|
|
|
1417
1698
|
|
|
1418
1699
|
// From `target_level` down perform proper extensive search
|
|
1419
1700
|
for (level_t level = (std::min)(target_level, max_level_copy); level >= 0; --level) {
|
|
1420
|
-
search_to_insert_(closest_id, vector, level, context);
|
|
1701
|
+
search_to_insert_(closest_id, vector, level, context); // TODO: Handle out of memory conditions
|
|
1421
1702
|
closest_id = connect_new_node_(new_id, level, context);
|
|
1422
1703
|
}
|
|
1423
1704
|
|
|
@@ -1434,20 +1715,18 @@ class index_gt {
|
|
|
1434
1715
|
}
|
|
1435
1716
|
|
|
1436
1717
|
/**
|
|
1437
|
-
* @brief Searches for the closest
|
|
1718
|
+
* @brief Searches for the closest elements to the given ::query. Thread-safe.
|
|
1438
1719
|
*
|
|
1439
1720
|
* @param[in] query Contiguous range of scalars forming a vector view.
|
|
1440
1721
|
* @param[in] wanted The upper bound for the number of results to return.
|
|
1441
1722
|
* @param[in] config Configuration options for this specific operation.
|
|
1442
1723
|
* @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
|
|
1443
1724
|
*/
|
|
1444
|
-
|
|
1445
|
-
vector_view_t query, std::size_t wanted, //
|
|
1446
|
-
search_config_t config = {}) const noexcept(false) {
|
|
1725
|
+
search_result_t search(vector_view_t query, std::size_t wanted, search_config_t config = {}) const noexcept {
|
|
1447
1726
|
|
|
1448
|
-
|
|
1727
|
+
context_t& context = contexts_[config.thread];
|
|
1449
1728
|
top_candidates_t& top = context.top_candidates;
|
|
1450
|
-
|
|
1729
|
+
search_result_t result{*this, top};
|
|
1451
1730
|
if (!size_)
|
|
1452
1731
|
return result;
|
|
1453
1732
|
|
|
@@ -1456,12 +1735,21 @@ class index_gt {
|
|
|
1456
1735
|
result.cycles = context.iteration_cycles;
|
|
1457
1736
|
|
|
1458
1737
|
if (config.exact) {
|
|
1738
|
+
if (!top.reserve(wanted))
|
|
1739
|
+
return result.failed("Out of memory!");
|
|
1459
1740
|
search_exact_(query, wanted, context);
|
|
1460
1741
|
} else {
|
|
1461
|
-
|
|
1742
|
+
next_candidates_t& next = context.next_candidates;
|
|
1462
1743
|
std::size_t expansion = (std::max)(config_.expansion_search, wanted);
|
|
1744
|
+
if (!next.reserve(expansion))
|
|
1745
|
+
return result.failed("Out of memory!");
|
|
1746
|
+
if (!top.reserve(expansion))
|
|
1747
|
+
return result.failed("Out of memory!");
|
|
1748
|
+
|
|
1749
|
+
id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
|
|
1463
1750
|
// For bottom layer we need a more optimized procedure
|
|
1464
|
-
search_to_find_in_base_(closest_id, query, expansion, context)
|
|
1751
|
+
if (!search_to_find_in_base_(closest_id, query, expansion, context))
|
|
1752
|
+
return result.failed("Out of memory!");
|
|
1465
1753
|
}
|
|
1466
1754
|
|
|
1467
1755
|
top.sort_ascending();
|
|
@@ -1474,21 +1762,28 @@ class index_gt {
|
|
|
1474
1762
|
return result;
|
|
1475
1763
|
}
|
|
1476
1764
|
|
|
1477
|
-
|
|
1765
|
+
search_result_t search_around( //
|
|
1478
1766
|
id_t hint, vector_view_t query, std::size_t wanted, //
|
|
1479
|
-
search_config_t config = {}) const noexcept
|
|
1767
|
+
search_config_t config = {}) const noexcept {
|
|
1480
1768
|
|
|
1481
|
-
|
|
1769
|
+
context_t& context = contexts_[config.thread];
|
|
1482
1770
|
top_candidates_t& top = context.top_candidates;
|
|
1483
|
-
|
|
1771
|
+
next_candidates_t& next = context.next_candidates;
|
|
1772
|
+
search_result_t result{*this, top};
|
|
1773
|
+
|
|
1484
1774
|
if (!size_)
|
|
1485
1775
|
return result;
|
|
1486
1776
|
|
|
1777
|
+
std::size_t expansion = (std::max)(config_.expansion_search, wanted);
|
|
1778
|
+
if (!next.reserve(expansion))
|
|
1779
|
+
return result.failed("Out of memory!");
|
|
1780
|
+
if (!top.reserve(expansion))
|
|
1781
|
+
return result.failed("Out of memory!");
|
|
1782
|
+
|
|
1487
1783
|
// Go down the level, tracking only the closest match
|
|
1488
1784
|
result.measurements = context.measurements_count;
|
|
1489
1785
|
result.cycles = context.iteration_cycles;
|
|
1490
1786
|
|
|
1491
|
-
std::size_t expansion = (std::max)(config_.expansion_search, wanted);
|
|
1492
1787
|
search_to_find_in_base_(hint, query, expansion, context);
|
|
1493
1788
|
top.sort_ascending();
|
|
1494
1789
|
top.shrink(wanted);
|
|
@@ -1531,7 +1826,7 @@ class index_gt {
|
|
|
1531
1826
|
std::size_t neighbors_bytes = !level ? pre_.neighbors_base_bytes : pre_.neighbors_bytes;
|
|
1532
1827
|
for (std::size_t i = 0; i != result.nodes; ++i) {
|
|
1533
1828
|
node_t node = node_with_id_(i);
|
|
1534
|
-
if (node.level() < level)
|
|
1829
|
+
if (static_cast<std::size_t>(node.level()) < level)
|
|
1535
1830
|
continue;
|
|
1536
1831
|
|
|
1537
1832
|
result.edges += neighbors_(node, level).size();
|
|
@@ -1556,33 +1851,44 @@ class index_gt {
|
|
|
1556
1851
|
}
|
|
1557
1852
|
|
|
1558
1853
|
// Temporary data-structures, proportional to the number of nodes:
|
|
1559
|
-
total +=
|
|
1854
|
+
total += limits_.elements * sizeof(node_t) + allocator_entry_bytes;
|
|
1560
1855
|
|
|
1561
1856
|
// Temporary data-structures, proportional to the number of threads:
|
|
1562
1857
|
std::size_t per_thread = (std::max)(config_.expansion_search, config_.expansion_add) * sizeof(candidate_t);
|
|
1563
|
-
total +=
|
|
1858
|
+
total += limits_.threads() * (sizeof(context_t) + per_thread) + allocator_entry_bytes * 3;
|
|
1564
1859
|
return total;
|
|
1565
1860
|
}
|
|
1566
1861
|
|
|
1567
1862
|
void change_expansion_add(std::size_t n) noexcept { config_.expansion_add = n; }
|
|
1568
1863
|
void change_expansion_search(std::size_t n) noexcept { config_.expansion_search = n; }
|
|
1569
|
-
void change_metric(metric_t const& m) noexcept
|
|
1864
|
+
void change_metric(metric_t const& m) noexcept {
|
|
1570
1865
|
metric_ = m;
|
|
1571
|
-
for (
|
|
1572
|
-
|
|
1866
|
+
for (std::size_t i = 0; i != limits_.threads(); ++i)
|
|
1867
|
+
contexts_[i].metric = metric_;
|
|
1573
1868
|
}
|
|
1574
1869
|
|
|
1575
1870
|
#pragma endregion
|
|
1576
1871
|
|
|
1577
1872
|
#pragma region Serialization
|
|
1578
1873
|
|
|
1874
|
+
struct serialization_result_t {
|
|
1875
|
+
error_t error;
|
|
1876
|
+
|
|
1877
|
+
explicit operator bool() const noexcept { return !error; }
|
|
1878
|
+
serialization_result_t failed(char const* message) noexcept {
|
|
1879
|
+
error = message;
|
|
1880
|
+
return std::move(*this);
|
|
1881
|
+
}
|
|
1882
|
+
};
|
|
1883
|
+
|
|
1579
1884
|
/**
|
|
1580
1885
|
* @brief Saves serialized binary index representation to disk,
|
|
1581
1886
|
* co-locating vectors and neighbors lists.
|
|
1582
1887
|
* Available on Linux, MacOS, Windows.
|
|
1583
1888
|
*/
|
|
1584
|
-
|
|
1889
|
+
serialization_result_t save(char const* file_path) const noexcept {
|
|
1585
1890
|
|
|
1891
|
+
serialization_result_t result;
|
|
1586
1892
|
file_header_t state_buffer{};
|
|
1587
1893
|
std::memset(state_buffer, 0, sizeof(file_header_t));
|
|
1588
1894
|
file_head_t state{state_buffer};
|
|
@@ -1592,7 +1898,7 @@ class index_gt {
|
|
|
1592
1898
|
state.version_major = USEARCH_VERSION_MAJOR;
|
|
1593
1899
|
state.version_minor = USEARCH_VERSION_MINOR;
|
|
1594
1900
|
state.version_patch = USEARCH_VERSION_PATCH;
|
|
1595
|
-
state.metric =
|
|
1901
|
+
state.metric = common_metric_kind_t::unknown_k;
|
|
1596
1902
|
|
|
1597
1903
|
// Describe state
|
|
1598
1904
|
state.connectivity = config_.connectivity;
|
|
@@ -1603,20 +1909,22 @@ class index_gt {
|
|
|
1603
1909
|
state.size = size_;
|
|
1604
1910
|
state.entry_idx = entry_id_;
|
|
1605
1911
|
|
|
1606
|
-
std::FILE* file = std::fopen(file_path, "
|
|
1912
|
+
std::FILE* file = std::fopen(file_path, "wb");
|
|
1607
1913
|
if (!file)
|
|
1608
|
-
|
|
1914
|
+
return result.failed(std::strerror(errno));
|
|
1609
1915
|
|
|
1610
1916
|
auto write_chunk = [&](void* begin, std::size_t length) {
|
|
1611
1917
|
std::size_t written = std::fwrite(begin, length, 1, file);
|
|
1612
1918
|
if (!written) {
|
|
1613
1919
|
std::fclose(file);
|
|
1614
|
-
|
|
1920
|
+
result.failed(std::strerror(errno));
|
|
1615
1921
|
}
|
|
1616
1922
|
};
|
|
1617
1923
|
|
|
1618
1924
|
// Write the header
|
|
1619
1925
|
write_chunk(&state_buffer[0], sizeof(file_header_t));
|
|
1926
|
+
if (result.error)
|
|
1927
|
+
return result;
|
|
1620
1928
|
|
|
1621
1929
|
// Serialize nodes one by one
|
|
1622
1930
|
for (std::size_t i = 0; i != state.size; ++i) {
|
|
@@ -1625,10 +1933,15 @@ class index_gt {
|
|
|
1625
1933
|
std::size_t node_vector_bytes = node_vector_bytes_(node);
|
|
1626
1934
|
// Dump neighbors and vectors, as vectors may be in a disjoint location
|
|
1627
1935
|
write_chunk(node.tape(), node_bytes - node_vector_bytes);
|
|
1936
|
+
if (result.error)
|
|
1937
|
+
return result;
|
|
1628
1938
|
write_chunk(node.vector(), node_vector_bytes);
|
|
1939
|
+
if (result.error)
|
|
1940
|
+
return result;
|
|
1629
1941
|
}
|
|
1630
1942
|
|
|
1631
1943
|
std::fclose(file);
|
|
1944
|
+
return {};
|
|
1632
1945
|
}
|
|
1633
1946
|
|
|
1634
1947
|
/**
|
|
@@ -1636,38 +1949,48 @@ class index_gt {
|
|
|
1636
1949
|
* copying both vectors and neighbors lists into RAM.
|
|
1637
1950
|
* Available on Linux, MacOS, Windows.
|
|
1638
1951
|
*/
|
|
1639
|
-
|
|
1952
|
+
serialization_result_t load(char const* file_path) noexcept {
|
|
1953
|
+
|
|
1954
|
+
serialization_result_t result;
|
|
1640
1955
|
file_header_t state_buffer{};
|
|
1641
|
-
std::FILE* file = std::fopen(file_path, "
|
|
1956
|
+
std::FILE* file = std::fopen(file_path, "rb");
|
|
1642
1957
|
if (!file)
|
|
1643
|
-
|
|
1958
|
+
return result.failed(std::strerror(errno));
|
|
1644
1959
|
|
|
1645
1960
|
auto read_chunk = [&](void* begin, std::size_t length) {
|
|
1646
1961
|
std::size_t read = std::fread(begin, length, 1, file);
|
|
1647
1962
|
if (!read) {
|
|
1648
1963
|
std::fclose(file);
|
|
1649
|
-
|
|
1964
|
+
result.failed(std::strerror(errno));
|
|
1650
1965
|
}
|
|
1651
1966
|
};
|
|
1652
1967
|
|
|
1653
1968
|
// Read the header
|
|
1654
1969
|
{
|
|
1655
1970
|
read_chunk(&state_buffer[0], sizeof(file_header_t));
|
|
1971
|
+
if (result.error)
|
|
1972
|
+
return result;
|
|
1973
|
+
|
|
1656
1974
|
file_head_t state{state_buffer};
|
|
1657
1975
|
if (state.bytes_per_label != sizeof(label_t)) {
|
|
1658
1976
|
std::fclose(file);
|
|
1659
|
-
|
|
1977
|
+
return result.failed("Incompatible label type!");
|
|
1660
1978
|
}
|
|
1661
1979
|
if (state.bytes_per_id != sizeof(id_t)) {
|
|
1662
1980
|
std::fclose(file);
|
|
1663
|
-
|
|
1981
|
+
return result.failed("Incompatible ID type!");
|
|
1664
1982
|
}
|
|
1665
1983
|
|
|
1666
1984
|
config_.connectivity = state.connectivity;
|
|
1667
|
-
config_.max_elements = state.size;
|
|
1668
1985
|
config_.vector_alignment = state.vector_alignment;
|
|
1669
1986
|
pre_ = precompute_(config_);
|
|
1670
|
-
|
|
1987
|
+
|
|
1988
|
+
index_limits_t limits;
|
|
1989
|
+
limits.elements = state.size;
|
|
1990
|
+
if (!reserve(limits)) {
|
|
1991
|
+
std::fclose(file);
|
|
1992
|
+
return result.failed("Out of memory");
|
|
1993
|
+
}
|
|
1671
1994
|
size_ = state.size;
|
|
1672
1995
|
max_level_ = static_cast<level_t>(state.max_level);
|
|
1673
1996
|
entry_id_ = static_cast<id_t>(state.entry_idx);
|
|
@@ -1679,17 +2002,26 @@ class index_gt {
|
|
|
1679
2002
|
dim_t dim;
|
|
1680
2003
|
level_t level;
|
|
1681
2004
|
read_chunk(&label, sizeof(label));
|
|
2005
|
+
if (result.error)
|
|
2006
|
+
return result;
|
|
1682
2007
|
read_chunk(&dim, sizeof(dim));
|
|
2008
|
+
if (result.error)
|
|
2009
|
+
return result;
|
|
1683
2010
|
read_chunk(&level, sizeof(level));
|
|
2011
|
+
if (result.error)
|
|
2012
|
+
return result;
|
|
1684
2013
|
|
|
1685
2014
|
std::size_t node_bytes = node_bytes_(dim, level);
|
|
1686
2015
|
node_t node = node_malloc_(label, {nullptr, dim}, level, true);
|
|
1687
2016
|
read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
|
|
2017
|
+
if (result.error)
|
|
2018
|
+
return result;
|
|
1688
2019
|
nodes_[i] = node;
|
|
1689
2020
|
}
|
|
1690
2021
|
|
|
1691
2022
|
std::fclose(file);
|
|
1692
2023
|
viewed_file_descriptor_ = 0;
|
|
2024
|
+
return {};
|
|
1693
2025
|
}
|
|
1694
2026
|
|
|
1695
2027
|
/**
|
|
@@ -1697,9 +2029,10 @@ class index_gt {
|
|
|
1697
2029
|
* @b without copying the vectors and neighbors lists into RAM.
|
|
1698
2030
|
* Available on Linux, MacOS, but @b not on Windows.
|
|
1699
2031
|
*/
|
|
1700
|
-
|
|
2032
|
+
serialization_result_t view(char const* file_path) noexcept {
|
|
2033
|
+
serialization_result_t result;
|
|
1701
2034
|
#if defined(USEARCH_IS_WINDOWS)
|
|
1702
|
-
|
|
2035
|
+
return result.failed("Memory-mapping is not yet available for Windows");
|
|
1703
2036
|
#else
|
|
1704
2037
|
int open_flags = O_RDONLY;
|
|
1705
2038
|
#if __linux__
|
|
@@ -1712,14 +2045,14 @@ class index_gt {
|
|
|
1712
2045
|
int fstat_status = fstat(descriptor, &file_stat);
|
|
1713
2046
|
if (fstat_status < 0) {
|
|
1714
2047
|
close(descriptor);
|
|
1715
|
-
|
|
2048
|
+
return result.failed(std::strerror(errno));
|
|
1716
2049
|
}
|
|
1717
2050
|
|
|
1718
2051
|
// Map the entire file
|
|
1719
2052
|
byte_t* file = (byte_t*)mmap(NULL, file_stat.st_size, PROT_READ, MAP_PRIVATE, descriptor, 0);
|
|
1720
2053
|
if (file == MAP_FAILED) {
|
|
1721
2054
|
close(descriptor);
|
|
1722
|
-
|
|
2055
|
+
return result.failed(std::strerror(errno));
|
|
1723
2056
|
}
|
|
1724
2057
|
|
|
1725
2058
|
// Read the header
|
|
@@ -1727,19 +2060,23 @@ class index_gt {
|
|
|
1727
2060
|
file_head_t state{file};
|
|
1728
2061
|
if (state.bytes_per_label != sizeof(label_t)) {
|
|
1729
2062
|
close(descriptor);
|
|
1730
|
-
|
|
2063
|
+
return result.failed("Incompatible label type!");
|
|
1731
2064
|
}
|
|
1732
2065
|
if (state.bytes_per_id != sizeof(id_t)) {
|
|
1733
2066
|
close(descriptor);
|
|
1734
|
-
|
|
2067
|
+
return result.failed("Incompatible ID type!");
|
|
1735
2068
|
}
|
|
1736
2069
|
|
|
1737
2070
|
config_.connectivity = state.connectivity;
|
|
1738
|
-
config_.max_elements = state.size;
|
|
1739
2071
|
config_.vector_alignment = state.vector_alignment;
|
|
1740
|
-
config_.max_threads_add = 0;
|
|
1741
2072
|
pre_ = precompute_(config_);
|
|
1742
|
-
|
|
2073
|
+
|
|
2074
|
+
index_limits_t limits;
|
|
2075
|
+
limits.elements = state.size;
|
|
2076
|
+
limits.threads_add = 0;
|
|
2077
|
+
if (!reserve(limits))
|
|
2078
|
+
return result.failed("Out of memory!");
|
|
2079
|
+
|
|
1743
2080
|
size_ = state.size;
|
|
1744
2081
|
max_level_ = static_cast<level_t>(state.max_level);
|
|
1745
2082
|
entry_id_ = static_cast<id_t>(state.entry_idx);
|
|
@@ -1760,13 +2097,14 @@ class index_gt {
|
|
|
1760
2097
|
}
|
|
1761
2098
|
|
|
1762
2099
|
viewed_file_descriptor_ = descriptor;
|
|
2100
|
+
return {};
|
|
1763
2101
|
#endif // POSIX
|
|
1764
2102
|
}
|
|
1765
2103
|
|
|
1766
2104
|
#pragma endregion
|
|
1767
2105
|
|
|
1768
2106
|
private:
|
|
1769
|
-
inline static precomputed_constants_t precompute_(
|
|
2107
|
+
inline static precomputed_constants_t precompute_(index_config_t const& config) noexcept {
|
|
1770
2108
|
precomputed_constants_t pre;
|
|
1771
2109
|
pre.connectivity_max_base = config.connectivity * base_level_multiple_();
|
|
1772
2110
|
pre.inverse_log_connectivity = 1.0 / std::log(static_cast<double>(config.connectivity));
|
|
@@ -1799,14 +2137,16 @@ class index_gt {
|
|
|
1799
2137
|
node = node_t{};
|
|
1800
2138
|
}
|
|
1801
2139
|
|
|
1802
|
-
node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept
|
|
2140
|
+
node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
|
|
1803
2141
|
|
|
1804
2142
|
std::size_t dim = vector.size();
|
|
1805
2143
|
std::size_t stored_vector_bytes = node_vector_bytes_(dim) * store_vector;
|
|
1806
2144
|
std::size_t node_bytes = node_bytes_(dim, level) - node_vector_bytes_(dim) * !store_vector;
|
|
1807
2145
|
|
|
1808
2146
|
byte_t* data = (byte_t*)allocator_t{}.allocate(node_bytes);
|
|
1809
|
-
|
|
2147
|
+
if (!data)
|
|
2148
|
+
return {};
|
|
2149
|
+
|
|
1810
2150
|
std::memset(data, 0, node_bytes);
|
|
1811
2151
|
if (vector.data())
|
|
1812
2152
|
std::memcpy(data + node_bytes - stored_vector_bytes, vector.data(), stored_vector_bytes);
|
|
@@ -1846,7 +2186,7 @@ class index_gt {
|
|
|
1846
2186
|
return {nodes_mutexes_, idx};
|
|
1847
2187
|
}
|
|
1848
2188
|
|
|
1849
|
-
id_t connect_new_node_(id_t new_id, level_t level,
|
|
2189
|
+
id_t connect_new_node_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
|
|
1850
2190
|
|
|
1851
2191
|
node_t new_node = node_with_id_(new_id);
|
|
1852
2192
|
top_candidates_t& top = context.top_candidates;
|
|
@@ -1855,12 +2195,12 @@ class index_gt {
|
|
|
1855
2195
|
// Outgoing links from `new_id`:
|
|
1856
2196
|
neighbors_ref_t new_neighbors = neighbors_(new_node, level);
|
|
1857
2197
|
{
|
|
1858
|
-
|
|
2198
|
+
usearch_assert_m(!new_neighbors.size(), "The newly inserted element should have blank link list");
|
|
1859
2199
|
candidates_view_t top_view = refine_(top, config_.connectivity, context);
|
|
1860
2200
|
|
|
1861
2201
|
for (std::size_t idx = 0; idx != top_view.size(); idx++) {
|
|
1862
|
-
|
|
1863
|
-
|
|
2202
|
+
usearch_assert_m(!new_neighbors[idx], "Possible memory corruption");
|
|
2203
|
+
usearch_assert_m(level <= node_with_id_(top_view[idx].id).level(), "Linking to missing level");
|
|
1864
2204
|
new_neighbors.push_back(top_view[idx].id);
|
|
1865
2205
|
}
|
|
1866
2206
|
}
|
|
@@ -1871,9 +2211,9 @@ class index_gt {
|
|
|
1871
2211
|
node_lock_t close_lock = node_lock_(close_id);
|
|
1872
2212
|
|
|
1873
2213
|
neighbors_ref_t close_header = neighbors_(close_node, level);
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
2214
|
+
usearch_assert_m(close_header.size() <= connectivity_max, "Possible corruption");
|
|
2215
|
+
usearch_assert_m(close_id != new_id, "Self-loops are impossible");
|
|
2216
|
+
usearch_assert_m(level <= close_node.level(), "Linking to missing level");
|
|
1877
2217
|
|
|
1878
2218
|
// If `new_id` is already present in the neighboring connections of `close_id`
|
|
1879
2219
|
// then no need to modify any connections or run the heuristics.
|
|
@@ -1884,10 +2224,10 @@ class index_gt {
|
|
|
1884
2224
|
|
|
1885
2225
|
// To fit a new connection we need to drop an existing one.
|
|
1886
2226
|
top.clear();
|
|
1887
|
-
top.reserve(close_header.size() + 1);
|
|
1888
|
-
top.
|
|
2227
|
+
usearch_assert_m((top.reserve(close_header.size() + 1)), "The memory must have been reserved in `add`");
|
|
2228
|
+
top.insert_reserved({context.measure(new_node, close_node), new_id});
|
|
1889
2229
|
for (id_t successor_id : close_header)
|
|
1890
|
-
top.
|
|
2230
|
+
top.insert_reserved({context.measure(node_with_id_(successor_id), close_node), successor_id});
|
|
1891
2231
|
|
|
1892
2232
|
// Export the results:
|
|
1893
2233
|
close_header.clear();
|
|
@@ -1908,7 +2248,7 @@ class index_gt {
|
|
|
1908
2248
|
id_t search_for_one_( //
|
|
1909
2249
|
id_t closest_id, vector_view_t query, //
|
|
1910
2250
|
level_t begin_level, level_t end_level, //
|
|
1911
|
-
|
|
2251
|
+
context_t& context) const noexcept {
|
|
1912
2252
|
|
|
1913
2253
|
distance_t closest_dist = context.measure(query, node_with_id_(closest_id));
|
|
1914
2254
|
for (level_t level = begin_level; level > end_level; --level) {
|
|
@@ -1932,32 +2272,34 @@ class index_gt {
|
|
|
1932
2272
|
return closest_id;
|
|
1933
2273
|
}
|
|
1934
2274
|
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
2275
|
+
/**
|
|
2276
|
+
* @brief Traverses a layer of a graph, to find the best place to insert a new node.
|
|
2277
|
+
* Locks the nodes in the process, assuming other threads are updating neighbors lists.
|
|
2278
|
+
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
2279
|
+
*/
|
|
2280
|
+
bool search_to_insert_(id_t start_id, vector_view_t query, level_t level, context_t& context) noexcept {
|
|
1938
2281
|
|
|
1939
2282
|
visits_bitset_t& visits = context.visits;
|
|
1940
|
-
next_candidates_t&
|
|
1941
|
-
top_candidates_t& top = context.top_candidates;
|
|
2283
|
+
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
2284
|
+
top_candidates_t& top = context.top_candidates; // pop max, push
|
|
1942
2285
|
std::size_t const top_limit = config_.expansion_add;
|
|
1943
2286
|
|
|
1944
2287
|
visits.clear();
|
|
1945
|
-
|
|
2288
|
+
next.clear();
|
|
1946
2289
|
top.clear();
|
|
1947
|
-
top.reserve(top_limit + 1);
|
|
1948
2290
|
|
|
1949
2291
|
distance_t radius = context.measure(query, node_with_id_(start_id));
|
|
1950
|
-
|
|
1951
|
-
top.
|
|
2292
|
+
next.insert_reserved({-radius, start_id});
|
|
2293
|
+
top.insert_reserved({radius, start_id});
|
|
1952
2294
|
visits.set(start_id);
|
|
1953
2295
|
|
|
1954
|
-
while (!
|
|
2296
|
+
while (!next.empty()) {
|
|
1955
2297
|
|
|
1956
|
-
candidate_t candidacy =
|
|
2298
|
+
candidate_t candidacy = next.top();
|
|
1957
2299
|
if ((-candidacy.distance) > radius && top.size() == top_limit)
|
|
1958
2300
|
break;
|
|
1959
2301
|
|
|
1960
|
-
|
|
2302
|
+
next.pop();
|
|
1961
2303
|
context.iteration_cycles++;
|
|
1962
2304
|
|
|
1963
2305
|
id_t candidate_id = candidacy.id;
|
|
@@ -1974,42 +2316,46 @@ class index_gt {
|
|
|
1974
2316
|
distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
|
|
1975
2317
|
|
|
1976
2318
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
2319
|
+
// This can substantially grow our priority queue:
|
|
2320
|
+
next.insert({-successor_dist, successor_id});
|
|
2321
|
+
// This will automatically evict poor matches:
|
|
2322
|
+
top.insert({successor_dist, successor_id}, top_limit);
|
|
1981
2323
|
radius = top.top().distance;
|
|
1982
2324
|
}
|
|
1983
2325
|
}
|
|
1984
2326
|
}
|
|
2327
|
+
return true;
|
|
1985
2328
|
}
|
|
1986
2329
|
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
2330
|
+
/**
|
|
2331
|
+
* @brief Traverses the @b base layer of a graph, to find a close match.
|
|
2332
|
+
* Doesn't lock any nodes, assuming read-only simultaneous access.
|
|
2333
|
+
* @return `true` if procedure succeeded, `false` if run out of memory.
|
|
2334
|
+
*/
|
|
2335
|
+
bool search_to_find_in_base_( //
|
|
2336
|
+
id_t start_id, vector_view_t query, std::size_t expansion, context_t& context) const noexcept {
|
|
1990
2337
|
|
|
1991
2338
|
visits_bitset_t& visits = context.visits;
|
|
1992
|
-
next_candidates_t&
|
|
1993
|
-
top_candidates_t& top = context.top_candidates;
|
|
2339
|
+
next_candidates_t& next = context.next_candidates; // pop min, push
|
|
2340
|
+
top_candidates_t& top = context.top_candidates; // pop max, push
|
|
1994
2341
|
std::size_t const top_limit = expansion;
|
|
1995
2342
|
|
|
1996
2343
|
visits.clear();
|
|
1997
|
-
|
|
2344
|
+
next.clear();
|
|
1998
2345
|
top.clear();
|
|
1999
|
-
top.reserve(top_limit + 1);
|
|
2000
2346
|
|
|
2001
2347
|
distance_t radius = context.measure(query, node_with_id_(start_id));
|
|
2002
|
-
|
|
2003
|
-
top.
|
|
2348
|
+
next.insert_reserved({-radius, start_id});
|
|
2349
|
+
top.insert_reserved({radius, start_id});
|
|
2004
2350
|
visits.set(start_id);
|
|
2005
2351
|
|
|
2006
|
-
while (!
|
|
2352
|
+
while (!next.empty()) {
|
|
2007
2353
|
|
|
2008
|
-
candidate_t candidate =
|
|
2354
|
+
candidate_t candidate = next.top();
|
|
2009
2355
|
if ((-candidate.distance) > radius)
|
|
2010
2356
|
break;
|
|
2011
2357
|
|
|
2012
|
-
|
|
2358
|
+
next.pop();
|
|
2013
2359
|
context.iteration_cycles++;
|
|
2014
2360
|
|
|
2015
2361
|
id_t candidate_id = candidate.id;
|
|
@@ -2024,22 +2370,26 @@ class index_gt {
|
|
|
2024
2370
|
distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
|
|
2025
2371
|
|
|
2026
2372
|
if (top.size() < top_limit || successor_dist < radius) {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2373
|
+
// This can substantially grow our priority queue:
|
|
2374
|
+
next.insert({-successor_dist, successor_id});
|
|
2375
|
+
// This will automatically evict poor matches:
|
|
2376
|
+
top.insert({successor_dist, successor_id}, top_limit);
|
|
2031
2377
|
radius = top.top().distance;
|
|
2032
2378
|
}
|
|
2033
2379
|
}
|
|
2034
2380
|
}
|
|
2381
|
+
|
|
2382
|
+
return true;
|
|
2035
2383
|
}
|
|
2036
2384
|
|
|
2037
|
-
|
|
2385
|
+
/**
|
|
2386
|
+
* @brief Iterates through all managed vectors, without actually touching the index.
|
|
2387
|
+
*/
|
|
2388
|
+
void search_exact_(vector_view_t query, std::size_t count, context_t& context) const noexcept {
|
|
2038
2389
|
top_candidates_t& top = context.top_candidates;
|
|
2039
2390
|
top.clear();
|
|
2040
|
-
top.reserve(count);
|
|
2041
2391
|
for (std::size_t i = 0; i != size(); ++i)
|
|
2042
|
-
top.
|
|
2392
|
+
top.insert({context.measure(query, node_with_id_(i)), static_cast<id_t>(i)}, count);
|
|
2043
2393
|
}
|
|
2044
2394
|
|
|
2045
2395
|
void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}
|
|
@@ -2049,10 +2399,7 @@ class index_gt {
|
|
|
2049
2399
|
* that massively reduces the number of connections a point has,
|
|
2050
2400
|
* to keep only the neighbors, that are from each other.
|
|
2051
2401
|
*/
|
|
2052
|
-
|
|
2053
|
-
candidates_view_t refine_( //
|
|
2054
|
-
top_candidates_t& top, std::size_t needed, //
|
|
2055
|
-
thread_context_t& context) const noexcept {
|
|
2402
|
+
candidates_view_t refine_(top_candidates_t& top, std::size_t needed, context_t& context) const noexcept {
|
|
2056
2403
|
|
|
2057
2404
|
top.sort_ascending();
|
|
2058
2405
|
candidate_t* top_data = top.data();
|