usearch 0.19.3 → 0.21.0

package/include/usearch/index.hpp

@@ -162,6 +162,10 @@ template <std::size_t multiple_ak> std::size_t divide_round_up(std::size_t num)
     return (num + multiple_ak - 1) / multiple_ak;
 }
 
+inline std::size_t divide_round_up(std::size_t num, std::size_t denominator) noexcept {
+    return (num + denominator - 1) / denominator;
+}
+
 inline std::size_t ceil2(std::size_t v) noexcept {
     v--;
     v |= v >> 1;
@@ -174,12 +178,16 @@ inline std::size_t ceil2(std::size_t v) noexcept {
     return v;
 }
 
-template <typename at> void misaligned_store(void* ptr, at v) noexcept { std::memcpy(ptr, &v, sizeof(v)); }
+template <typename at> void misaligned_store(void* ptr, at v) noexcept {
+    static_assert(!std::is_reference<at>::value);
+    std::memcpy(ptr, &v, sizeof(at));
+}
 
 /// @brief  Simply dereferencing misaligned pointers can be dangerous.
 template <typename at> at misaligned_load(void* ptr) noexcept {
+    static_assert(!std::is_reference<at>::value);
     at v;
-    std::memcpy(&v, ptr, sizeof(v));
+    std::memcpy(&v, ptr, sizeof(at));
     return v;
 }
 
@@ -250,10 +258,13 @@ template <typename scalar_at> class span_gt {
     std::size_t size_;
 
   public:
-    span_gt(scalar_at* begin, scalar_at const* end) noexcept : data_(begin), size_(end - begin) {}
+    span_gt() noexcept : data_(nullptr), size_(0u) {}
+    span_gt(scalar_at* begin, scalar_at* end) noexcept : data_(begin), size_(end - begin) {}
     span_gt(scalar_at* begin, std::size_t size) noexcept : data_(begin), size_(size) {}
     scalar_at* data() const noexcept { return data_; }
     std::size_t size() const noexcept { return size_; }
+    scalar_at* begin() const noexcept { return data_; }
+    scalar_at* end() const noexcept { return data_ + size_; }
     operator scalar_at*() const noexcept { return data(); }
 };
 
@@ -609,8 +620,8 @@ template <typename result_at> struct expected_gt {
     }
     result_at const& operator*() const noexcept { return result; }
     explicit operator bool() const noexcept { return !error; }
-    expected_gt failed(char const* message) noexcept {
-        error = message;
+    expected_gt failed(error_t message) noexcept {
+        error = std::move(message);
         return std::move(*this);
     }
 };
@@ -640,7 +651,9 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
 
     void reset() noexcept {
         if (slots_)
-            allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(slot_t)), slots_ = nullptr, count_ = 0;
+            allocator_t{}.deallocate((byte_t*)slots_, count_ * sizeof(slot_t));
+        slots_ = nullptr;
+        count_ = 0;
     }
 
     bool resize(std::size_t capacity) noexcept {
@@ -661,10 +674,8 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
     }
 
     visits_bitset_gt(visits_bitset_gt&& other) noexcept {
-        slots_ = other.slots_;
-        count_ = other.count_;
-        other.slots_ = nullptr;
-        other.count_ = 0;
+        slots_ = exchange(other.slots_, nullptr);
+        count_ = exchange(other.count_, 0);
     }
 
     visits_bitset_gt& operator=(visits_bitset_gt&& other) noexcept {
@@ -673,6 +684,9 @@ template <typename allocator_at = std::allocator<char>> class visits_bitset_gt {
         return *this;
     }
 
+    visits_bitset_gt(visits_bitset_gt const&) = delete;
+    visits_bitset_gt& operator=(visits_bitset_gt const&) = delete;
+
     inline bool test(std::size_t i) const noexcept { return slots_[i / bits_per_slot()] & (1ul << (i & bits_mask())); }
     inline void set(std::size_t i) noexcept { slots_[i / bits_per_slot()] |= (1ul << (i & bits_mask())); }
 
@@ -911,17 +925,18 @@ class sorted_buffer_gt {
         if (!new_elements)
             return false;
 
-        if (size_) {
+        if (size_)
             std::memcpy(new_elements, elements_, size_ * sizeof(element_t));
+        if (elements_)
             allocator.deallocate(elements_, capacity_);
-        }
+
         elements_ = new_elements;
         capacity_ = new_capacity;
         return true;
     }
 
     inline void insert_reserved(element_t&& element) noexcept {
-        std::size_t slot = std::lower_bound(elements_, elements_ + size_, element, &less) - elements_;
+        std::size_t slot = size_ ? std::lower_bound(elements_, elements_ + size_, element, &less) - elements_ : 0;
         std::size_t to_move = size_ - slot;
         element_t* source = elements_ + size_ - 1;
         for (; to_move; --to_move, --source)
@@ -934,7 +949,7 @@ class sorted_buffer_gt {
      *  @return `true` if the entry was added, `false` if it wasn't relevant enough.
      */
     inline bool insert(element_t&& element, std::size_t limit) noexcept {
-        std::size_t slot = std::lower_bound(elements_, elements_ + size_, element, &less) - elements_;
+        std::size_t slot = size_ ? std::lower_bound(elements_, elements_ + size_, element, &less) - elements_ : 0;
         if (slot == limit)
             return false;
         std::size_t to_move = size_ - slot - (size_ == limit);
@@ -975,10 +990,10 @@ class usearch_pack_m uint40_t {
 
   public:
     inline uint40_t() noexcept { std::memset(octets, 0, 5); }
-    inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets + 1, (char*)&n, 4), octets[0] = 0; }
-    inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
+    inline uint40_t(std::uint32_t n) noexcept { std::memcpy(octets, (char*)&n, 4); }
+    inline uint40_t(std::uint64_t n) noexcept { std::memcpy(octets, (char*)&n, 5); }
 #if defined(USEARCH_DEFINED_CLANG) && defined(USEARCH_DEFINED_APPLE)
-    inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n + 3, 5); }
+    inline uint40_t(std::size_t n) noexcept { std::memcpy(octets, (char*)&n, 5); }
 #endif
 
     uint40_t(uint40_t&&) = default;
@@ -986,24 +1001,40 @@ class usearch_pack_m uint40_t {
     uint40_t& operator=(uint40_t&&) = default;
     uint40_t& operator=(uint40_t const&) = default;
 
-    inline uint40_t& operator+=(std::uint32_t i) noexcept {
-        std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets + 1);
-        octets[0] += static_cast<unsigned char>((tail + i) < tail);
-        tail += i;
+    inline uint40_t& operator+=(std::uint32_t n) noexcept {
+        std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
+        octets[4] += static_cast<unsigned char>((tail + n) < tail);
+        tail += n;
+        return *this;
+    }
+
+    inline uint40_t& operator+=(std::size_t n) noexcept {
+        unsigned char* n_octets = reinterpret_cast<unsigned char*>(&n);
+        std::uint32_t& n_tail = *reinterpret_cast<std::uint32_t*>(n_octets);
+        std::uint32_t& tail = *reinterpret_cast<std::uint32_t*>(octets);
+        octets[4] += static_cast<unsigned char>((tail + n_tail) < tail);
+        tail += n_tail;
+        octets[4] += n_octets[4];
         return *this;
     }
 
+    inline uint40_t operator+(std::size_t n) noexcept {
+        uint40_t other(*this);
+        other += n;
+        return other;
+    }
+
     inline operator std::size_t() const noexcept {
         std::size_t result = 0;
-        std::memcpy((char*)&result + 3, octets, 5);
+        std::memcpy((char*)&result, octets, 5);
         return result;
     }
 
-    inline uint40_t& operator++() noexcept { return *this += 1; }
+    inline uint40_t& operator++() noexcept { return *this += 1u; }
 
     inline uint40_t operator++(int) noexcept {
         uint40_t old = *this;
-        *this += 1;
+        *this += 1u;
         return old;
     }
 };
@@ -1014,6 +1045,89 @@ class usearch_pack_m uint40_t {
 
 static_assert(sizeof(uint40_t) == 5, "uint40_t must be exactly 5 bytes");
 
+template <typename element_at, typename allocator_at = std::allocator<element_at>> //
+class ring_gt {
+  public:
+    using element_t = element_at;
+    using allocator_t = allocator_at;
+
+    static_assert(std::is_trivially_destructible<element_t>(), "This heap is designed for trivial structs");
+    static_assert(std::is_trivially_copy_constructible<element_t>(), "This heap is designed for trivial structs");
+
+    using value_type = element_t;
+
+  private:
+    element_t* elements_;
+    std::size_t capacity_;
+    std::size_t head_;
+    std::size_t tail_;
+    bool empty_;
+    allocator_t allocator_;
+
+  public:
+    explicit ring_gt(allocator_t const& alloc = allocator_t()) noexcept
+        : elements_(nullptr), capacity_(0), head_(0), tail_(0), empty_(true), allocator_(alloc) {}
+
+    ring_gt(ring_gt const&) = delete;
+    ring_gt& operator=(ring_gt const&) = delete;
+
+    ~ring_gt() noexcept { reset(); }
+
+    bool empty() const noexcept { return empty_; }
+    size_t size() const noexcept {
+        if (empty_)
+            return 0;
+        else if (head_ >= tail_)
+            return head_ - tail_;
+        else
+            return capacity_ - (tail_ - head_);
+    }
+
+    void reset() noexcept {
+        if (elements_)
+            allocator_.deallocate(elements_, capacity_);
+        elements_ = nullptr;
+        capacity_ = 0;
+        head_ = 0;
+        tail_ = 0;
+        empty_ = true;
+    }
+
+    bool resize(std::size_t n) noexcept {
+        element_t* elements = allocator_.allocate(n);
+        if (!elements)
+            return false;
+        reset();
+        elements_ = elements;
+        capacity_ = n;
+        return true;
+    }
+
+    void push(element_t const& value) noexcept {
+        elements_[head_] = value;
+        head_ = (head_ + 1) % capacity_;
+        empty_ = false;
+    }
+
+    bool try_push(element_t const& value) noexcept {
+        if (head_ == tail_ && !empty_)
+            return false; // elements_ is full
+
+        return push(value);
+        return true;
+    }
+
+    bool try_pop(element_t& value) noexcept {
+        if (empty_)
+            return false;
+
+        value = std::move(elements_[tail_]);
+        tail_ = (tail_ + 1) % capacity_;
+        empty_ = head_ == tail_;
+        return true;
+    }
+};
+
 /// @brief Number of neighbors per graph node.
 /// Defaults to 32 in FAISS and 16 in hnswlib.
 /// > It is called `M` in the paper.
@@ -1095,6 +1209,21 @@ struct search_config_t {
     bool exact = false;
 };
 
+struct copy_config_t {
+    bool copy_vectors = true;
+};
+
+struct join_config_t {
+    /// @brief Controls maximum number of proposals per man during stable marriage.
+    std::size_t max_proposals = 0;
+    /// @brief Hyper-parameter controlling the quality of search.
+    /// Defaults to 16 in FAISS and 10 in hnswlib.
+    /// > It is called `ef` in the paper.
+    std::size_t expansion = default_expansion_search();
+    /// @brief Brute-forces exhaustive search over all entries in the index.
+    bool exact = false;
+};
+
 using file_header_t = byte_t[64];
 
 /**
@@ -1199,8 +1328,8 @@ struct file_head_result_t {
     error_t error;
 
     explicit operator bool() const noexcept { return !error; }
-    file_head_result_t failed(char const* message) noexcept {
-        error = message;
+    file_head_result_t failed(error_t message) noexcept {
+        error = std::move(message);
         return std::move(*this);
     }
 };
@@ -1234,6 +1363,37 @@ struct viewed_file_t {
 };
 #endif
 
+struct dummy_predicate_t {
+    template <typename match_at> constexpr bool operator()(match_at&&) const noexcept { return true; }
+};
+
+struct dummy_progress_t {
+    inline void operator()(std::size_t /*progress*/, std::size_t /*total*/) const noexcept {}
+};
+
+struct dummy_executor_t {
+    dummy_executor_t() noexcept {}
+    std::size_t size() const noexcept { return 1; }
+
+    template <typename thread_aware_function_at>
+    void execute_bulk(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept {
+        for (std::size_t task_idx = 0; task_idx != tasks; ++task_idx)
+            thread_aware_function(0, task_idx);
+    }
+
+    template <typename thread_aware_function_at>
+    void execute_bulk(thread_aware_function_at&& thread_aware_function) noexcept {
+        thread_aware_function(0);
+    }
+};
+
+struct dummy_label_to_label_mapping_t {
+    struct member_ref_t {
+        template <typename label_at> member_ref_t& operator=(label_at&&) noexcept { return *this; }
+    };
+    template <typename label_at> member_ref_t operator[](label_at&&) const noexcept { return {}; }
+};
+
 /**
  *  @brief  Approximate Nearest Neighbors Search index using the
  *          Hierarchical Navigable Small World graph algorithm.
@@ -1317,11 +1477,13 @@ class index_gt {
     struct member_ref_t {
         misaligned_ref_gt<label_t> label;
         vector_view_t vector;
+        id_t id;
     };
 
     struct member_cref_t {
         label_t label;
         vector_view_t vector;
+        id_t id;
     };
 
     template <typename ref_at, typename index_at> class member_iterator_gt {
@@ -1341,7 +1503,11 @@ class index_gt {
         using pointer = void;
         using reference = ref_t;
 
-        reference operator*() const noexcept { return ref_t(index_->node_with_id_(offset_)); }
+        reference operator*() const noexcept {
+            node_t node = index_->node_with_id_(offset_);
+            return {node.label(), node.vector_view(), static_cast<id_t>(offset_)};
+        }
+
         member_iterator_gt operator++(int) noexcept { return member_iterator_gt(index_, offset_ + 1); }
         member_iterator_gt operator--(int) noexcept { return member_iterator_gt(index_, offset_ - 1); }
         member_iterator_gt operator+(difference_type d) noexcept { return member_iterator_gt(index_, offset_ + d); }
@@ -1453,9 +1619,9 @@ class index_gt {
         node_t(node_t const&) = default;
         node_t& operator=(node_t const&) = default;
 
-        label_t label() const noexcept { return misaligned_load<label_t>(tape_); }
-        dim_t dim() const noexcept { return misaligned_load<dim_t>(tape_ + sizeof(label_t)); }
-        level_t level() const noexcept { return misaligned_load<level_t>(tape_ + sizeof(label_t) + sizeof(dim_t)); }
+        misaligned_ref_gt<label_t> label() const noexcept { return {tape_}; }
+        misaligned_ref_gt<dim_t> dim() const noexcept { return {tape_ + sizeof(label_t)}; }
+        misaligned_ref_gt<level_t> level() const noexcept { return {tape_ + sizeof(label_t) + sizeof(dim_t)}; }
 
         void label(label_t v) noexcept { return misaligned_store<label_t>(tape_, v); }
         void dim(dim_t v) noexcept { return misaligned_store<dim_t>(tape_ + sizeof(label_t), v); }
@@ -1518,7 +1684,7 @@ class index_gt {
     index_config_t config_{};
     index_limits_t limits_{};
     metric_t metric_{};
-    allocator_t allocator_{};
+    mutable allocator_t allocator_{};
     point_allocator_t point_allocator_{};
     precomputed_constants_t pre_{};
     viewed_file_t viewed_file_{};
@@ -1572,6 +1738,35 @@ class index_gt {
         return *this;
     }
 
+    struct copy_result_t {
+        error_t error;
+        index_gt index;
+
+        explicit operator bool() const noexcept { return !error; }
+        copy_result_t failed(error_t message) noexcept {
+            error = std::move(message);
+            return std::move(*this);
+        }
+    };
+
+    copy_result_t copy(copy_config_t /*config*/ = {}) const noexcept {
+        copy_result_t result;
+        index_gt& other = result.index;
+        other = index_gt(config_, metric_, allocator_, point_allocator_);
+        if (!other.reserve(limits_))
+            return result.failed("Failed to reserve the contexts");
+
+        // Now all is left - is to allocate new `node_t` instances and populate
+        // the `other.nodes_` array into it.
+        for (std::size_t i = 0; i != size_; ++i)
+            other.nodes_[i] = other.node_make_copy_(node_bytes_split_(nodes_[i]));
+
+        other.size_ = size_.load();
+        other.max_level_ = max_level_;
+        other.entry_id_ = entry_id_;
+        return result;
+    }
+
     member_citerator_t cbegin() const noexcept { return {this, 0}; }
     member_citerator_t cend() const noexcept { return {this, size()}; }
     member_citerator_t begin() const noexcept { return {this, 0}; }
@@ -1605,8 +1800,11 @@ class index_gt {
 
         if (nodes_)
             node_allocator_t{}.deallocate(exchange(nodes_, nullptr), limits_.elements);
-        if (contexts_)
+        if (contexts_) {
+            for (std::size_t i = 0; i != limits_.threads(); ++i)
+                contexts_[i].~context_t();
             context_allocator_t{}.deallocate(exchange(contexts_, nullptr), limits_.threads());
+        }
         limits_ = index_limits_t{0, 0};
         capacity_ = 0;
         reset_view_();
@@ -1620,11 +1818,13 @@ class index_gt {
         std::swap(limits_, other.limits_);
         std::swap(metric_, other.metric_);
         std::swap(allocator_, other.allocator_);
+        std::swap(point_allocator_, other.point_allocator_);
         std::swap(pre_, other.pre_);
         std::swap(viewed_file_, other.viewed_file_);
         std::swap(max_level_, other.max_level_);
         std::swap(entry_id_, other.entry_id_);
         std::swap(nodes_, other.nodes_);
+        std::swap(nodes_mutexes_, other.nodes_mutexes_);
         std::swap(contexts_, other.contexts_);
 
         // Non-atomic parts.
@@ -1723,14 +1923,14 @@ class index_gt {
         id_t id{};
 
         explicit operator bool() const noexcept { return !error; }
-        add_result_t failed(char const* message) noexcept {
-            error = message;
+        add_result_t failed(error_t message) noexcept {
+            error = std::move(message);
             return std::move(*this);
         }
     };
 
     struct match_t {
-        member_cref_t element;
+        member_cref_t member;
         distance_t distance;
     };
 
@@ -1756,21 +1956,33 @@ class index_gt {
         inline search_result_t& operator=(search_result_t&&) = default;
 
         explicit operator bool() const noexcept { return !error; }
-        search_result_t failed(char const* message) noexcept {
-            error = message;
+        search_result_t failed(error_t message) noexcept {
+            error = std::move(message);
             return std::move(*this);
         }
 
         inline operator std::size_t() const noexcept { return count; }
         inline std::size_t size() const noexcept { return count; }
-        inline match_t operator[](std::size_t i) const noexcept {
+        inline bool empty() const noexcept { return !count; }
+        inline match_t operator[](std::size_t i) const noexcept { return at(i); }
+        inline match_t front() const noexcept { return at(0); }
+        inline match_t back() const noexcept { return at(count - 1); }
+        inline bool contains(label_t label) const noexcept {
+            for (std::size_t i = 0; i != count; ++i)
+                if (at(i).member.label == label)
+                    return true;
+            return false;
+        }
+        inline match_t at(std::size_t i) const noexcept {
             candidate_t const* top_ordered = top_.data();
-            return {member_cref_t(index_.node_with_id_(top_ordered[i].id)), top_ordered[i].distance};
+            candidate_t candidate = top_ordered[i];
+            node_t node = index_.node_with_id_(candidate.id);
+            return {member_cref_t{node.label(), node.vector_view(), candidate.id}, candidate.distance};
         }
         inline std::size_t dump_to(label_t* labels, distance_t* distances) const noexcept {
             for (std::size_t i = 0; i != count; ++i) {
                 match_t result = operator[](i);
-                labels[i] = result.element.label;
+                labels[i] = result.member.label;
                 distances[i] = result.distance;
             }
             return count;
@@ -1778,7 +1990,7 @@ class index_gt {
         inline std::size_t dump_to(label_t* labels) const noexcept {
             for (std::size_t i = 0; i != count; ++i) {
                 match_t result = operator[](i);
-                labels[i] = result.element.label;
+                labels[i] = result.member.label;
             }
             return count;
         }
@@ -1820,7 +2032,7 @@ class index_gt {
             new_level_lock.unlock();
 
         // Allocate the neighbors
-        node_t node = node_malloc_(label, vector, target_level, config.store_vector);
+        node_t node = node_make_(label, vector, target_level, config.store_vector);
         if (!node)
             return result.failed("Out of memory!");
         std::size_t old_size = size_.fetch_add(1);
@@ -1849,6 +2061,7 @@ class index_gt {
             // TODO: Handle out of memory conditions
             search_to_insert_(closest_id, vector, level, config.expansion, context);
             closest_id = connect_new_node_(new_id, level, context);
+            reconnect_neighbor_nodes_(new_id, level, context);
         }
 
         // Normalize stats
@@ -1869,9 +2082,13 @@ class index_gt {
      *  @param[in] query Contiguous range of scalars forming a vector view.
      *  @param[in] wanted The upper bound for the number of results to return.
      *  @param[in] config Configuration options for this specific operation.
+     *  @param[in] predicate Optional filtering predicate for `member_cref_t`.
      *  @return Smart object referencing temporary memory. Valid until next `search()` or `add()`.
      */
-    search_result_t search(vector_view_t query, std::size_t wanted, search_config_t config = {}) const noexcept {
+    template <typename predicate_at = dummy_predicate_t>
+    search_result_t search( //
+        vector_view_t query, std::size_t wanted, search_config_t config = {},
+        predicate_at&& predicate = dummy_predicate_t{}) const noexcept {
 
         context_t& context = contexts_[config.thread];
         top_candidates_t& top = context.top_candidates;
@@ -1886,7 +2103,7 @@ class index_gt {
         if (config.exact) {
             if (!top.reserve(wanted))
                 return result.failed("Out of memory!");
-            search_exact_(query, wanted, context);
+            search_exact_(query, wanted, context, std::forward<predicate_at>(predicate));
         } else {
             next_candidates_t& next = context.next_candidates;
             std::size_t expansion = (std::max)(config.expansion, wanted);
@@ -1897,7 +2114,7 @@ class index_gt {
 
             id_t closest_id = search_for_one_(entry_id_, query, max_level_, 0, context);
             // For bottom layer we need a more optimized procedure
-            if (!search_to_find_in_base_(closest_id, query, expansion, context))
+            if (!search_to_find_in_base_(closest_id, query, expansion, context, std::forward<predicate_at>(predicate)))
                 return result.failed("Out of memory!");
         }
 
@@ -1911,9 +2128,10 @@ class index_gt {
         return result;
     }
 
+    template <typename predicate_at = dummy_predicate_t>
     search_result_t search_around(                          //
         id_t hint, vector_view_t query, std::size_t wanted, //
-        search_config_t config = {}) const noexcept {
+        search_config_t config = {}, predicate_at&& predicate = dummy_predicate_t{}) const noexcept {
 
         context_t& context = contexts_[config.thread];
         top_candidates_t& top = context.top_candidates;
@@ -1933,7 +2151,7 @@ class index_gt {
         result.measurements = context.measurements_count;
         result.cycles = context.iteration_cycles;
 
-        search_to_find_in_base_(hint, query, expansion, context);
+        search_to_find_in_base_(hint, query, expansion, context, std::forward<predicate_at>(predicate));
         top.sort_ascending();
         top.shrink(wanted);
 
@@ -1944,6 +2162,10 @@ class index_gt {
         return result;
     }
 
+#pragma endregion
+
+#pragma region Metadata
+
     struct stats_t {
         std::size_t nodes;
         std::size_t edges;
@@ -1958,7 +2180,7 @@ class index_gt {
             node_t node = node_with_id_(i);
             std::size_t max_edges = node.level() * config_.connectivity + base_level_multiple_() * config_.connectivity;
             std::size_t edges = 0;
-            for (level_t level = 0; level != node.level(); ++level)
+            for (level_t level = 0; level <= node.level(); ++level)
                 edges += neighbors_(node, level).size();
 
             result.allocated_bytes += node_bytes_(node);
@@ -2007,6 +2229,8 @@ class index_gt {
         return total;
     }
 
+    std::size_t memory_usage_per_node(dim_t dim, level_t level) const noexcept { return node_bytes_(dim, level); }
+
     void change_metric(metric_t const& m) noexcept {
         metric_ = m;
         for (std::size_t i = 0; i != limits_.threads(); ++i)
@@ -2021,8 +2245,8 @@ class index_gt {
         error_t error;
 
         explicit operator bool() const noexcept { return !error; }
-        serialization_result_t failed(char const* message) noexcept {
-            error = message;
+        serialization_result_t failed(error_t message) noexcept {
+            error = std::move(message);
             return std::move(*this);
         }
     };
@@ -2032,7 +2256,8 @@ class index_gt {
      *          co-locating vectors and neighbors lists.
      *          Available on Linux, MacOS, Windows.
      */
-    serialization_result_t save(char const* file_path) const noexcept {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t save(char const* file_path, progress_at&& progress = {}) const noexcept {
 
         // Make sure we have right to write to that file
         serialization_result_t result;
@@ -2102,6 +2327,7 @@ class index_gt {
             write_chunk(node.vector(), node_vector_bytes);
             if (result.error)
                 return result;
+            progress(i, state.size);
         }
 
         std::fclose(file);
@@ -2113,7 +2339,8 @@ class index_gt {
      *          copying both vectors and neighbors lists into RAM.
      *          Available on Linux, MacOS, Windows.
      */
-    serialization_result_t load(char const* file_path) noexcept {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t load(char const* file_path, progress_at&& progress = {}) noexcept {
 
         serialization_result_t result;
         file_header_t state_buffer{};
@@ -2161,7 +2388,8 @@ class index_gt {
         }
 
         // Load nodes one by one
-        for (std::size_t i = 0; i != size_; ++i) {
+        std::size_t const size = size_;
+        for (std::size_t i = 0; i != size; ++i) {
             label_t label;
             dim_t dim;
             level_t level;
@@ -2176,11 +2404,15 @@ class index_gt {
                 return result;
 
             std::size_t node_bytes = node_bytes_(dim, level);
-            node_t node = node_malloc_(label, {nullptr, dim}, level, true);
+            node_t node = node_malloc_(dim, level);
+            node.label(label);
+            node.dim(dim);
+            node.level(level);
             read_chunk(node.tape() + node_head_bytes_(), node_bytes - node_head_bytes_());
             if (result.error)
                 return result;
             nodes_[i] = node;
+            progress(i, size);
         }
 
         std::fclose(file);
@@ -2193,7 +2425,8 @@ class index_gt {
      *          @b without copying the vectors and neighbors lists into RAM.
      *          Available on Linux, MacOS, but @b not on Windows.
      */
-    serialization_result_t view(char const* file_path) noexcept {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t view(char const* file_path, progress_at&& progress = {}) noexcept {
         serialization_result_t result;
 
 #if defined(USEARCH_DEFINED_WINDOWS)
@@ -2275,17 +2508,18 @@ class index_gt {
         }
 
         // Locate every node packed into file
-        std::size_t progress = sizeof(file_header_t);
-        for (std::size_t i = 0; i != size_; ++i) {
-            byte_t* tape = (byte_t*)(file + progress);
+        std::size_t progress_bytes = sizeof(file_header_t);
+        std::size_t const size = size_;
+        for (std::size_t i = 0; i != size; ++i) {
+            byte_t* tape = (byte_t*)(file + progress_bytes);
             dim_t dim = misaligned_load<dim_t>(tape + sizeof(label_t));
             level_t level = misaligned_load<level_t>(tape + sizeof(label_t) + sizeof(dim_t));
 
             std::size_t node_bytes = node_bytes_(dim, level);
             std::size_t node_vector_bytes = dim * sizeof(scalar_t);
             nodes_[i] = node_t{tape, (scalar_t*)(tape + node_bytes - node_vector_bytes)};
-            progress += node_bytes;
-            max_level_ = (std::max)(max_level_, level);
+            progress_bytes += node_bytes;
+            progress(i, size);
         }
 
         return {};
@@ -2293,7 +2527,224 @@ class index_gt {
 
 #pragma endregion
 
+    struct join_result_t {
+        error_t error{};
+        std::size_t intersection_size{};
+        std::size_t engagements{};
+        std::size_t cycles{};
+        std::size_t measurements{};
+
+        explicit operator bool() const noexcept { return !error; }
+        join_result_t failed(error_t message) noexcept {
+            error = std::move(message);
+            return std::move(*this);
+        }
+    };
+
+    /**
+     *  @brief  Adapts the Male-Optimal Stable Marriage algorithm for unequal sets
+     *          to perform fast one-to-one matching between two large collections
+     *          of vectors, using approximate nearest neighbors search.
+     */
+    template <                                                        //
+        typename first_to_second_at = dummy_label_to_label_mapping_t, //
+        typename second_to_first_at = dummy_label_to_label_mapping_t, //
+        typename executor_at = dummy_executor_t,                      //
+        typename progress_at = dummy_progress_t                       //
+        >
+    static join_result_t join(                                       //
+        index_gt const& first, index_gt const& second,               //
+        join_config_t config = {},                                   //
+        first_to_second_at&& first_to_second = first_to_second_at{}, //
+        second_to_first_at&& second_to_first = second_to_first_at{}, //
+        executor_at&& executor = executor_at{},                      //
+        progress_at&& progress = progress_at{}) noexcept {
+
+        if (second.size() < first.size())
+            return join_small_and_big_(                            //
+                second, first,                                     //
+                config,                                            //
+                std::forward<second_to_first_at>(second_to_first), //
+                std::forward<first_to_second_at>(first_to_second), //
+                std::forward<executor_at>(executor),               //
+                std::forward<progress_at>(progress));
+        else
+            return join_small_and_big_(                            //
+                first, second,                                     //
+                config,                                            //
+                std::forward<first_to_second_at>(first_to_second), //
+                std::forward<second_to_first_at>(second_to_first), //
+                std::forward<executor_at>(executor),               //
+                std::forward<progress_at>(progress));
+    }
+
   private:
+    template <typename first_to_second_at, typename second_to_first_at, typename executor_at, typename progress_at>
+    static join_result_t join_small_and_big_(       //
+        index_gt const& men, index_gt const& women, //
+        join_config_t config,                       //
+        first_to_second_at&& man_to_woman,          //
+        second_to_first_at&& woman_to_man,          //
+        executor_at&& executor,                     //
+        progress_at&& progress) noexcept {
+
+        join_result_t result;
+
+        // Sanity checks and argument validation:
+        if (&men == &women)
+            return result.failed("Can't join with itself, consider copying");
+        if (config.max_proposals == 0)
+            config.max_proposals = std::log(men.size()) + executor.size();
+        config.max_proposals = (std::min)(men.size(), config.max_proposals);
+
+        using proposals_count_t = std::uint16_t;
+        using ids_allocator_t = typename allocator_traits_t::template rebind_alloc<id_t>;
+        allocator_t& alloc = men.allocator_;
+
+        // Create an atomic queue, as a ring structure, from/to which
+        // free men will be added/pulled.
+        std::mutex free_men_mutex{};
+        ring_gt<id_t, ids_allocator_t> free_men;
+        free_men.resize(men.size());
+        for (std::size_t i = 0; i != men.size(); ++i)
+            free_men.push(static_cast<id_t>(i));
+
+        // We are gonna need some temporary memory.
+        proposals_count_t* proposal_counts = (proposals_count_t*)alloc.allocate(sizeof(proposals_count_t) * men.size());
+        id_t* man_to_woman_ids = (id_t*)alloc.allocate(sizeof(id_t) * men.size());
+        id_t* woman_to_man_ids = (id_t*)alloc.allocate(sizeof(id_t) * women.size());
+        if (!proposal_counts || !man_to_woman_ids || !woman_to_man_ids)
+            return result.failed("Can't temporary mappings");
+
+        id_t missing_id;
+        std::memset((void*)&missing_id, 0xFF, sizeof(id_t));
+        std::memset((void*)man_to_woman_ids, 0xFF, sizeof(id_t) * men.size());
+        std::memset((void*)woman_to_man_ids, 0xFF, sizeof(id_t) * women.size());
+        std::memset(proposal_counts, 0, sizeof(proposals_count_t) * men.size());
+
+        // Define locks, to limit concurrent accesses to `man_to_woman_ids` and `woman_to_man_ids`.
+        visits_bitset_t men_locks, women_locks;
+        if (!men_locks.resize(men.size()) || !women_locks.resize(women.size()))
+            return result.failed("Can't allocate locks");
+
+        // Accumulate statistics from all the contexts,
+        // to have a baseline to compare with, by the time the `join` is finished.
+        std::size_t old_measurements{};
+        std::size_t old_cycles{};
+        for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
+            old_measurements += women.contexts_[thread_idx].measurements_count;
+            old_cycles += women.contexts_[thread_idx].iteration_cycles;
+        }
+        std::atomic<std::size_t> rounds{0};
+        std::atomic<std::size_t> engagements{0};
+
+        // Concurrently process all the men
+        executor.execute_bulk([&](std::size_t thread_idx) {
+            context_t& context = women.contexts_[thread_idx];
+            search_config_t search_config;
+            search_config.expansion = config.expansion;
+            search_config.exact = config.exact;
+            search_config.thread = thread_idx;
+            id_t free_man_id;
+
+            // While there exist a free man who still has a woman to propose to.
+            while (true) {
+                std::size_t passed_rounds = 0;
+                std::size_t total_rounds = 0;
+                {
+                    std::unique_lock<std::mutex> pop_lock(free_men_mutex);
+                    if (!free_men.try_pop(free_man_id))
+                        // Primary exit path, we have exhausted the list of candidates
+                        break;
+                    passed_rounds = ++rounds;
+                    total_rounds = passed_rounds + free_men.size();
+                }
+                progress(passed_rounds, total_rounds);
+                while (men_locks.atomic_set(free_man_id))
+                    ;
+
+                node_t free_man = men.node_with_id_(free_man_id);
+                proposals_count_t& free_man_proposals = proposal_counts[free_man_id];
+                if (free_man_proposals >= config.max_proposals)
+                    continue;
+
+                // Find the closest woman, to whom this man hasn't proposed yet.
+                free_man_proposals++;
+                search_result_t candidates = women.search(free_man.vector_view(), free_man_proposals, search_config);
+                if (!candidates) {
+                    // TODO:
+                }
+
+                match_t match = candidates.back();
+                member_cref_t woman = match.member;
+                while (women_locks.atomic_set(woman.id))
+                    ;
+
+                id_t husband_id = woman_to_man_ids[woman.id];
+                bool woman_is_free = husband_id == missing_id;
+                if (woman_is_free) {
+                    // Engagement
+                    man_to_woman_ids[free_man_id] = woman.id;
+                    woman_to_man_ids[woman.id] = free_man_id;
+                    engagements++;
+                } else {
+                    distance_t distance_from_husband = context.measure(woman.vector, men.node_with_id_(husband_id));
+                    distance_t distance_from_candidate = match.distance;
+                    if (distance_from_husband > distance_from_candidate) {
+                        // Break-up
+                        while (men_locks.atomic_set(husband_id))
+                            ;
+                        man_to_woman_ids[husband_id] = missing_id;
+                        men_locks.atomic_reset(husband_id);
+
+                        // New Engagement
+                        man_to_woman_ids[free_man_id] = woman.id;
+                        woman_to_man_ids[woman.id] = free_man_id;
+                        engagements++;
+
+                        std::unique_lock<std::mutex> push_lock(free_men_mutex);
+                        free_men.push(husband_id);
+                    } else {
+                        std::unique_lock<std::mutex> push_lock(free_men_mutex);
+                        free_men.push(free_man_id);
+                    }
+                }
+
+                men_locks.atomic_reset(free_man_id);
+                women_locks.atomic_reset(woman.id);
+            }
+        });
+
+        // Export the IDs into labels:
+        std::size_t intersection_size = 0;
+        for (std::size_t i = 0; i != men.size(); ++i) {
+            id_t woman_id = man_to_woman_ids[i];
+            if (woman_id != missing_id) {
+                label_t man = men.node_with_id_(i).label();
+                label_t woman = women.node_with_id_(woman_id).label();
+                man_to_woman[man] = woman;
+                woman_to_man[woman] = man;
+                intersection_size++;
+            }
+        }
+
+        // Deallocate memory
+        alloc.deallocate((byte_t*)proposal_counts, sizeof(proposals_count_t) * men.size());
+        alloc.deallocate((byte_t*)man_to_woman_ids, sizeof(id_t) * men.size());
+        alloc.deallocate((byte_t*)woman_to_man_ids, sizeof(id_t) * women.size());
+
+        // Export stats
+        result.engagements = engagements;
+        result.intersection_size = intersection_size;
+        for (std::size_t thread_idx = 0; thread_idx != executor.size(); ++thread_idx) {
+            result.measurements += women.contexts_[thread_idx].measurements_count;
+            result.cycles += women.contexts_[thread_idx].iteration_cycles;
+        }
+        result.measurements -= old_measurements;
+        result.cycles -= old_cycles;
+        return result;
+    }
+
     void reset_view_() noexcept {
         if (!viewed_file_)
             return;
@@ -2323,51 +2774,80 @@ class index_gt {
         return node_head_bytes_() + pre_.neighbors_base_bytes + pre_.neighbors_bytes * level + sizeof(scalar_t) * dim;
     }
 
-    inline bool node_stored_(node_t node) const noexcept {
+    using span_bytes_t = span_gt<byte_t>;
+    struct node_bytes_split_t {
+        span_bytes_t tape{};
+        span_bytes_t vector{};
+
+        node_bytes_split_t() {}
+        node_bytes_split_t(span_bytes_t tape, span_bytes_t vector) noexcept : tape(tape), vector(vector) {}
+
+        std::size_t memory_usage() const noexcept { return tape.size() + vector.size(); }
+        bool colocated() const noexcept { return tape.end() == vector.begin(); }
+        operator node_t() const noexcept { return node_t{tape.begin(), reinterpret_cast<scalar_t*>(vector.begin())}; }
+    };
+
+    inline node_bytes_split_t node_bytes_split_(node_t node) const noexcept {
         std::size_t levels_bytes = pre_.neighbors_base_bytes + pre_.neighbors_bytes * node.level();
-        return (node.tape() + node_head_bytes_() + levels_bytes) == (byte_t*)node.vector();
+        std::size_t bytes_in_tape = node_head_bytes_() + levels_bytes;
+        return {{node.tape(), bytes_in_tape}, {(byte_t*)node.vector(), node_vector_bytes_(node)}};
     }
 
     inline std::size_t node_vector_bytes_(dim_t dim) const noexcept { return dim * sizeof(scalar_t); }
     inline std::size_t node_vector_bytes_(node_t node) const noexcept { return node_vector_bytes_(node.dim()); }
 
-    void node_free_(std::size_t id) noexcept {
-
-        if (viewed_file_)
-            return;
-
-        node_t& node = nodes_[id];
-        std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_stored_(node);
-        point_allocator_.deallocate(node.tape(), node_bytes);
-        node = node_t{};
-    }
-
-    node_t node_malloc_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
+    node_bytes_split_t node_malloc_(dim_t dims_to_store, level_t level) noexcept {
 
-        std::size_t dim = vector.size();
-        std::size_t stored_vector_bytes = node_vector_bytes_(static_cast<dim_t>(dim)) * std::size_t(store_vector);
-        std::size_t node_bytes = node_bytes_(static_cast<dim_t>(dim), level) -
-                                 node_vector_bytes_(static_cast<dim_t>(dim)) * std::size_t(!store_vector);
+        std::size_t vector_bytes = node_vector_bytes_(dims_to_store);
+        std::size_t node_bytes = node_bytes_(dims_to_store, level);
+        std::size_t non_vector_bytes = node_bytes - vector_bytes;
 
         byte_t* data = (byte_t*)point_allocator_.allocate(node_bytes);
         if (!data)
-            return {};
-
-        std::memset(data, 0, node_bytes);
-        if (vector.data())
-            std::memcpy(data + node_bytes - stored_vector_bytes, vector.data(), stored_vector_bytes);
-
-        scalar_t* scalars = store_vector //
-                                ? (scalar_t*)(data + node_bytes - stored_vector_bytes)
-                                : (scalar_t*)(vector.data());
+            return node_bytes_split_t{};
+        return {{data, non_vector_bytes}, {data + non_vector_bytes, vector_bytes}};
+    }
 
-        node_t node{data, scalars};
+    node_t node_make_(label_t label, vector_view_t vector, level_t level, bool store_vector) noexcept {
+        node_bytes_split_t node_bytes = node_malloc_(vector.size() * store_vector, level);
+        if (store_vector) {
+            std::memset(node_bytes.tape.data(), 0, node_bytes.tape.size());
+            std::memcpy(node_bytes.vector.data(), vector.data(), node_bytes.vector.size());
+        } else {
+            std::memset(node_bytes.tape.data(), 0, node_bytes.memory_usage());
+        }
+        node_t node = node_bytes;
         node.label(label);
-        node.dim(static_cast<dim_t>(dim));
+        node.dim(static_cast<dim_t>(vector.size()));
         node.level(level);
         return node;
     }
 
+    node_t node_make_copy_(node_bytes_split_t old_bytes) noexcept {
+        if (old_bytes.colocated()) {
+            byte_t* data = (byte_t*)point_allocator_.allocate(old_bytes.memory_usage());
+            std::memcpy(data, old_bytes.tape.data(), old_bytes.memory_usage());
+            return node_t{data, reinterpret_cast<scalar_t*>(data + old_bytes.tape.size())};
+        } else {
+            node_t old_node = old_bytes;
+            node_bytes_split_t node_bytes = node_malloc_(old_node.vector_view().size(), old_node.level());
+            std::memcpy(node_bytes.tape.data(), old_bytes.tape.data(), old_bytes.tape.size());
+            std::memcpy(node_bytes.vector.data(), old_bytes.vector.data(), old_bytes.vector.size());
+            return node_bytes;
+        }
+    }
+
+    void node_free_(std::size_t id) noexcept {
+
+        if (viewed_file_)
+            return;
+
+        node_t& node = nodes_[id];
+        std::size_t node_bytes = node_bytes_(node) - node_vector_bytes_(node) * !node_bytes_split_(node).colocated();
+        point_allocator_.deallocate(node.tape(), node_bytes);
+        node = node_t{};
+    }
+
     inline node_t node_with_id_(std::size_t idx) const noexcept { return nodes_[idx]; }
     inline neighbors_ref_t neighbors_base_(node_t node) const noexcept { return {node.neighbors_tape()}; }
 
@@ -2396,7 +2876,6 @@ class index_gt {
 
         node_t new_node = node_with_id_(new_id);
         top_candidates_t& top = context.top_candidates;
-        std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
 
         // Outgoing links from `new_id`:
         neighbors_ref_t new_neighbors = neighbors_(new_node, level);
@@ -2411,7 +2890,17 @@ class index_gt {
             }
         }
 
+        return new_neighbors[0];
+    }
+
+    void reconnect_neighbor_nodes_(id_t new_id, level_t level, context_t& context) usearch_noexcept_m {
+
+        node_t new_node = node_with_id_(new_id);
+        top_candidates_t& top = context.top_candidates;
+        neighbors_ref_t new_neighbors = neighbors_(new_node, level);
+
         // Reverse links from the neighbors:
+        std::size_t const connectivity_max = level ? config_.connectivity : pre_.connectivity_max_base;
         for (id_t close_id : new_neighbors) {
             node_t close_node = node_with_id_(close_id);
             node_lock_t close_lock = node_lock_(close_id);
@@ -2441,8 +2930,6 @@ class index_gt {
             for (std::size_t idx = 0; idx != top_view.size(); idx++)
                 close_header.push_back(top_view[idx].id);
         }
-
-        return new_neighbors[0];
     }
 
     level_t choose_random_level_(std::default_random_engine& level_generator) const noexcept {
@@ -2538,8 +3025,10 @@ class index_gt {
      *          Doesn't lock any nodes, assuming read-only simultaneous access.
      *  @return `true` if procedure succeeded, `false` if run out of memory.
      */
+    template <typename predicate_at>
     bool search_to_find_in_base_( //
-        id_t start_id, vector_view_t query, std::size_t expansion, context_t& context) const noexcept {
+        id_t start_id, vector_view_t query, std::size_t expansion, context_t& context,
+        predicate_at&& predicate) const noexcept {
 
         visits_bitset_t& visits = context.visits;
         next_candidates_t& next = context.next_candidates; // pop min, push
@@ -2573,14 +3062,19 @@ class index_gt {
                     continue;
 
                 visits.set(successor_id);
-                distance_t successor_dist = context.measure(query, node_with_id_(successor_id));
+                node_t successor = node_with_id_(successor_id);
+                distance_t successor_dist = context.measure(query, successor);
 
                 if (top.size() < top_limit || successor_dist < radius) {
                     // This can substantially grow our priority queue:
                     next.insert({-successor_dist, successor_id});
-                    // This will automatically evict poor matches:
-                    top.insert({successor_dist, successor_id}, top_limit);
-                    radius = top.top().distance;
+                    if (predicate( //
+                            match_t{member_cref_t{successor.label(), successor.vector_view(), successor_id},
+                                    successor_dist})) {
+                        // This will automatically evict poor matches:
+                        top.insert({successor_dist, successor_id}, top_limit);
+                        radius = top.top().distance;
+                    }
                 }
             }
         }
@@ -2591,12 +3085,21 @@ class index_gt {
     /**
      *  @brief  Iterates through all managed vectors, without actually touching the index.
      */
-    void search_exact_(vector_view_t query, std::size_t count, context_t& context) const noexcept {
+    template <typename predicate_at>
+    void search_exact_(                                             //
+        vector_view_t query, std::size_t count, context_t& context, //
+        predicate_at&& predicate) const noexcept {
+
         top_candidates_t& top = context.top_candidates;
         top.clear();
         top.reserve(count);
-        for (std::size_t i = 0; i != size(); ++i)
-            top.insert({context.measure(query, node_with_id_(i)), static_cast<id_t>(i)}, count);
+        for (std::size_t i = 0; i != size(); ++i) {
+            id_t id = static_cast<id_t>(i);
+            node_t node = node_with_id_(i);
+            distance_t distance = context.measure(query, node);
+            if (predicate(match_t{member_cref_t{node.label(), node.vector_view(), id}, distance}))
+                top.insert(candidate_t{distance, id}, count);
+        }
     }
 
     void prefetch_neighbors_(neighbors_ref_t, visits_bitset_t const&) const noexcept {}