usearch 2.2.1 → 2.3.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "2.2.1"
+version = "2.3.0"
 authors = ["Ash Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '2.2.1'
+    version = '2.3.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/cpp/bench.cpp

@@ -372,6 +372,7 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
                 executor, [&](std::size_t progress, std::size_t total) {
                     if (progress % 1000 == 0)
                         printer.print(progress, total);
+                    return true;
                 });
             join_attempts = result.visited_members;
         }

package/include/usearch/index.hpp

@@ -1212,9 +1212,10 @@ struct dummy_callback_t {
  *
  *  This is particularly helpful when handling long-running tasks, like serialization,
  *  saving, and loading from disk, or index-level joins.
+ *  The reporter checks return value to continue or stop the process, `false` means need to stop.
  */
 struct dummy_progress_t {
-    inline void operator()(std::size_t, std::size_t) const noexcept {}
+    inline bool operator()(std::size_t /*processed*/, std::size_t /*total*/) const noexcept { return true; }
 };
 
 /**
@@ -2690,6 +2691,10 @@ class index_gt {
         if (!output(&header, sizeof(header)))
             return result.failed("Failed to serialize the header into stream");
 
+        // Progress status
+        std::size_t processed = 0;
+        std::size_t const total = 2 * header.size;
+
         // Export the number of levels per node
         // That is both enough to estimate the overall memory consumption,
         // and to be able to estimate the offsets of every entry in the file.
@@ -2697,15 +2702,18 @@ class index_gt {
             node_t node = node_at_(i);
             level_t level = node.level();
             if (!output(&level, sizeof(level)))
-                return result.failed("Failed to serialize nodes levels into stream");
+                return result.failed("Failed to serialize into stream");
+            if (!progress(++processed, total))
+                return result.failed("Terminated by user");
         }
 
         // After that dump the nodes themselves
         for (std::size_t i = 0; i != header.size; ++i) {
             span_bytes_t node_bytes = node_bytes_(node_at_(i));
             if (!output(node_bytes.data(), node_bytes.size()))
-                return result.failed("Failed to serialize nodes into stream");
-            progress(i, header.size);
+                return result.failed("Failed to serialize into stream");
+            if (!progress(++processed, total))
+                return result.failed("Terminated by user");
         }
 
         return {};
@@ -2763,7 +2771,8 @@ class index_gt {
                 return result.failed("Failed to pull nodes from the stream");
             }
             nodes_[i] = node_t{node_bytes.data()};
-            progress(i, header.size);
+            if (!progress(i + 1, header.size))
+                return result.failed("Terminated by user");
         }
         return {};
     }
@@ -2936,7 +2945,8 @@ class index_gt {
         // Rapidly address all the nodes
         for (std::size_t i = 0; i != header.size; ++i) {
             nodes_[i] = node_t{(byte_t*)file.data() + offsets[i]};
-            progress(i, header.size);
+            if (!progress(i + 1, header.size))
+                return result.failed("Terminated by user");
         }
         viewed_file_ = std::move(file);
         return {};
@@ -2986,8 +2996,13 @@ class index_gt {
         using slot_level_allocator_t = typename dynamic_allocator_traits_t::template rebind_alloc<slot_level_t>;
         buffer_gt<slot_level_t, slot_level_allocator_t> slots_and_levels(size());
 
+        // Progress status
+        std::atomic<bool> do_tasks{true};
+        std::atomic<std::size_t> processed{0};
+        std::size_t const total = 3 * slots_and_levels.size();
+
         // For every bottom level node, determine its parent cluster
-        executor.fixed(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
+        executor.dynamic(slots_and_levels.size(), [&](std::size_t thread_idx, std::size_t old_slot) {
             context_t& context = contexts_[thread_idx];
             std::size_t cluster = search_for_one_( //
                 values[citerator_at(old_slot)],    //
@@ -2997,7 +3012,13 @@ class index_gt {
                                           static_cast<compressed_slot_t>(old_slot), //
                                           static_cast<compressed_slot_t>(cluster),  //
                                           node_at_(old_slot).level()};
+            ++processed;
+            if (thread_idx == 0)
+                do_tasks = progress(processed.load(), total);
+            return do_tasks.load();
         });
+        if (!do_tasks.load())
+            return;
 
         // Where the actual permutation happens:
         std::sort(slots_and_levels.begin(), slots_and_levels.end(), [](slot_level_t const& a, slot_level_t const& b) {
@@ -3027,8 +3048,8 @@ class index_gt {
                     neighbor = static_cast<compressed_slot_t>(old_slot_to_new[compressed_slot_t(neighbor)]);
 
             reordered_nodes[new_slot] = new_node;
-
-            progress(new_slot, slots_and_levels.size());
+            if (!progress(++processed, total))
+                return;
         }
 
         for (std::size_t new_slot = 0; new_slot != slots_and_levels.size(); ++new_slot) {
@@ -3036,6 +3057,8 @@ class index_gt {
             slot_transition(node_at_(old_slot).ckey(),                //
                             static_cast<compressed_slot_t>(old_slot), //
                             static_cast<compressed_slot_t>(new_slot));
+            if (!progress(++processed, total))
+                return;
         }
 
         nodes_ = std::move(reordered_nodes);
@@ -3064,9 +3087,13 @@ class index_gt {
         executor_at&& executor = executor_at{}, //
         progress_at&& progress = progress_at{}) noexcept {
 
+        // Progress status
+        std::atomic<bool> do_tasks{true};
+        std::atomic<std::size_t> processed{0};
+
         // Erase all the incoming links
         std::size_t nodes_count = size();
-        executor.fixed(nodes_count, [&](std::size_t, std::size_t node_idx) {
+        executor.dynamic(nodes_count, [&](std::size_t thread_idx, std::size_t node_idx) {
             node_t node = node_at_(node_idx);
             for (level_t level = 0; level <= node.level(); ++level) {
                 neighbors_ref_t neighbors = neighbors_(node, level);
@@ -3079,8 +3106,14 @@ class index_gt {
                         neighbors.push_back(neighbor_slot);
                 }
             }
-            progress(node_idx, nodes_count);
+            ++processed;
+            if (thread_idx == 0)
+                do_tasks = progress(processed.load(), nodes_count);
+            return do_tasks.load();
         });
+
+        // At the end report the latest numbers, because the reporter thread may be finished earlier
+        progress(processed.load(), nodes_count);
     }
 
   private:
@@ -3710,7 +3743,10 @@ static join_result_t join(               //
                 passed_rounds = ++rounds;
                 total_rounds = passed_rounds + free_men.size();
             }
-            progress(passed_rounds, total_rounds);
+            if (thread_idx == 0 && !progress(passed_rounds, total_rounds)) {
+                atomic_error.store("Terminated by user");
+                break;
+            }
             while (men_locks.atomic_set(free_man_slot))
                 ;
 

package/include/usearch/index_dense.hpp

@@ -767,7 +767,9 @@ class index_dense_gt {
      *  @brief  Saves serialized binary index representation to a stream.
      */
     template <typename output_callback_at, typename progress_at = dummy_progress_t>
-    serialization_result_t save_to_stream(output_callback_at&& output, serialization_config_t config = {}) const {
+    serialization_result_t save_to_stream(output_callback_at&& output,        //
+                                          serialization_config_t config = {}, //
+                                          progress_at&& progress = {}) const {
 
         serialization_result_t result;
         std::uint64_t matrix_rows = 0;
@@ -831,7 +833,7 @@ class index_dense_gt {
         }
 
         // Save the actual proximity graph
-        return typed_->save_to_stream(std::forward<output_callback_at>(output));
+        return typed_->save_to_stream(std::forward<output_callback_at>(output), std::forward<progress_at>(progress));
     }
 
     /**
@@ -853,8 +855,10 @@ class index_dense_gt {
      *  @param[in] config Configuration parameters for imports.
      *  @return Outcome descriptor explicitly convertible to boolean.
      */
-    template <typename input_callback_at>
-    serialization_result_t load_from_stream(input_callback_at&& input, serialization_config_t config = {}) {
+    template <typename input_callback_at, typename progress_at = dummy_progress_t>
+    serialization_result_t load_from_stream(input_callback_at&& input,          //
+                                            serialization_config_t config = {}, //
+                                            progress_at&& progress = {}) {
 
         // Discard all previous memory allocations of `vectors_tape_allocator_`
         reset();
@@ -915,7 +919,7 @@ class index_dense_gt {
         }
 
         // Pull the actual proximity graph
-        result = typed_->load_from_stream(std::forward<input_callback_at>(input));
+        result = typed_->load_from_stream(std::forward<input_callback_at>(input), std::forward<progress_at>(progress));
         if (!result)
             return result;
         if (typed_->size() != static_cast<std::size_t>(matrix_rows))
@@ -931,7 +935,10 @@ class index_dense_gt {
      *  @param[in] config Configuration parameters for imports.
      *  @return Outcome descriptor explicitly convertible to boolean.
      */
-    serialization_result_t view(memory_mapped_file_t file, std::size_t offset = 0, serialization_config_t config = {}) {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t view(memory_mapped_file_t file,                                  //
+                                std::size_t offset = 0, serialization_config_t config = {}, //
+                                progress_at&& progress = {}) {
 
         // Discard all previous memory allocations of `vectors_tape_allocator_`
         reset();
@@ -997,7 +1004,7 @@ class index_dense_gt {
         }
 
         // Pull the actual proximity graph
-        result = typed_->view(std::move(file), offset);
+        result = typed_->view(std::move(file), offset, std::forward<progress_at>(progress));
         if (!result)
             return result;
         if (typed_->size() != static_cast<std::size_t>(matrix_rows))
@@ -1019,7 +1026,9 @@ class index_dense_gt {
      *  @param[in] config Configuration parameters for exports.
      *  @return Outcome descriptor explicitly convertible to boolean.
      */
-    serialization_result_t save(output_file_t file, serialization_config_t config = {}) const {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t save(output_file_t file, serialization_config_t config = {},
+                                progress_at&& progress = {}) const {
 
         serialization_result_t io_result = file.open_if_not();
         if (!io_result)
@@ -1030,7 +1039,7 @@ class index_dense_gt {
                 io_result = file.write(buffer, length);
                 return !!io_result;
             },
-            config);
+            config, std::forward<progress_at>(progress));
 
         if (!stream_result)
             return stream_result;
@@ -1041,8 +1050,11 @@ class index_dense_gt {
      *  @brief  Memory-maps the serialized binary index representation from disk,
      *          @b without copying data into RAM, and fetching it on-demand.
      */
-    serialization_result_t save(memory_mapped_file_t file, std::size_t offset = 0,
-                                serialization_config_t config = {}) const {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t save(memory_mapped_file_t file,          //
+                                std::size_t offset = 0,             //
+                                serialization_config_t config = {}, //
+                                progress_at&& progress = {}) const {
 
         serialization_result_t io_result = file.open_if_not();
         if (!io_result)
@@ -1056,7 +1068,7 @@ class index_dense_gt {
                 offset += length;
                 return true;
             },
-            config);
+            config, std::forward<progress_at>(progress));
 
         return stream_result;
     }
@@ -1067,7 +1079,8 @@ class index_dense_gt {
      *  @param[in] config Configuration parameters for imports.
      *  @return Outcome descriptor explicitly convertible to boolean.
      */
-    serialization_result_t load(input_file_t file, serialization_config_t config = {}) {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t load(input_file_t file, serialization_config_t config = {}, progress_at&& progress = {}) {
 
         serialization_result_t io_result = file.open_if_not();
         if (!io_result)
@@ -1078,7 +1091,7 @@ class index_dense_gt {
                 io_result = file.read(buffer, length);
                 return !!io_result;
             },
-            config);
+            config, std::forward<progress_at>(progress));
 
         if (!stream_result)
             return stream_result;
@@ -1089,7 +1102,11 @@ class index_dense_gt {
      *  @brief  Memory-maps the serialized binary index representation from disk,
      *          @b without copying data into RAM, and fetching it on-demand.
      */
-    serialization_result_t load(memory_mapped_file_t file, std::size_t offset = 0, serialization_config_t config = {}) {
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t load(memory_mapped_file_t file,          //
+                                std::size_t offset = 0,             //
+                                serialization_config_t config = {}, //
+                                progress_at&& progress = {}) {
 
         serialization_result_t io_result = file.open_if_not();
         if (!io_result)
@@ -1103,17 +1120,23 @@ class index_dense_gt {
                 offset += length;
                 return true;
             },
-            config);
+            config, std::forward<progress_at>(progress));
 
         return stream_result;
     }
 
-    serialization_result_t save(char const* file_path, serialization_config_t config = {}) const {
-        return save(output_file_t(file_path), config);
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t save(char const* file_path,              //
+                                serialization_config_t config = {}, //
+                                progress_at&& progress = {}) const {
+        return save(output_file_t(file_path), config, std::forward<progress_at>(progress));
     }
 
-    serialization_result_t load(char const* file_path, serialization_config_t config = {}) {
-        return load(input_file_t(file_path), config);
+    template <typename progress_at = dummy_progress_t>
+    serialization_result_t load(char const* file_path,              //
+                                serialization_config_t config = {}, //
+                                progress_at&& progress = {}) {
+        return load(input_file_t(file_path), config, std::forward<progress_at>(progress));
     }
 
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "2.2.1",
+    "version": "2.3.0",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ash Vardanian",
     "license": "Apache 2.0",

package/python/lib.cpp

@@ -21,6 +21,7 @@
 
 #define _CRT_SECURE_NO_WARNINGS
 #define PY_SSIZE_T_CLEAN
+#include <pybind11/functional.h>
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
@@ -55,6 +56,18 @@ using dense_labeling_result_t = typename index_dense_t::labeling_result_t;
 using dense_cluster_result_t = typename index_dense_t::cluster_result_t;
 using dense_clustering_result_t = typename index_dense_t::clustering_result_t;
 
+using progress_func_t = std::function<bool(std::size_t /*processed*/, std::size_t /*total*/)>;
+
+struct progress_t {
+    inline progress_t(progress_func_t const& func) : func_(func ? func : &dummy_progress) {}
+    inline bool operator()(std::size_t processed, std::size_t total) const noexcept { return func_(processed, total); }
+
+  private:
+    static inline bool dummy_progress(std::size_t /*processed*/, std::size_t /*total*/) { return true; }
+
+    progress_func_t func_;
+};
+
 struct dense_index_py_t : public index_dense_t {
     using native_t = index_dense_t;
     using native_t::add;
@@ -208,13 +221,18 @@ template <typename scalar_at>
 static void add_typed_to_index(                                            //
     dense_index_py_t& index,                                               //
     py::buffer_info const& keys_info, py::buffer_info const& vectors_info, //
-    bool force_copy, std::size_t threads) {
+    bool force_copy, std::size_t threads,                                  //
+    progress_func_t const& progress) {
 
     Py_ssize_t vectors_count = vectors_info.shape[0];
     byte_t const* vectors_data = reinterpret_cast<byte_t const*>(vectors_info.ptr);
     byte_t const* keys_data = reinterpret_cast<byte_t const*>(keys_info.ptr);
     atomic_error_t atomic_error{nullptr};
 
+    // Progress status
+    progress_t progress_{progress};
+    std::atomic<std::size_t> processed{0};
+
     executor_default_t{threads}.dynamic(vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
         dense_key_t key = *reinterpret_cast<dense_key_t const*>(keys_data + task_idx * keys_info.strides[0]);
         scalar_at const* vector = reinterpret_cast<scalar_at const*>(vectors_data + task_idx * vectors_info.strides[0]);
@@ -225,12 +243,18 @@ static void add_typed_to_index(                                            //
         }
 
         // We don't want to check for signals from multiple threads
+        ++processed;
         if (thread_idx == 0)
-            if (PyErr_CheckSignals() != 0)
+            if (PyErr_CheckSignals() != 0 || !progress_(processed.load(), vectors_count)) {
+                atomic_error.store("Operation has been terminated");
                 return false;
+            }
         return true;
     });
 
+    // At the end report the latest numbers, because the reporter thread may be finished earlier
+    progress_(processed.load(), vectors_count);
+
     // Raise the error from a single thread
     auto error = atomic_error.load();
     if (error) {
@@ -242,7 +266,8 @@ static void add_typed_to_index(                                            //
 template <typename index_at>
 static void add_many_to_index(                            //
     index_at& index, py::buffer keys, py::buffer vectors, //
-    bool force_copy, std::size_t threads) {
+    bool force_copy, std::size_t threads,                 //
+    progress_func_t const& progress) {
 
     py::buffer_info keys_info = keys.request();
     py::buffer_info vectors_info = vectors.request();
@@ -272,11 +297,11 @@ static void add_many_to_index(                            //
 
     // clang-format off
     switch (numpy_string_to_kind(vectors_info.format)) {
-    case scalar_kind_t::b1x8_k: add_typed_to_index<b1x8_t>(index, keys_info, vectors_info, force_copy, threads); break;
-    case scalar_kind_t::i8_k: add_typed_to_index<i8_t>(index, keys_info, vectors_info, force_copy, threads); break;
-    case scalar_kind_t::f16_k: add_typed_to_index<f16_t>(index, keys_info, vectors_info, force_copy, threads); break;
-    case scalar_kind_t::f32_k: add_typed_to_index<f32_t>(index, keys_info, vectors_info, force_copy, threads); break;
-    case scalar_kind_t::f64_k: add_typed_to_index<f64_t>(index, keys_info, vectors_info, force_copy, threads); break;
+    case scalar_kind_t::b1x8_k: add_typed_to_index<b1x8_t>(index, keys_info, vectors_info, force_copy, threads, progress); break;
+    case scalar_kind_t::i8_k: add_typed_to_index<i8_t>(index, keys_info, vectors_info, force_copy, threads, progress); break;
+    case scalar_kind_t::f16_k: add_typed_to_index<f16_t>(index, keys_info, vectors_info, force_copy, threads, progress); break;
+    case scalar_kind_t::f32_k: add_typed_to_index<f32_t>(index, keys_info, vectors_info, force_copy, threads, progress); break;
+    case scalar_kind_t::f64_k: add_typed_to_index<f64_t>(index, keys_info, vectors_info, force_copy, threads, progress); break;
     default: throw std::invalid_argument("Incompatible scalars in the vectors matrix: " + vectors_info.format);
     }
     // clang-format on
@@ -287,7 +312,8 @@ static void search_typed(                                   //
     dense_index_py_t& index, py::buffer_info& vectors_info, //
     std::size_t wanted, bool exact, std::size_t threads,    //
     py::array_t<dense_key_t>& keys_py, py::array_t<distance_t>& distances_py, py::array_t<Py_ssize_t>& counts_py,
-    std::atomic<std::size_t>& stats_visited_members, std::atomic<std::size_t>& stats_computed_distances) {
+    std::atomic<std::size_t>& stats_visited_members, std::atomic<std::size_t>& stats_computed_distances,
+    progress_func_t const& progress) {
 
     auto keys_py2d = keys_py.template mutable_unchecked<2>();
     auto distances_py2d = distances_py.template mutable_unchecked<2>();
@@ -301,6 +327,10 @@ static void search_typed(                                   //
     if (!index.reserve(index_limits_t(index.size(), threads)))
         throw std::invalid_argument("Out of memory!");
 
+    // Progress status
+    progress_t progress_{progress};
+    std::atomic<std::size_t> processed{0};
+
     atomic_error_t atomic_error{nullptr};
     executor_default_t{threads}.dynamic(vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
         scalar_at const* vector = (scalar_at const*)(vectors_data + task_idx * vectors_info.strides[0]);
@@ -317,12 +347,18 @@ static void search_typed(                                   //
         stats_computed_distances += result.computed_distances;
 
         // We don't want to check for signals from multiple threads
+        ++processed;
         if (thread_idx == 0)
-            if (PyErr_CheckSignals() != 0)
+            if (PyErr_CheckSignals() != 0 || !progress_(processed.load(), vectors_count)) {
+                atomic_error.store("Operation has been terminated");
                 return false;
+            }
         return true;
     });
 
+    // At the end report the latest numbers, because the reporter thread may be finished earlier
+    progress_(processed.load(), vectors_count);
+
     // Raise the error from a single thread
     auto error = atomic_error.load();
     if (error) {
@@ -336,7 +372,8 @@ static void search_typed(                                       //
     dense_indexes_py_t& indexes, py::buffer_info& vectors_info, //
     std::size_t wanted, bool exact, std::size_t threads,        //
     py::array_t<dense_key_t>& keys_py, py::array_t<distance_t>& distances_py, py::array_t<Py_ssize_t>& counts_py,
-    std::atomic<std::size_t>& stats_visited_members, std::atomic<std::size_t>& stats_computed_distances) {
+    std::atomic<std::size_t>& stats_visited_members, std::atomic<std::size_t>& stats_computed_distances,
+    progress_func_t const& progress) {
 
     auto keys_py2d = keys_py.template mutable_unchecked<2>();
     auto distances_py2d = distances_py.template mutable_unchecked<2>();
@@ -354,6 +391,10 @@ static void search_typed(                                       //
     if (!query_mutexes)
         throw std::bad_alloc();
 
+    // Progress status
+    progress_t progress_{progress};
+    std::atomic<std::size_t> processed{0};
+
     atomic_error_t atomic_error{nullptr};
     executor_default_t{threads}.dynamic(indexes.shards_.size(), [&](std::size_t thread_idx, std::size_t task_idx) {
         dense_index_py_t& index = *indexes.shards_[task_idx].get();
@@ -388,13 +429,19 @@ static void search_typed(                                       //
             stats_computed_distances += result.computed_distances;
 
             // We don't want to check for signals from multiple threads
+            ++processed;
             if (thread_idx == 0)
-                if (PyErr_CheckSignals() != 0)
+                if (PyErr_CheckSignals() != 0 || !progress_(processed.load(), indexes.shards_.size())) {
+                    atomic_error.store("Operation has been terminated");
                     return false;
+                }
             return true;
         }
     });
 
+    // At the end report the latest numbers, because the reporter thread may be finished earlier
+    progress_(processed.load(), indexes.shards_.size());
+
     // Raise the error from a single thread
     auto error = atomic_error.load();
     if (error) {
@@ -416,7 +463,8 @@ static void search_typed(                                       //
  */
 template <typename index_at>
 static py::tuple search_many_in_index( //
-    index_at& index, py::buffer vectors, std::size_t wanted, bool exact, std::size_t threads) {
+    index_at& index, py::buffer vectors, std::size_t wanted, bool exact, std::size_t threads,
+    progress_func_t const& progress) {
 
     if (wanted == 0)
         return py::tuple(5);
@@ -441,11 +489,11 @@ static py::tuple search_many_in_index( //
 
     // clang-format off
     switch (numpy_string_to_kind(vectors_info.format)) {
-    case scalar_kind_t::b1x8_k: search_typed<b1x8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
-    case scalar_kind_t::i8_k: search_typed<i8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
-    case scalar_kind_t::f16_k: search_typed<f16_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
-    case scalar_kind_t::f32_k: search_typed<f32_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
-    case scalar_kind_t::f64_k: search_typed<f64_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
+    case scalar_kind_t::b1x8_k: search_typed<b1x8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances, progress); break;
+    case scalar_kind_t::i8_k: search_typed<i8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances, progress); break;
+    case scalar_kind_t::f16_k: search_typed<f16_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances, progress); break;
+    case scalar_kind_t::f32_k: search_typed<f32_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances, progress); break;
+    case scalar_kind_t::f64_k: search_typed<f64_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances, progress); break;
     default: throw std::invalid_argument("Incompatible scalars in the query matrix: " + vectors_info.format);
     }
     // clang-format on
@@ -463,7 +511,8 @@ template <typename scalar_at>
 static void search_typed_brute_force(                                //
     py::buffer_info& dataset_info, py::buffer_info& queries_info,    //
     std::size_t wanted, std::size_t threads, metric_t const& metric, //
-    py::array_t<dense_key_t>& keys_py, py::array_t<distance_t>& distances_py, py::array_t<Py_ssize_t>& counts_py) {
+    py::array_t<dense_key_t>& keys_py, py::array_t<distance_t>& distances_py, py::array_t<Py_ssize_t>& counts_py,
+    progress_func_t const& progress) {
 
     auto keys_py2d = keys_py.template mutable_unchecked<2>();
     auto distances_py2d = distances_py.template mutable_unchecked<2>();
@@ -485,6 +534,10 @@ static void search_typed_brute_force(                                //
     if (!query_mutexes)
         throw std::bad_alloc();
 
+    // Progress status
+    progress_t progress_{progress};
+    std::atomic<std::size_t> processed{0};
+
     executor_default_t{threads}.dynamic(tasks_count, [&](std::size_t thread_idx, std::size_t task_idx) {
         //
         std::size_t dataset_idx = task_idx / queries_count;
@@ -514,11 +567,15 @@ static void search_typed_brute_force(                                //
         }
 
         // We don't want to check for signals from multiple threads
+        ++processed;
         if (thread_idx == 0)
-            if (PyErr_CheckSignals() != 0)
+            if (PyErr_CheckSignals() != 0 || !progress_(processed.load(), tasks_count))
                 return false;
         return true;
     });
+
+    // At the end report the latest numbers, because the reporter thread may be finished earlier
+    progress_(processed.load(), tasks_count);
 }
 
 static py::tuple search_many_brute_force(    //
@@ -526,7 +583,8 @@ static py::tuple search_many_brute_force(    //
     std::size_t wanted, std::size_t threads, //
     metric_kind_t metric_kind,               //
     metric_signature_t metric_signature,     //
-    std::uintptr_t metric_uintptr) {
+    std::uintptr_t metric_uintptr,           //
+    progress_func_t const& progress) {
 
     if (wanted == 0)
         return py::tuple(5);
@@ -560,11 +618,11 @@ static py::tuple search_many_brute_force(    //
 
     // clang-format off
     switch (dataset_kind) {
-    case scalar_kind_t::b1x8_k: search_typed_brute_force<b1x8_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
-    case scalar_kind_t::i8_k: search_typed_brute_force<i8_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
-    case scalar_kind_t::f16_k: search_typed_brute_force<f16_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
-    case scalar_kind_t::f32_k: search_typed_brute_force<f32_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
-    case scalar_kind_t::f64_k: search_typed_brute_force<f64_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
+    case scalar_kind_t::b1x8_k: search_typed_brute_force<b1x8_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py, progress); break;
+    case scalar_kind_t::i8_k: search_typed_brute_force<i8_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py, progress); break;
+    case scalar_kind_t::f16_k: search_typed_brute_force<f16_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py, progress); break;
+    case scalar_kind_t::f32_k: search_typed_brute_force<f32_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py, progress); break;
+    case scalar_kind_t::f64_k: search_typed_brute_force<f64_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py, progress); break;
     default: throw std::invalid_argument("Incompatible vector types: " + dataset_info.format);
     }
     // clang-format on
@@ -603,7 +661,7 @@ template <typename scalar_at> struct rows_lookup_gt {
 template <typename index_at>
 static py::tuple cluster_vectors(        //
     index_at& index, py::buffer queries, //
-    std::size_t min_count, std::size_t max_count, std::size_t threads) {
+    std::size_t min_count, std::size_t max_count, std::size_t threads, progress_func_t const& progress) {
 
     if (index.limits().threads_search < threads)
         throw std::invalid_argument("Can't use that many threads!");
@@ -637,11 +695,11 @@ static py::tuple cluster_vectors(        //
 
     // clang-format off
     switch (numpy_string_to_kind(queries_info.format)) {
-    case scalar_kind_t::b1x8_k: cluster_result = index.cluster(queries_begin.as<b1x8_t const>(), queries_end.as<b1x8_t const>(), config, keys_ptr, distances_ptr, executor); break;
-    case scalar_kind_t::i8_k: cluster_result = index.cluster(queries_begin.as<i8_t const>(), queries_end.as<i8_t const>(), config, keys_ptr, distances_ptr, executor); break;
-    case scalar_kind_t::f16_k: cluster_result = index.cluster(queries_begin.as<f16_t const>(), queries_end.as<f16_t const>(), config, keys_ptr, distances_ptr, executor); break;
-    case scalar_kind_t::f32_k: cluster_result = index.cluster(queries_begin.as<f32_t const>(), queries_end.as<f32_t const>(), config, keys_ptr, distances_ptr, executor); break;
-    case scalar_kind_t::f64_k: cluster_result = index.cluster(queries_begin.as<f64_t const>(), queries_end.as<f64_t const>(), config, keys_ptr, distances_ptr, executor); break;
+    case scalar_kind_t::b1x8_k: cluster_result = index.cluster(queries_begin.as<b1x8_t const>(), queries_end.as<b1x8_t const>(), config, keys_ptr, distances_ptr, executor, progress_t{progress}); break;
+    case scalar_kind_t::i8_k: cluster_result = index.cluster(queries_begin.as<i8_t const>(), queries_end.as<i8_t const>(), config, keys_ptr, distances_ptr, executor, progress_t{progress}); break;
+    case scalar_kind_t::f16_k: cluster_result = index.cluster(queries_begin.as<f16_t const>(), queries_end.as<f16_t const>(), config, keys_ptr, distances_ptr, executor, progress_t{progress}); break;
+    case scalar_kind_t::f32_k: cluster_result = index.cluster(queries_begin.as<f32_t const>(), queries_end.as<f32_t const>(), config, keys_ptr, distances_ptr, executor, progress_t{progress}); break;
+    case scalar_kind_t::f64_k: cluster_result = index.cluster(queries_begin.as<f64_t const>(), queries_end.as<f64_t const>(), config, keys_ptr, distances_ptr, executor, progress_t{progress}); break;
     default: throw std::invalid_argument("Incompatible scalars in the query matrix: " + queries_info.format);
     }
     // clang-format on
@@ -677,7 +735,7 @@ static py::tuple cluster_vectors(        //
 template <typename index_at>
 static py::tuple cluster_keys(                            //
     index_at& index, py::array_t<dense_key_t> queries_py, //
-    std::size_t min_count, std::size_t max_count, std::size_t threads) {
+    std::size_t min_count, std::size_t max_count, std::size_t threads, progress_func_t const& progress) {
 
     if (index.limits().threads_search < threads)
         throw std::invalid_argument("Can't use that many threads!");
@@ -701,7 +759,7 @@ static py::tuple cluster_keys(                            //
     config.max_clusters = max_count;
 
     dense_clustering_result_t cluster_result =
-        index.cluster(queries_begin, queries_end, config, keys_ptr, distances_ptr, executor);
+        index.cluster(queries_begin, queries_end, config, keys_ptr, distances_ptr, executor, progress_t{progress});
     cluster_result.error.raise();
 
     // Those would be set to 1 for all entries, in case of success
@@ -721,7 +779,8 @@ static py::tuple cluster_keys(                            //
 
 static std::unordered_map<dense_key_t, dense_key_t> join_index( //
     dense_index_py_t const& a, dense_index_py_t const& b,       //
-    std::size_t max_proposals, bool exact) {
+    std::size_t max_proposals, bool exact,                      //
+    progress_func_t const& progress) {
 
     std::unordered_map<dense_key_t, dense_key_t> a_to_b;
     dummy_key_to_key_mapping_t b_to_a;
@@ -733,7 +792,7 @@ static std::unordered_map<dense_key_t, dense_key_t> join_index( //
     config.expansion = (std::max)(a.expansion_search(), b.expansion_search());
     std::size_t threads = (std::min)(a.limits().threads(), b.limits().threads());
     executor_default_t executor{threads};
-    join_result_t result = a.join(b, config, a_to_b, b_to_a, executor);
+    join_result_t result = a.join(b, config, a_to_b, b_to_a, executor, progress_t{progress});
     forward_error(result);
 
     return a_to_b;
@@ -749,14 +808,14 @@ static dense_index_py_t copy_index(dense_index_py_t const& index, bool force_cop
     return std::move(result.index);
 }
 
-static void compact_index(dense_index_py_t& index, std::size_t threads) {
+static void compact_index(dense_index_py_t& index, std::size_t threads, progress_func_t const& progress) {
 
     if (!threads)
         threads = std::thread::hardware_concurrency();
     if (!index.reserve(index_limits_t(index.size(), threads)))
         throw std::invalid_argument("Out of memory!");
 
-    index.compact(executor_default_t{threads});
+    index.compact(executor_default_t{threads}, progress_t{progress});
 }
 
 static py::dict index_metadata(index_dense_metadata_result_t const& meta) {
@@ -782,9 +841,9 @@ static py::dict index_metadata(index_dense_metadata_result_t const& meta) {
 }
 
 // clang-format off
-template <typename index_at> void save_index_to_path(index_at const& index, std::string const& path) { index.save(path.c_str()).error.raise(); }
-template <typename index_at> void load_index_from_path(index_at& index, std::string const& path) { index.load(path.c_str()).error.raise(); }
-template <typename index_at> void view_index_from_path(index_at& index, std::string const& path) { index.view(path.c_str()).error.raise(); }
+template <typename index_at> void save_index_to_path(index_at const& index, std::string const& path, progress_func_t const& progress) { index.save(path.c_str(), {}, progress_t{progress}).error.raise(); }
+template <typename index_at> void load_index_from_path(index_at& index, std::string const& path, progress_func_t const& progress) { index.load(path.c_str(), {}, progress_t{progress}).error.raise(); }
+template <typename index_at> void view_index_from_path(index_at& index, std::string const& path, progress_func_t const& progress) { index.view(path.c_str(), 0, {}, progress_t{progress}).error.raise(); }
 template <typename index_at> void reset_index(index_at& index) { index.reset(); }
 template <typename index_at> void clear_index(index_at& index) { index.clear(); }
 template <typename index_at> std::size_t max_level(index_at const &index) { return index.max_level(); }
@@ -798,7 +857,7 @@ template <typename py_bytes_at> memory_mapped_file_t memory_map_from_bytes(py_by
     return {(byte_t*)(info.ptr), static_cast<std::size_t>(info.size)};
 }
 
-template <typename index_at> py::object save_index_to_buffer(index_at const& index) {
+template <typename index_at> py::object save_index_to_buffer(index_at const& index, progress_func_t const& progress) {
     std::size_t serialized_length = index.serialized_length();
 
     // Create an empty bytearray object using CPython API
@@ -814,7 +873,7 @@ template <typename index_at> py::object save_index_to_buffer(index_at const& ind
 
     char* buffer = PyByteArray_AS_STRING(byte_array);
     memory_mapped_file_t memory_map((byte_t*)buffer, serialized_length);
-    serialization_result_t result = index.save(std::move(memory_map));
+    serialization_result_t result = index.save(std::move(memory_map), {}, {}, progress_t{progress});
 
     if (!result) {
         Py_XDECREF(byte_array);
@@ -824,11 +883,13 @@ template <typename index_at> py::object save_index_to_buffer(index_at const& ind
     return py::reinterpret_steal<py::object>(byte_array);
 }
 
-template <typename index_at> void load_index_from_buffer(index_at& index, py::bytes const& buffer) {
-    index.load(memory_map_from_bytes(buffer)).error.raise();
+template <typename index_at>
+void load_index_from_buffer(index_at& index, py::bytes const& buffer, progress_func_t const& progress) {
+    index.load(memory_map_from_bytes(buffer), {}, {}, progress_t{progress}).error.raise();
 }
-template <typename index_at> void view_index_from_buffer(index_at& index, py::bytes const& buffer) {
-    index.view(memory_map_from_bytes(buffer)).error.raise();
+template <typename index_at>
+void view_index_from_buffer(index_at& index, py::bytes const& buffer, progress_func_t const& progress) {
+    index.view(memory_map_from_bytes(buffer), {}, {}, progress_t{progress}).error.raise();
 }
 
 template <typename index_at> std::vector<typename index_at::stats_t> compute_levels_stats(index_at const& index) {
@@ -954,7 +1015,8 @@ PYBIND11_MODULE(compiled, m) {
           py::arg("threads") = 0,                                          //
           py::arg("metric_kind") = metric_kind_t::cos_k,                   //
           py::arg("metric_signature") = metric_signature_t::array_array_k, //
-          py::arg("metric_pointer") = 0                                    //
+          py::arg("metric_pointer") = 0,                                   //
+          py::arg("progress") = nullptr                                    //
     );
 
     m.def(
@@ -989,7 +1051,8 @@ PYBIND11_MODULE(compiled, m) {
         py::arg("vectors"),                               //
         py::kw_only(),                                    //
         py::arg("copy") = true,                           //
-        py::arg("threads") = 0                            //
+        py::arg("threads") = 0,                           //
+        py::arg("progress") = nullptr                     //
     );
 
     i.def(                                                      //
@@ -997,7 +1060,8 @@ PYBIND11_MODULE(compiled, m) {
         py::arg("queries"),                                     //
         py::arg("count") = 10,                                  //
         py::arg("exact") = false,                               //
-        py::arg("threads") = 0                                  //
+        py::arg("threads") = 0,                                 //
+        py::arg("progress") = nullptr                           //
     );
 
     i.def(                                                     //
@@ -1005,7 +1069,8 @@ PYBIND11_MODULE(compiled, m) {
         py::arg("queries"),                                    //
         py::arg("min_count") = 0,                              //
         py::arg("max_count") = 0,                              //
-        py::arg("threads") = 0                                 //
+        py::arg("threads") = 0,                                //
+        py::arg("progress") = nullptr                          //
     );
 
     i.def(                                               //
@@ -1013,7 +1078,8 @@ PYBIND11_MODULE(compiled, m) {
         py::arg("queries"),                              //
         py::arg("min_count") = 0,                        //
         py::arg("max_count") = 0,                        //
-        py::arg("threads") = 0                           //
+        py::arg("threads") = 0,                          //
+        py::arg("progress") = nullptr                    //
     );
 
     i.def(
@@ -1205,19 +1271,24 @@ PYBIND11_MODULE(compiled, m) {
         },
         py::arg("offset"));
 
-    i.def("save_index_to_path", &save_index_to_path<dense_index_py_t>);
-    i.def("load_index_from_path", &load_index_from_path<dense_index_py_t>);
-    i.def("view_index_from_path", &view_index_from_path<dense_index_py_t>);
+    i.def("save_index_to_path", &save_index_to_path<dense_index_py_t>, py::arg("path"), py::arg("progress") = nullptr);
+    i.def("load_index_from_path", &load_index_from_path<dense_index_py_t>, py::arg("path"),
+          py::arg("progress") = nullptr);
+    i.def("view_index_from_path", &view_index_from_path<dense_index_py_t>, py::arg("path"),
+          py::arg("progress") = nullptr);
 
-    i.def("save_index_to_buffer", &save_index_to_buffer<dense_index_py_t>);
-    i.def("load_index_from_buffer", &load_index_from_buffer<dense_index_py_t>);
-    i.def("view_index_from_buffer", &view_index_from_buffer<dense_index_py_t>);
+    i.def("save_index_to_buffer", &save_index_to_buffer<dense_index_py_t>, py::arg("progress") = nullptr);
+    i.def("load_index_from_buffer", &load_index_from_buffer<dense_index_py_t>, py::arg("buffer"),
+          py::arg("progress") = nullptr);
+    i.def("view_index_from_buffer", &view_index_from_buffer<dense_index_py_t>, py::arg("buffer"),
+          py::arg("progress") = nullptr);
 
     i.def("reset", &reset_index<dense_index_py_t>);
     i.def("clear", &clear_index<dense_index_py_t>);
     i.def("copy", &copy_index, py::kw_only(), py::arg("copy") = true);
-    i.def("compact", &compact_index);
-    i.def("join", &join_index, py::arg("other"), py::arg("max_proposals") = 0, py::arg("exact") = false);
+    i.def("compact", &compact_index, py::arg("threads"), py::arg("progress") = nullptr);
+    i.def("join", &join_index, py::arg("other"), py::arg("max_proposals") = 0, py::arg("exact") = false,
+          py::arg("progress") = nullptr);
 
     using punned_index_stats_t = typename dense_index_py_t::stats_t;
     auto i_stats = py::class_<punned_index_stats_t>(m, "IndexStats");
@@ -1242,6 +1313,7 @@ PYBIND11_MODULE(compiled, m) {
         py::arg("query"),                                         //
         py::arg("count") = 10,                                    //
         py::arg("exact") = false,                                 //
-        py::arg("threads") = 0                                    //
+        py::arg("threads") = 0,                                   //
+        py::arg("progress") = nullptr                             //
     );
 }

package/python/usearch/index.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+from inspect import signature
 
 # The purpose of this file is to provide Pythonic wrapper on top
 # the native precompiled CPython module. It improves compatibility
@@ -8,6 +9,7 @@ import os
 import math
 from dataclasses import dataclass
 from typing import (
+    Any,
     Optional,
     Union,
     NamedTuple,
@@ -72,6 +74,17 @@ NoneType = type(None)
 
 PathOrBuffer = Union[str, os.PathLike, bytes]
 
+ProgressCallback = Callable[[int, int], bool]
+
+
+def _match_signature(
+    func: Callable[[Any], Any], arg_types: List[type], ret_type: type
+) -> bool:
+    assert callable(func), "Not callable"
+    sig = signature(func)
+    param_types = [param.annotation for param in sig.parameters.values()]
+    return param_types == arg_types and sig.return_annotation == ret_type
+
 
 def _normalize_dtype(
     dtype,
@@ -153,12 +166,16 @@ def _search_in_compiled(
     vectors: np.ndarray,
     *,
     log: Union[str, bool],
-    batch_size: int,
+    progress: Optional[ProgressCallback],
     **kwargs,
 ) -> Union[Matches, BatchMatches]:
     #
     assert isinstance(vectors, np.ndarray), "Expects a NumPy array"
     assert vectors.ndim == 1 or vectors.ndim == 2, "Expects a matrix or vector"
+    assert not progress or _match_signature(
+        progress, [int, int], bool
+    ), "Invalid callback"
+
     if vectors.ndim == 1:
         vectors = vectors.reshape(1, len(vectors))
     count_vectors = vectors.shape[0]
@@ -168,42 +185,25 @@ def _search_in_compiled(
     ) -> Union[BatchMatches, Matches]:
         return batch_matches[0] if count_vectors == 1 else batch_matches
 
-    if log and batch_size == 0:
-        batch_size = int(math.ceil(count_vectors / 100))
-
-    if batch_size:
-        tasks = [
-            vectors[start_row : start_row + batch_size, :]
-            for start_row in range(0, count_vectors, batch_size)
-        ]
-        tasks_matches = []
+    # Create progress bar if needed
+    if log:
         name = log if isinstance(log, str) else "Search"
         pbar = tqdm(
-            tasks,
             desc=name,
             total=count_vectors,
             unit="vector",
-            disable=log is False,
         )
-        for vectors in tasks:
-            tuple_ = compiled_callable(vectors, **kwargs)
-            tasks_matches.append(BatchMatches(*tuple_))
-            pbar.update(vectors.shape[0])
 
-        pbar.close()
-        return distil_batch(
-            BatchMatches(
-                keys=np.vstack([m.keys for m in tasks_matches]),
-                distances=np.vstack([m.distances for m in tasks_matches]),
-                counts=np.concatenate([m.counts for m in tasks_matches], axis=None),
-                visited_members=sum([m.visited_members for m in tasks_matches]),
-                computed_distances=sum([m.computed_distances for m in tasks_matches]),
-            )
-        )
+        def update_progress_bar(processed: int, total: int) -> bool:
+            pbar.update(processed - pbar.n)
+            return progress if progress else True
 
+        tuple_ = compiled_callable(vectors, progress=update_progress_bar, **kwargs)
+        pbar.close()
     else:
         tuple_ = compiled_callable(vectors, **kwargs)
-        return distil_batch(BatchMatches(*tuple_))
+
+    return distil_batch(BatchMatches(*tuple_))
 
 
 def _add_to_compiled(
@@ -214,9 +214,13 @@ def _add_to_compiled(
     copy: bool,
     threads: int,
     log: Union[str, bool],
-    batch_size: int,
+    progress: Optional[ProgressCallback],
 ) -> Union[int, np.ndarray]:
+    #
     assert isinstance(vectors, np.ndarray), "Expects a NumPy array"
+    assert not progress or _match_signature(
+        progress, [int, int], bool
+    ), "Invalid callback"
     assert vectors.ndim == 1 or vectors.ndim == 2, "Expects a matrix or vector"
     if vectors.ndim == 1:
         vectors = vectors.reshape(1, len(vectors))
@@ -235,37 +239,29 @@ def _add_to_compiled(
 
     assert len(keys) == count_vectors
 
-    # If logging is requested, and batch size is undefined, set it to grow 1% at a time:
-    if log and batch_size == 0:
-        batch_size = int(math.ceil(count_vectors / 100))
-
-    # Split into batches and log progress, if needed
-    if batch_size:
-        keys = [
-            keys[start_row : start_row + batch_size]
-            for start_row in range(0, count_vectors, batch_size)
-        ]
-        vectors = [
-            vectors[start_row : start_row + batch_size, :]
-            for start_row in range(0, count_vectors, batch_size)
-        ]
-        tasks = zip(keys, vectors)
+    # Create progress bar if needed
+    if log:
         name = log if isinstance(log, str) else "Add"
         pbar = tqdm(
-            tasks,
             desc=name,
             total=count_vectors,
             unit="vector",
-            disable=log is False,
         )
-        for keys, vectors in tasks:
-            compiled.add_many(keys, vectors, copy=copy, threads=threads)
-            pbar.update(len(keys))
 
-        pbar.close()
+        def update_progress_bar(processed: int, total: int) -> bool:
+            pbar.update(processed - pbar.n)
+            return progress(processed, total) if progress else True
 
+        compiled.add_many(
+            keys,
+            vectors,
+            copy=copy,
+            threads=threads,
+            progress=update_progress_bar,
+        )
+        pbar.close()
     else:
-        compiled.add_many(keys, vectors, copy=copy, threads=threads)
+        compiled.add_many(keys, vectors, copy=copy, threads=threads, progress=progress)
 
     return keys
 
@@ -625,7 +621,7 @@ class Index:
         copy: bool = True,
         threads: int = 0,
         log: Union[str, bool] = False,
-        batch_size: int = 0,
+        progress: Optional[ProgressCallback] = None,
     ) -> Union[int, np.ndarray]:
         """Inserts one or move vectors into the index.
 
@@ -651,8 +647,8 @@ class Index:
         :type threads: int, defaults to 0
         :param log: Whether to print the progress bar
         :type log: Union[str, bool], defaults to False
-        :param batch_size: Number of vectors to process at once
-        :type batch_size: int, defaults to 0
+        :param progress: Callback to report stats of the progress and control it
+        :type progress: Optional[ProgressCallback], defaults to None
         :return: Inserted key or keys
         :type: Union[int, np.ndarray]
         """
@@ -663,7 +659,7 @@ class Index:
             copy=copy,
             threads=threads,
             log=log,
-            batch_size=batch_size,
+            progress=progress,
         )
 
     def search(
@@ -675,7 +671,7 @@ class Index:
         threads: int = 0,
         exact: bool = False,
         log: Union[str, bool] = False,
-        batch_size: int = 0,
+        progress: Optional[ProgressCallback] = None,
     ) -> Union[Matches, BatchMatches]:
         """
         Performs approximate nearest neighbors search for one or more queries.
@@ -690,8 +686,8 @@ class Index:
         :type exact: bool, defaults to False
         :param log: Whether to print the progress bar, default to False
         :type log: Union[str, bool], optional
-        :param batch_size: Number of vectors to process at once
-        :type batch_size: int, defaults to 0
+        :param progress: Callback to report stats of the progress and control it
+        :type progress: Optional[ProgressCallback], defaults to None
         :return: Matches for one or more queries
         :rtype: Union[Matches, BatchMatches]
         """
@@ -701,11 +697,11 @@ class Index:
             vectors,
             # Batch scheduling:
             log=log,
-            batch_size=batch_size,
             # Search constraints:
             count=count,
             exact=exact,
             threads=threads,
+            progress=progress,
         )
 
     def contains(self, keys: KeyOrKeysLike) -> Union[bool, np.ndarray]:
@@ -932,37 +928,57 @@ class Index:
         self._compiled.expansion_search = v
 
     def save(
-        self, path_or_buffer: Union[str, os.PathLike, NoneType] = None
+        self,
+        path_or_buffer: Union[str, os.PathLike, NoneType] = None,
+        progress: Optional[ProgressCallback] = None,
     ) -> Optional[bytes]:
+        assert not progress or _match_signature(
+            progress, [int, int], bool
+        ), "Invalid callback signature"
+
         path_or_buffer = path_or_buffer if path_or_buffer else self.path
         if path_or_buffer is None:
-            return self._compiled.save_index_to_buffer()
+            return self._compiled.save_index_to_buffer(progress)
         else:
-            self._compiled.save_index_to_path(os.fspath(path_or_buffer))
+            self._compiled.save_index_to_path(os.fspath(path_or_buffer), progress)
+
+    def load(
+        self,
+        path_or_buffer: Union[str, os.PathLike, bytes, NoneType] = None,
+        progress: Optional[ProgressCallback] = None,
+    ):
+        assert not progress or _match_signature(
+            progress, [int, int], bool
+        ), "Invalid callback signature"
 
-    def load(self, path_or_buffer: Union[str, os.PathLike, bytes, NoneType] = None):
         path_or_buffer = path_or_buffer if path_or_buffer else self.path
         if path_or_buffer is None:
             raise Exception("Define the source")
         if isinstance(path_or_buffer, bytearray):
             path_or_buffer = bytes(path_or_buffer)
         if isinstance(path_or_buffer, bytes):
-            self._compiled.load_index_from_buffer(path_or_buffer)
+            self._compiled.load_index_from_buffer(path_or_buffer, progress)
         else:
-            self._compiled.load_index_from_path(os.fspath(path_or_buffer))
+            self._compiled.load_index_from_path(os.fspath(path_or_buffer), progress)
 
     def view(
-        self, path_or_buffer: Union[str, os.PathLike, bytes, bytearray, NoneType] = None
+        self,
+        path_or_buffer: Union[str, os.PathLike, bytes, bytearray, NoneType] = None,
+        progress: Optional[ProgressCallback] = None,
     ):
+        assert not progress or _match_signature(
+            progress, [int, int], bool
+        ), "Invalid callback signature"
+
         path_or_buffer = path_or_buffer if path_or_buffer else self.path
         if path_or_buffer is None:
             raise Exception("Define the source")
         if isinstance(path_or_buffer, bytearray):
             path_or_buffer = bytes(path_or_buffer)
         if isinstance(path_or_buffer, bytes):
-            self._compiled.view_index_from_buffer(path_or_buffer)
+            self._compiled.view_index_from_buffer(path_or_buffer, progress)
         else:
-            self._compiled.view_index_from_path(os.fspath(path_or_buffer))
+            self._compiled.view_index_from_path(os.fspath(path_or_buffer), progress)
 
     def clear(self):
         """Erases all the vectors from the index, preserving the space for future insertions."""
@@ -993,6 +1009,7 @@ class Index:
         other: Index,
         max_proposals: int = 0,
         exact: bool = False,
+        progress: Optional[ProgressCallback] = None,
     ) -> Dict[Key, Key]:
         """Performs "Semantic Join" or pairwise matching between `self` & `other` index.
         Is different from `search`, as no collisions are allowed in resulting pairs.
@@ -1005,13 +1022,20 @@ class Index:
         :type max_proposals: int, optional
         :param exact: Controls if underlying `search` should be exact, defaults to False
         :type exact: bool, optional
+        :param progress: Callback to report stats of the progress and control it
+        :type progress: Optional[ProgressCallback], defaults to None
         :return: Mapping from keys of `self` to keys of `other`
         :rtype: Dict[Key, Key]
         """
+        assert not progress or _match_signature(
+            progress, [int, int], bool
+        ), "Invalid callback signature"
+
         return self._compiled.join(
             other=other._compiled,
             max_proposals=max_proposals,
             exact=exact,
+            progress=progress,
         )
 
     def cluster(
@@ -1023,7 +1047,7 @@ class Index:
         max_count: Optional[int] = None,
         threads: int = 0,
         log: Union[str, bool] = False,
-        batch_size: int = 0,
+        progress: Optional[ProgressCallback] = None,
     ) -> Clustering:
         """
         Clusters already indexed or provided `vectors`, mapping them to various centroids.
@@ -1037,11 +1061,15 @@ class Index:
         :type threads: int, defaults to 0
         :param log: Whether to print the progress bar
         :type log: Union[str, bool], defaults to False
-        :param batch_size: Number of vectors to process at once, defaults to 0
-        :type batch_size: int, defaults to 0
+        :param progress: Callback to report stats of the progress and control it
+        :type progress: Optional[ProgressCallback], defaults to None
         :return: Matches for one or more queries
         :rtype: Union[Matches, BatchMatches]
         """
+        assert not progress or _match_signature(
+            progress, [int, int], bool
+        ), "Invalid callback signature"
+
         if min_count is None:
             min_count = 0
         if max_count is None:
@@ -1054,6 +1082,7 @@ class Index:
                 min_count=min_count,
                 max_count=max_count,
                 threads=threads,
+                progress=progress,
             )
         else:
             if keys is None:
@@ -1066,6 +1095,7 @@ class Index:
                 min_count=min_count,
                 max_count=max_count,
                 threads=threads,
+                progress=progress,
             )
 
         batch_matches = BatchMatches(*results)
@@ -1231,17 +1261,18 @@ class Indexes:
         *,
         threads: int = 0,
         exact: bool = False,
+        progress: Optional[ProgressCallback] = None,
     ):
         return _search_in_compiled(
             self._compiled.search_many,
             vectors,
             # Batch scheduling:
             log=False,
-            batch_size=None,
             # Search constraints:
             count=count,
             exact=exact,
             threads=threads,
+            progress=progress,
         )
 
 
@@ -1254,7 +1285,7 @@ def search(
     exact: bool = False,
     threads: int = 0,
     log: Union[str, bool] = False,
-    batch_size: int = 0,
+    progress: Optional[ProgressCallback] = None,
 ) -> Union[Matches, BatchMatches]:
     """Shortcut for search, that can avoid index construction. Particularly useful for
     tiny datasets, where brute-force exact search works fast enough.
@@ -1279,12 +1310,14 @@ def search(
     :type exact: bool, optional
     :param log: Whether to print the progress bar, default to False
     :type log: Union[str, bool], optional
-    :param batch_size: Number of vectors to process at once, defaults to 0
-    :type batch_size: int, optional
-
+    :param progress: Callback to report stats of the progress and control it
+    :type progress: Optional[ProgressCallback], defaults to None
     :return: Matches for one or more queries
     :rtype: Union[Matches, BatchMatches]
     """
+    assert not progress or _match_signature(
+        progress, [int, int], bool
+    ), "Invalid callback signature"
     assert dataset.ndim == 2, "Dataset must be a matrix, with a vector in each row"
 
     if not exact:
@@ -1298,14 +1331,14 @@ def search(
             dataset,
             threads=threads,
             log=log,
-            batch_size=batch_size,
+            progress=progress,
         )
         return index.search(
             query,
             count,
             threads=threads,
             log=log,
-            batch_size=batch_size,
+            progress=progress,
         )
 
     metric = _normalize_metric(metric)
@@ -1329,8 +1362,8 @@ def search(
             dataset,
             query,
             metric_kind=metric_kind,
-            metric_pointer=metric_pointer,
             metric_signature=metric_signature,
+            metric_pointer=metric_pointer,
             **kwargs,
         )
 
@@ -1339,8 +1372,8 @@ def search(
         query,
         # Batch scheduling:
         log=log,
-        batch_size=batch_size,
         # Search constraints:
         count=count,
         threads=threads,
+        progress=progress,
     )

package/wasmer.toml

@@ -1,6 +1,6 @@
 [package]
 name="unum/usearch"
-version="2.2.1"
+version="2.3.0"
 description="Smaller & Faster Single-File Vector Search Engine from Unum"
 license="Apache-2.0"
 readme="README.md"