usearch 1.2.2 → 2.0.0

package/include/usearch/index_plugins.hpp

@@ -54,8 +54,8 @@ struct uuid_t {
     std::uint8_t octets[16];
 };
 
-class i8_bits_t;
 class f16_bits_t;
+class i8_converted_t;
 
 #if USEARCH_USE_NATIVE_F16
 #if defined(USEARCH_DEFINED_ARM)
@@ -149,7 +149,7 @@ template <typename scalar_at> scalar_kind_t scalar_kind() noexcept {
         return scalar_kind_t::f32_k;
     if (std::is_same<scalar_at, f16_t>())
         return scalar_kind_t::f16_k;
-    if (std::is_same<scalar_at, i8_bits_t>())
+    if (std::is_same<scalar_at, i8_t>())
         return scalar_kind_t::i8_k;
     if (std::is_same<scalar_at, u64_t>())
         return scalar_kind_t::u64_k;
@@ -364,7 +364,7 @@ class f16_bits_t {
     inline operator float() const noexcept { return f16_to_f32(uint16_); }
     inline explicit operator bool() const noexcept { return f16_to_f32(uint16_) > 0.5f; }
 
-    inline f16_bits_t(i8_bits_t) noexcept;
+    inline f16_bits_t(i8_converted_t) noexcept;
     inline f16_bits_t(bool v) noexcept : uint16_(f32_to_f16(v)) {}
     inline f16_bits_t(float v) noexcept : uint16_(f32_to_f16(v)) {}
     inline f16_bits_t(double v) noexcept : uint16_(f32_to_f16(v)) {}
@@ -403,45 +403,6 @@ class f16_bits_t {
     }
 };
 
-/**
- *  @brief  Numeric type for uniformly-distributed floating point
- *          values within [-1,1] range, quantized to integers [-100,100].
- */
-class i8_bits_t {
-    std::int8_t int8_{};
-
-  public:
-    constexpr static float divisor_k = 100.f;
-    constexpr static std::int8_t min_k = -100;
-    constexpr static std::int8_t max_k = 100;
-
-    inline i8_bits_t() noexcept : int8_(0) {}
-    inline i8_bits_t(bool v) noexcept : int8_(v ? max_k : 0) {}
-
-    inline i8_bits_t(i8_bits_t&&) = default;
-    inline i8_bits_t& operator=(i8_bits_t&&) = default;
-    inline i8_bits_t(i8_bits_t const&) = default;
-    inline i8_bits_t& operator=(i8_bits_t const&) = default;
-
-    inline operator float() const noexcept { return float(int8_) / divisor_k; }
-    inline operator f16_t() const noexcept { return float(int8_) / divisor_k; }
-    inline operator double() const noexcept { return double(int8_) / divisor_k; }
-    inline explicit operator bool() const noexcept { return int8_ > (max_k / 2); }
-    inline explicit operator std::int8_t() const noexcept { return int8_; }
-    inline explicit operator std::int16_t() const noexcept { return int8_; }
-    inline explicit operator std::int32_t() const noexcept { return int8_; }
-    inline explicit operator std::int64_t() const noexcept { return int8_; }
-
-    inline i8_bits_t(f16_t v)
-        : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
-    inline i8_bits_t(float v)
-        : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
-    inline i8_bits_t(double v)
-        : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
-};
-
-inline f16_bits_t::f16_bits_t(i8_bits_t v) noexcept : f16_bits_t(float(v)) {}
-
 /**
  *  @brief  An STL-based executor or a "thread-pool" for parallel execution.
  *          Isn't efficient for small batches, as it recreates the threads on every call.
@@ -669,6 +630,34 @@ class aligned_allocator_gt {
 
 using aligned_allocator_t = aligned_allocator_gt<>;
 
+class page_allocator_t {
+  public:
+    static constexpr std::size_t page_size() { return 4096; }
+
+    /**
+     *  @brief Allocates an @b uninitialized block of memory of the specified size.
+     *  @param count_bytes The number of bytes to allocate.
+     *  @return A pointer to the allocated memory block, or `nullptr` if allocation fails.
+     */
+    byte_t* allocate(std::size_t count_bytes) const noexcept {
+        count_bytes = divide_round_up(count_bytes, page_size()) * page_size();
+#if defined(USEARCH_DEFINED_WINDOWS)
+        return (byte_t*)(::VirtualAlloc(NULL, count_bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
+#else
+        return (byte_t*)mmap(NULL, count_bytes, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+#endif
+    }
+
+    void deallocate(byte_t* page_pointer, std::size_t count_bytes) const noexcept {
+#if defined(USEARCH_DEFINED_WINDOWS)
+        ::VirtualFree(page_pointer, 0, MEM_RELEASE);
+#else
+        count_bytes = divide_round_up(count_bytes, page_size()) * page_size();
+        munmap(page_pointer, count_bytes);
+#endif
+    }
+};
+
 /**
  *  @brief  Memory-mapping allocator designed for "alloc many, free at once" usage patterns.
  *          @b Thread-safe, @b except constructors and destructors.
@@ -718,15 +707,11 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
     void reset() noexcept {
         byte_t* last_arena = last_arena_;
         while (last_arena) {
-            byte_t* previous_arena;
+            byte_t* previous_arena = nullptr;
             std::memcpy(&previous_arena, last_arena, sizeof(byte_t*));
-            std::size_t last_cap;
+            std::size_t last_cap = 0;
             std::memcpy(&last_cap, last_arena + sizeof(byte_t*), sizeof(std::size_t));
-#if defined(USEARCH_DEFINED_WINDOWS)
-            ::VirtualFree(last_arena, 0, MEM_RELEASE);
-#else
-            munmap(last_arena, last_cap);
-#endif
+            page_allocator_t{}.deallocate(last_arena, last_cap);
             last_arena = previous_arena;
         }
 
@@ -760,22 +745,12 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
      */
     inline byte_t* allocate(std::size_t count_bytes) noexcept {
         std::size_t extended_bytes = divide_round_up<alignment_ak>(count_bytes) * alignment_ak;
-        // Check if the requested allocation size is absurd
-        if (extended_bytes > min_capacity())
-            return nullptr;
-
         std::unique_lock<std::mutex> lock(mutex_);
-        if (!last_arena_ || (last_usage_ + extended_bytes > last_capacity_)) {
-            std::size_t new_cap = last_capacity_ * capacity_multiplier();
-#if defined(USEARCH_DEFINED_WINDOWS)
-            byte_t* new_arena = (byte_t*)(::VirtualAlloc(NULL, new_cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
-            if (new_arena == nullptr)
-                return nullptr;
-#else
-            byte_t* new_arena = (byte_t*)mmap(NULL, new_cap, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+        if (!last_arena_ || (last_usage_ + extended_bytes >= last_capacity_)) {
+            std::size_t new_cap = (std::max)(last_capacity_, ceil2(extended_bytes)) * capacity_multiplier();
+            byte_t* new_arena = page_allocator_t{}.allocate(new_cap);
             if (!new_arena)
                 return nullptr;
-#endif
             std::memcpy(new_arena, &last_arena_, sizeof(byte_t*));
             std::memcpy(new_arena + sizeof(byte_t*), &new_cap, sizeof(std::size_t));
 
@@ -939,7 +914,7 @@ template <> struct cast_gt<f16_bits_t, f16_bits_t> {
     bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
 };
 
-template <> struct cast_gt<i8_bits_t, i8_bits_t> {
+template <> struct cast_gt<i8_t, i8_t> {
     bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
 };
 
@@ -967,6 +942,53 @@ template <typename to_scalar_at> struct cast_gt<b1x8_t, to_scalar_at> {
     }
 };
 
+/**
+ *  @brief  Numeric type for uniformly-distributed floating point
+ *          values within [-1,1] range, quantized to integers [-100,100].
+ */
+class i8_converted_t {
+    std::int8_t int8_{};
+
+  public:
+    constexpr static float divisor_k = 100.f;
+    constexpr static std::int8_t min_k = -100;
+    constexpr static std::int8_t max_k = 100;
+
+    inline i8_converted_t() noexcept : int8_(0) {}
+    inline i8_converted_t(bool v) noexcept : int8_(v ? max_k : 0) {}
+
+    inline i8_converted_t(i8_converted_t&&) = default;
+    inline i8_converted_t& operator=(i8_converted_t&&) = default;
+    inline i8_converted_t(i8_converted_t const&) = default;
+    inline i8_converted_t& operator=(i8_converted_t const&) = default;
+
+    inline operator float() const noexcept { return float(int8_) / divisor_k; }
+    inline operator f16_t() const noexcept { return float(int8_) / divisor_k; }
+    inline operator double() const noexcept { return double(int8_) / divisor_k; }
+    inline explicit operator bool() const noexcept { return int8_ > (max_k / 2); }
+    inline explicit operator std::int8_t() const noexcept { return int8_; }
+    inline explicit operator std::int16_t() const noexcept { return int8_; }
+    inline explicit operator std::int32_t() const noexcept { return int8_; }
+    inline explicit operator std::int64_t() const noexcept { return int8_; }
+
+    inline i8_converted_t(f16_t v)
+        : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
+    inline i8_converted_t(float v)
+        : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
+    inline i8_converted_t(double v)
+        : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
+};
+
+f16_bits_t::f16_bits_t(i8_converted_t v) noexcept : uint16_(f32_to_f16(v)) {}
+
+template <> struct cast_gt<i8_t, f16_t> : public cast_gt<i8_converted_t, f16_t> {};
+template <> struct cast_gt<i8_t, f32_t> : public cast_gt<i8_converted_t, f32_t> {};
+template <> struct cast_gt<i8_t, f64_t> : public cast_gt<i8_converted_t, f64_t> {};
+
+template <> struct cast_gt<f16_t, i8_t> : public cast_gt<f16_t, i8_converted_t> {};
+template <> struct cast_gt<f32_t, i8_t> : public cast_gt<f32_t, i8_converted_t> {};
+template <> struct cast_gt<f64_t, i8_t> : public cast_gt<f64_t, i8_converted_t> {};
+
 /**
  *  @brief  Inner (Dot) Product distance.
  */
@@ -1191,10 +1213,10 @@ template <typename scalar_at = float, typename result_at = float> struct metric_
 };
 
 struct cos_i8_t {
-    using scalar_t = i8_bits_t;
+    using scalar_t = i8_t;
     using result_t = f32_t;
 
-    inline result_t operator()(i8_bits_t const* a, i8_bits_t const* b, std::size_t dim) const noexcept {
+    inline result_t operator()(i8_t const* a, i8_t const* b, std::size_t dim) const noexcept {
         std::int32_t ab{}, a2{}, b2{};
 #if USEARCH_USE_OPENMP
 #pragma omp simd reduction(+ : ab, a2, b2)
@@ -1215,10 +1237,10 @@ struct cos_i8_t {
 };
 
 struct l2sq_i8_t {
-    using scalar_t = i8_bits_t;
+    using scalar_t = i8_t;
     using result_t = f32_t;
 
-    inline result_t operator()(i8_bits_t const* a, i8_bits_t const* b, std::size_t dim) const noexcept {
+    inline result_t operator()(i8_t const* a, i8_t const* b, std::size_t dim) const noexcept {
         std::int32_t ab_deltas_sq{};
 #if USEARCH_USE_OPENMP
 #pragma omp simd reduction(+ : ab_deltas_sq)
@@ -1417,7 +1439,6 @@ class metric_punned_t {
     static metric_punned_t haversine_metric_(scalar_kind_t scalar_kind) {
         std::size_t bytes_per_vector = 2u * bits_per_scalar(scalar_kind) / CHAR_BIT;
         switch (scalar_kind) {
-        case scalar_kind_t::i8_k: return {to_stl_<metric_haversine_gt<i8_bits_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
         case scalar_kind_t::f16_k: return {to_stl_<metric_haversine_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
         case scalar_kind_t::f32_k: return {to_stl_<metric_haversine_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
         case scalar_kind_t::f64_k: return {to_stl_<metric_haversine_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
@@ -1427,7 +1448,7 @@ class metric_punned_t {
 
     static metric_punned_t pearson_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
         switch (scalar_kind) {
-        case scalar_kind_t::i8_k: return {to_stl_<metric_pearson_gt<i8_bits_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
+        case scalar_kind_t::i8_k: return {to_stl_<metric_pearson_gt<i8_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
         case scalar_kind_t::f16_k: return {to_stl_<metric_pearson_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
         case scalar_kind_t::f32_k: return {to_stl_<metric_pearson_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
         case scalar_kind_t::f64_k: return {to_stl_<metric_pearson_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "usearch",
-    "version": "1.2.2",
+    "version": "2.0.0",
     "description": "Smaller & Faster Single-File Vector Search Engine from Unum",
     "author": "Ash Vardanian",
     "license": "Apache 2.0",
@@ -25,7 +25,7 @@
         "@semantic-release/exec": "^6.0.3",
         "@semantic-release/git": "^10.0.1",
         "conventional-changelog-eslint": "^3.0.9",
-        "semantic-release": "^21.1.0",
+        "semantic-release": "^21.1.1",
         "typescript": "^5.1.6"
     }
 }

package/python/lib.cpp

@@ -144,7 +144,7 @@ metric_t wrap_user_defined_metric(
     case scalar_kind_t::b1x8_k:
         return wrap_typed_user_defined_metric<b1x8_t>(kind, signature, metric_uintptr, scalar_kind, dimensions);
     case scalar_kind_t::i8_k:
-        return wrap_typed_user_defined_metric<i8_bits_t>(kind, signature, metric_uintptr, scalar_kind, dimensions);
+        return wrap_typed_user_defined_metric<i8_t>(kind, signature, metric_uintptr, scalar_kind, dimensions);
     case scalar_kind_t::f16_k:
         return wrap_typed_user_defined_metric<f16_t>(kind, signature, metric_uintptr, scalar_kind, dimensions);
     case scalar_kind_t::f32_k:
@@ -273,7 +273,7 @@ static void add_many_to_index(                            //
     // clang-format off
     switch (numpy_string_to_kind(vectors_info.format)) {
     case scalar_kind_t::b1x8_k: add_typed_to_index<b1x8_t>(index, keys_info, vectors_info, force_copy, threads); break;
-    case scalar_kind_t::i8_k: add_typed_to_index<i8_bits_t>(index, keys_info, vectors_info, force_copy, threads); break;
+    case scalar_kind_t::i8_k: add_typed_to_index<i8_t>(index, keys_info, vectors_info, force_copy, threads); break;
     case scalar_kind_t::f16_k: add_typed_to_index<f16_t>(index, keys_info, vectors_info, force_copy, threads); break;
     case scalar_kind_t::f32_k: add_typed_to_index<f32_t>(index, keys_info, vectors_info, force_copy, threads); break;
     case scalar_kind_t::f64_k: add_typed_to_index<f64_t>(index, keys_info, vectors_info, force_copy, threads); break;
@@ -442,7 +442,7 @@ static py::tuple search_many_in_index( //
     // clang-format off
     switch (numpy_string_to_kind(vectors_info.format)) {
     case scalar_kind_t::b1x8_k: search_typed<b1x8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
-    case scalar_kind_t::i8_k: search_typed<i8_bits_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
+    case scalar_kind_t::i8_k: search_typed<i8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
     case scalar_kind_t::f16_k: search_typed<f16_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
     case scalar_kind_t::f32_k: search_typed<f32_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
     case scalar_kind_t::f64_k: search_typed<f64_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
@@ -561,7 +561,7 @@ static py::tuple search_many_brute_force(    //
     // clang-format off
     switch (dataset_kind) {
     case scalar_kind_t::b1x8_k: search_typed_brute_force<b1x8_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
-    case scalar_kind_t::i8_k: search_typed_brute_force<i8_bits_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
+    case scalar_kind_t::i8_k: search_typed_brute_force<i8_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
     case scalar_kind_t::f16_k: search_typed_brute_force<f16_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
     case scalar_kind_t::f32_k: search_typed_brute_force<f32_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
     case scalar_kind_t::f64_k: search_typed_brute_force<f64_t>(dataset_info, queries_info, wanted, threads, metric, keys_py, distances_py, counts_py); break;
@@ -638,7 +638,7 @@ static py::tuple cluster_vectors(        //
     // clang-format off
     switch (numpy_string_to_kind(queries_info.format)) {
     case scalar_kind_t::b1x8_k: cluster_result = index.cluster(queries_begin.as<b1x8_t const>(), queries_end.as<b1x8_t const>(), config, keys_ptr, distances_ptr, executor); break;
-    case scalar_kind_t::i8_k: cluster_result = index.cluster(queries_begin.as<i8_bits_t const>(), queries_end.as<i8_bits_t const>(), config, keys_ptr, distances_ptr, executor); break;
+    case scalar_kind_t::i8_k: cluster_result = index.cluster(queries_begin.as<i8_t const>(), queries_end.as<i8_t const>(), config, keys_ptr, distances_ptr, executor); break;
     case scalar_kind_t::f16_k: cluster_result = index.cluster(queries_begin.as<f16_t const>(), queries_end.as<f16_t const>(), config, keys_ptr, distances_ptr, executor); break;
     case scalar_kind_t::f32_k: cluster_result = index.cluster(queries_begin.as<f32_t const>(), queries_end.as<f32_t const>(), config, keys_ptr, distances_ptr, executor); break;
     case scalar_kind_t::f64_k: cluster_result = index.cluster(queries_begin.as<f64_t const>(), queries_end.as<f64_t const>(), config, keys_ptr, distances_ptr, executor); break;
@@ -759,17 +759,86 @@ static void compact_index(dense_index_py_t& index, std::size_t threads) {
     index.compact(executor_default_t{threads});
 }
 
+static py::dict index_metadata(index_dense_metadata_result_t const& meta) {
+    py::dict result;
+    result["matrix_included"] = !meta.config.exclude_vectors;
+    result["matrix_uses_64_bit_dimensions"] = meta.config.use_64_bit_dimensions;
+
+    index_dense_head_t const& head = meta.head;
+    result["version"] = std::to_string(head.version_major) + "." + //
+                        std::to_string(head.version_minor) + "." + //
+                        std::to_string(head.version_patch);
+
+    result["kind_metric"] = metric_kind_t(head.kind_metric);
+    result["kind_scalar"] = scalar_kind_t(head.kind_scalar);
+    result["kind_key"] = scalar_kind_t(head.kind_key);
+    result["kind_compressed_slot"] = scalar_kind_t(head.kind_compressed_slot);
+
+    result["count_present"] = std::uint64_t(head.count_present);
+    result["count_deleted"] = std::uint64_t(head.count_deleted);
+    result["dimensions"] = std::uint64_t(head.dimensions);
+
+    return result;
+}
+
 // clang-format off
-template <typename index_at> void save_index(index_at const& index, std::string const& path) { index.save(path.c_str()).error.raise(); }
-template <typename index_at> void load_index(index_at& index, std::string const& path) { index.load(path.c_str()).error.raise(); }
-template <typename index_at> void view_index(index_at& index, std::string const& path) { index.view(path.c_str()).error.raise(); }
+template <typename index_at> void save_index_to_path(index_at const& index, std::string const& path) { index.save(path.c_str()).error.raise(); }
+template <typename index_at> void load_index_from_path(index_at& index, std::string const& path) { index.load(path.c_str()).error.raise(); }
+template <typename index_at> void view_index_from_path(index_at& index, std::string const& path) { index.view(path.c_str()).error.raise(); }
 template <typename index_at> void reset_index(index_at& index) { index.reset(); }
 template <typename index_at> void clear_index(index_at& index) { index.clear(); }
 template <typename index_at> std::size_t max_level(index_at const &index) { return index.max_level(); }
+template <typename index_at> std::size_t serialized_length(index_at const &index) { return index.serialized_length(); }
 template <typename index_at> typename index_at::stats_t compute_stats(index_at const &index) { return index.stats(); }
 template <typename index_at> typename index_at::stats_t compute_level_stats(index_at const &index, std::size_t level) { return index.stats(level); }
 // clang-format on
 
+template <typename py_bytes_at> memory_mapped_file_t memory_map_from_bytes(py_bytes_at&& bytes) {
+    py::buffer_info info(py::buffer(bytes).request());
+    return {(byte_t*)(info.ptr), static_cast<std::size_t>(info.size)};
+}
+
+template <typename index_at> py::object save_index_to_buffer(index_at const& index) {
+    std::size_t serialized_length = index.serialized_length();
+
+    // Create an empty bytearray object using CPython API
+    PyObject* byte_array = PyByteArray_FromStringAndSize(nullptr, 0);
+    if (!byte_array)
+        throw std::runtime_error("Could not allocate bytearray object");
+
+    // Resize the bytearray object to the desired length
+    if (PyByteArray_Resize(byte_array, static_cast<Py_ssize_t>(serialized_length)) != 0) {
+        Py_XDECREF(byte_array);
+        throw std::runtime_error("Could not resize bytearray object");
+    }
+
+    char* buffer = PyByteArray_AS_STRING(byte_array);
+    memory_mapped_file_t memory_map((byte_t*)buffer, serialized_length);
+    serialization_result_t result = index.save(std::move(memory_map));
+
+    if (!result) {
+        Py_XDECREF(byte_array);
+        result.error.raise();
+    }
+
+    return py::reinterpret_steal<py::object>(byte_array);
+}
+
+template <typename index_at> void load_index_from_buffer(index_at& index, py::bytes const& buffer) {
+    index.load(memory_map_from_bytes(buffer)).error.raise();
+}
+template <typename index_at> void view_index_from_buffer(index_at& index, py::bytes const& buffer) {
+    index.view(memory_map_from_bytes(buffer)).error.raise();
+}
+
+template <typename index_at> std::vector<typename index_at::stats_t> compute_levels_stats(index_at const& index) {
+    using stats_t = typename index_at::stats_t;
+    std::size_t max_level = index.max_level();
+    std::vector<stats_t> result(max_level + 1);
+    index.stats(result.data(), max_level);
+    return result;
+}
+
 template <typename internal_at, typename external_at = internal_at, typename index_at = void>
 static py::tuple get_typed_vectors_for_keys(index_at const& index, py::buffer keys) {
 
@@ -807,7 +876,7 @@ template <typename index_at> py::tuple get_many(index_at const& index, py::buffe
     else if (scalar_kind == scalar_kind_t::f16_k)
         return get_typed_vectors_for_keys<f16_t, std::uint16_t>(index, keys);
     else if (scalar_kind == scalar_kind_t::i8_k)
-        return get_typed_vectors_for_keys<i8_bits_t, std::int8_t>(index, keys);
+        return get_typed_vectors_for_keys<i8_t, std::int8_t>(index, keys);
     else if (scalar_kind == scalar_kind_t::b1x8_k)
         return get_typed_vectors_for_keys<b1x8_t, std::uint8_t>(index, keys);
     else
@@ -865,30 +934,16 @@ PYBIND11_MODULE(compiled, m) {
         .value("I16", scalar_kind_t::i16_k)
         .value("I8", scalar_kind_t::i8_k);
 
-    m.def("index_dense_metadata", [](std::string const& path) -> py::dict {
-        index_dense_metadata_result_t meta = index_dense_metadata(path.c_str());
+    m.def("index_dense_metadata_from_path", [](std::string const& path) -> py::dict {
+        index_dense_metadata_result_t meta = index_dense_metadata_from_path(path.c_str());
         forward_error(meta);
+        return index_metadata(meta);
+    });
 
-        index_dense_head_t const& head = meta.head;
-
-        py::dict result;
-        result["matrix_included"] = !meta.config.exclude_vectors;
-        result["matrix_uses_64_bit_dimensions"] = meta.config.use_64_bit_dimensions;
-
-        result["version"] = std::to_string(head.version_major) + "." + //
-                            std::to_string(head.version_minor) + "." + //
-                            std::to_string(head.version_patch);
-
-        result["kind_metric"] = metric_kind_t(head.kind_metric);
-        result["kind_scalar"] = scalar_kind_t(head.kind_scalar);
-        result["kind_key"] = scalar_kind_t(head.kind_key);
-        result["kind_compressed_slot"] = scalar_kind_t(head.kind_compressed_slot);
-
-        result["count_present"] = std::uint64_t(head.count_present);
-        result["count_deleted"] = std::uint64_t(head.count_deleted);
-        result["dimensions"] = std::uint64_t(head.dimensions);
-
-        return result;
+    m.def("index_dense_metadata_from_buffer", [](py::bytes const& buffer) -> py::dict {
+        index_dense_metadata_result_t meta = index_dense_metadata_from_buffer(memory_map_from_bytes(buffer));
+        forward_error(meta);
+        return index_metadata(meta);
     });
 
     m.def("exact_search", &search_many_brute_force,                        //
@@ -1034,8 +1089,9 @@ PYBIND11_MODULE(compiled, m) {
                             [](dense_index_py_t const& index) -> std::size_t { return index.metric().dimensions(); });
     i.def_property_readonly( //
         "dtype", [](dense_index_py_t const& index) -> scalar_kind_t { return index.scalar_kind(); });
-    i.def_property_readonly( //
-        "memory_usage", [](dense_index_py_t const& index) -> std::size_t { return index.memory_usage(); });
+
+    i.def_property_readonly("serialized_length", &dense_index_py_t::serialized_length);
+    i.def_property_readonly("memory_usage", &dense_index_py_t::memory_usage);
 
     i.def_property("expansion_add", &dense_index_py_t::expansion_add, &dense_index_py_t::change_expansion_add);
     i.def_property("expansion_search", &dense_index_py_t::expansion_search, &dense_index_py_t::change_expansion_search);
@@ -1138,9 +1194,14 @@ PYBIND11_MODULE(compiled, m) {
         },
         py::arg("offset"));
 
-    i.def("save", &save_index<dense_index_py_t>, py::arg("path"));
-    i.def("load", &load_index<dense_index_py_t>, py::arg("path"));
-    i.def("view", &view_index<dense_index_py_t>, py::arg("path"));
+    i.def("save_index_to_path", &save_index_to_path<dense_index_py_t>);
+    i.def("load_index_from_path", &load_index_from_path<dense_index_py_t>);
+    i.def("view_index_from_path", &view_index_from_path<dense_index_py_t>);
+
+    i.def("save_index_to_buffer", &save_index_to_buffer<dense_index_py_t>);
+    i.def("load_index_from_buffer", &load_index_from_buffer<dense_index_py_t>);
+    i.def("view_index_from_buffer", &view_index_from_buffer<dense_index_py_t>);
+
     i.def("reset", &reset_index<dense_index_py_t>);
     i.def("clear", &clear_index<dense_index_py_t>);
     i.def("copy", &copy_index, py::kw_only(), py::arg("copy") = true);
@@ -1155,7 +1216,8 @@ PYBIND11_MODULE(compiled, m) {
     i_stats.def_readonly("allocated_bytes", &punned_index_stats_t::allocated_bytes);
 
     i.def_property_readonly("max_level", &max_level<dense_index_py_t>);
-    i.def_property_readonly("levels_stats", &compute_stats<dense_index_py_t>);
+    i.def_property_readonly("stats", &compute_stats<dense_index_py_t>);
+    i.def_property_readonly("levels_stats", &compute_levels_stats<dense_index_py_t>);
     i.def("level_stats", &compute_level_stats<dense_index_py_t>, py::arg("level"));
 
     auto is = py::class_<dense_indexes_py_t>(m, "Indexes");

package/python/scripts/test_index.py

@@ -126,35 +126,63 @@ def test_index_stats(batch_size):
     index.add(keys, vectors, threads=threads)
 
     assert index.max_level >= 0
-    assert index.levels_stats.nodes >= batch_size
+    assert index.stats.nodes >= batch_size
+    assert index.levels_stats[0].nodes == batch_size
     assert index.level_stats(0).nodes == batch_size
 
+    assert index.levels_stats[index.max_level].nodes > 0
+
 
 @pytest.mark.parametrize("ndim", [1, 3, 8, 32, 256, 4096])
-@pytest.mark.parametrize("batch_size", [1, 7, 1024])
+@pytest.mark.parametrize("batch_size", [0, 1, 7, 1024])
 @pytest.mark.parametrize("quantization", [ScalarKind.F32, ScalarKind.I8])
 def test_index_save_load_restore_copy(ndim, quantization, batch_size):
     index = Index(ndim=ndim, dtype=quantization, multi=False)
-    keys = np.arange(batch_size)
-    vectors = random_vectors(count=batch_size, ndim=ndim)
-    index.add(keys, vectors, threads=threads)
+
+    if batch_size > 0:
+        keys = np.arange(batch_size)
+        vectors = random_vectors(count=batch_size, ndim=ndim)
+        index.add(keys, vectors, threads=threads)
 
     index.save("tmp.usearch")
     index.clear()
     assert len(index) == 0
+    assert os.path.exists("tmp.usearch")
 
     index.load("tmp.usearch")
     assert len(index) == batch_size
-    assert len(index[0].flatten()) == ndim
+    if batch_size > 0:
+        assert len(index[0].flatten()) == ndim
 
-    index = Index.restore("tmp.usearch", view=True)
+    index_meta = Index.metadata("tmp.usearch")
+    assert index_meta is not None
+
+    index = Index.restore("tmp.usearch", view=False)
     assert len(index) == batch_size
-    assert len(index[0].flatten()) == ndim
+    if batch_size > 0:
+        assert len(index[0].flatten()) == ndim
 
     copied_index = index.copy()
     assert len(copied_index) == len(index)
-    assert np.allclose(np.vstack(copied_index.get(keys)), np.vstack(index.get(keys)))
-
+    if batch_size > 0:
+        assert np.allclose(
+            np.vstack(copied_index.get(keys)), np.vstack(index.get(keys))
+        )
+
+    # Perform the same operations in RAM, without touching the filesystem
+    serialized_index = index.save()
+    deserialized_metadata = Index.metadata(serialized_index)
+    assert deserialized_metadata is not None
+
+    deserialized_index = Index.restore(serialized_index)
+    assert len(deserialized_index) == len(index)
+    assert set(np.array(deserialized_index.keys)) == set(np.array(index.keys))
+    if batch_size > 0:
+        assert np.allclose(
+            np.vstack(deserialized_index.get(keys)), np.vstack(index.get(keys))
+        )
+
+    deserialized_index.reset()
     index.reset()
     os.remove("tmp.usearch")