usearch 2.10.2 → 2.10.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +1 -1
  2. package/binding.gyp +101 -0
  3. package/c/CMakeLists.txt +27 -0
  4. package/c/README.md +34 -0
  5. package/c/lib.cpp +362 -0
  6. package/c/test.c +382 -0
  7. package/c/usearch.h +356 -0
  8. package/cmake/clipp.cmake +7 -0
  9. package/cmake/config.cmake.in +15 -0
  10. package/cmake/emscripten.cmake +8 -0
  11. package/cmake/jemalloc.cmake +62 -0
  12. package/cmake/pkg-config.pc.in +4 -0
  13. package/cmake/usearchConfigVersion.cmake.in +20 -0
  14. package/cpp/CMakeLists.txt +14 -0
  15. package/cpp/README.md +161 -0
  16. package/cpp/bench.cpp +616 -0
  17. package/cpp/test.cpp +324 -0
  18. package/fp16/CMakeLists.txt +176 -0
  19. package/fp16/LICENSE +11 -0
  20. package/fp16/README.md +20 -0
  21. package/fp16/bench/alt-element.cc +89 -0
  22. package/fp16/bench/from-alt-array.cc +175 -0
  23. package/fp16/bench/from-ieee-array.cc +399 -0
  24. package/fp16/bench/ieee-element.cc +272 -0
  25. package/fp16/bench/to-alt-array.cc +91 -0
  26. package/fp16/bench/to-ieee-array.cc +303 -0
  27. package/fp16/cmake/DownloadGoogleBenchmark.cmake +15 -0
  28. package/fp16/cmake/DownloadGoogleTest.cmake +15 -0
  29. package/fp16/cmake/DownloadPSimd.cmake +15 -0
  30. package/fp16/configure.py +65 -0
  31. package/fp16/confu.yaml +8 -0
  32. package/fp16/include/fp16/__init__.py +0 -0
  33. package/fp16/include/fp16/avx.py +50 -0
  34. package/fp16/include/fp16/avx2.py +53 -0
  35. package/fp16/include/fp16/bitcasts.h +92 -0
  36. package/fp16/include/fp16/fp16.h +451 -0
  37. package/fp16/include/fp16/psimd.h +131 -0
  38. package/fp16/include/fp16.h +11 -0
  39. package/fp16/jni/Android.mk +6 -0
  40. package/fp16/jni/Application.mk +4 -0
  41. package/fp16/test/alt-from-fp32-value.cc +498 -0
  42. package/fp16/test/alt-to-fp32-bits.cc +263 -0
  43. package/fp16/test/alt-to-fp32-psimd.cc +145 -0
  44. package/fp16/test/alt-to-fp32-value.cc +348 -0
  45. package/fp16/test/alt-to-fp32x2-psimd.cc +245 -0
  46. package/fp16/test/bitcasts.cc +56 -0
  47. package/fp16/test/ieee-from-fp32-value.cc +523 -0
  48. package/fp16/test/ieee-to-fp32-bits.cc +333 -0
  49. package/fp16/test/ieee-to-fp32-psimd.cc +312 -0
  50. package/fp16/test/ieee-to-fp32-value.cc +420 -0
  51. package/fp16/test/ieee-to-fp32x2-psimd.cc +541 -0
  52. package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +96 -0
  53. package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +96 -0
  54. package/fp16/test/peachpy/stubs.py +38 -0
  55. package/fp16/test/tables.cc +530 -0
  56. package/fp16/test/tables.h +13 -0
  57. package/fp16/third-party/THHalf.h +92 -0
  58. package/fp16/third-party/eigen-half.h +170 -0
  59. package/fp16/third-party/float16-compressor.h +80 -0
  60. package/fp16/third-party/half.hpp +2916 -0
  61. package/fp16/third-party/npy-halffloat.h +328 -0
  62. package/include/usearch/index.hpp +3838 -0
  63. package/include/usearch/index_dense.hpp +2004 -0
  64. package/include/usearch/index_plugins.hpp +2255 -0
  65. package/javascript/dist-package-cjs.json +3 -0
  66. package/javascript/dist-package-esm.json +3 -0
  67. package/javascript/lib.cpp +350 -0
  68. package/javascript/node-gyp-build.d.ts +1 -0
  69. package/javascript/tsconfig-base.json +25 -0
  70. package/javascript/tsconfig-cjs.json +8 -0
  71. package/javascript/tsconfig-esm.json +8 -0
  72. package/javascript/usearch.test.js +84 -0
  73. package/javascript/usearch.ts +640 -0
  74. package/package.json +10 -3
  75. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  76. package/prebuilds/linux-arm64/usearch.node +0 -0
  77. package/prebuilds/linux-x64/usearch.node +0 -0
  78. package/prebuilds/win32-ia32/usearch.node +0 -0
  79. package/prebuilds/win32-x64/usearch.node +0 -0
  80. package/simsimd/.releaserc +87 -0
  81. package/simsimd/CONTRIBUTING.md +114 -0
  82. package/simsimd/Cargo.lock +709 -0
  83. package/simsimd/Cargo.toml +44 -0
  84. package/simsimd/LICENSE +201 -0
  85. package/simsimd/README.md +708 -0
  86. package/simsimd/binding.gyp +36 -0
  87. package/simsimd/build.rs +51 -0
  88. package/simsimd/c/lib.c +110 -0
  89. package/simsimd/include/simsimd/binary.h +258 -0
  90. package/simsimd/include/simsimd/dot.h +1237 -0
  91. package/simsimd/include/simsimd/geospatial.h +37 -0
  92. package/simsimd/include/simsimd/probability.h +563 -0
  93. package/simsimd/include/simsimd/simsimd.h +1129 -0
  94. package/simsimd/include/simsimd/spatial.h +1019 -0
  95. package/simsimd/include/simsimd/types.h +309 -0
  96. package/simsimd/javascript/bench.js +127 -0
  97. package/simsimd/javascript/dist-package-cjs.json +3 -0
  98. package/simsimd/javascript/dist-package-esm.json +3 -0
  99. package/simsimd/javascript/fallback.ts +203 -0
  100. package/simsimd/javascript/lib.c +107 -0
  101. package/simsimd/javascript/node-gyp-build.d.ts +1 -0
  102. package/simsimd/javascript/simsimd.ts +135 -0
  103. package/simsimd/javascript/test.mjs +179 -0
  104. package/simsimd/javascript/tsconfig-base.json +37 -0
  105. package/simsimd/javascript/tsconfig-cjs.json +8 -0
  106. package/simsimd/javascript/tsconfig-esm.json +8 -0
  107. package/simsimd/package-lock.json +1709 -0
  108. package/simsimd/package.json +61 -0
  109. package/simsimd/rust/benches/cosine.rs +27 -0
  110. package/simsimd/rust/benches/native.rs +30 -0
  111. package/simsimd/rust/benches/sqeuclidean.rs +27 -0
  112. package/simsimd/rust/lib.rs +778 -0
@@ -0,0 +1,2255 @@
1
+ #pragma once
2
+ #define __STDC_WANT_IEC_60559_TYPES_EXT__
3
+ #include <float.h> // `_Float16`
4
+ #include <stdlib.h> // `aligned_alloc`
5
+
6
+ #include <cstring> // `std::strncmp`
7
+ #include <numeric> // `std::iota`
8
+ #include <thread> // `std::thread`
9
+ #include <vector> // `std::vector`
10
+
11
+ #include <atomic> // `std::atomic`
12
+ #include <thread> // `std::thread`
13
+
14
+ #include <usearch/index.hpp> // `expected_gt` and macros
15
+
16
+ #if !defined(USEARCH_USE_OPENMP)
17
+ #define USEARCH_USE_OPENMP 0
18
+ #endif
19
+
20
+ #if USEARCH_USE_OPENMP
21
+ #include <omp.h> // `omp_get_num_threads()`
22
+ #endif
23
+
24
+ #if defined(USEARCH_DEFINED_LINUX)
25
+ #include <sys/auxv.h> // `getauxval()`
26
+ #endif
27
+
28
+ #if !defined(USEARCH_USE_FP16LIB)
29
+ #if defined(__AVX512F__)
30
+ #define USEARCH_USE_FP16LIB 0
31
+ #elif defined(USEARCH_DEFINED_ARM)
32
+ #include <arm_fp16.h> // `__fp16`
33
+ #define USEARCH_USE_FP16LIB 0
34
+ #else
35
+ #define USEARCH_USE_FP16LIB 1
36
+ #endif
37
+ #endif
38
+
39
+ #if USEARCH_USE_FP16LIB
40
+ #include <fp16/fp16.h>
41
+ #endif
42
+
43
+ #if !defined(USEARCH_USE_SIMSIMD)
44
+ #define USEARCH_USE_SIMSIMD 0
45
+ #endif
46
+
47
+ #if USEARCH_USE_SIMSIMD
48
+ // Propagate the `f16` settings
49
+ #define SIMSIMD_NATIVE_F16 !USEARCH_USE_FP16LIB
50
+ #define SIMSIMD_DYNAMIC_DISPATCH 0
51
+ // No problem, if some of the functions are unused or undefined
52
+ #pragma GCC diagnostic push
53
+ #pragma GCC diagnostic ignored "-Wunused-function"
54
+ #pragma warning(push)
55
+ #pragma warning(disable : 4101)
56
+ #include <simsimd/simsimd.h>
57
+ #pragma warning(pop)
58
+ #pragma GCC diagnostic pop
59
+ #endif
60
+
61
+ namespace unum {
62
+ namespace usearch {
63
+
64
+ using u40_t = uint40_t;
65
+ enum b1x8_t : unsigned char {};
66
+
67
+ struct uuid_t {
68
+ std::uint8_t octets[16];
69
+ };
70
+
71
+ class f16_bits_t;
72
+ class i8_converted_t;
73
+
74
+ #if !USEARCH_USE_FP16LIB
75
+ #if defined(USEARCH_DEFINED_ARM)
76
+ using f16_native_t = __fp16;
77
+ #else
78
+ using f16_native_t = _Float16;
79
+ #endif
80
+ using f16_t = f16_native_t;
81
+ #else
82
+ using f16_native_t = void;
83
+ using f16_t = f16_bits_t;
84
+ #endif
85
+
86
+ using f64_t = double;
87
+ using f32_t = float;
88
+
89
+ using u64_t = std::uint64_t;
90
+ using u32_t = std::uint32_t;
91
+ using u16_t = std::uint16_t;
92
+ using u8_t = std::uint8_t;
93
+
94
+ using i64_t = std::int64_t;
95
+ using i32_t = std::int32_t;
96
+ using i16_t = std::int16_t;
97
+ using i8_t = std::int8_t;
98
+
99
+ enum class metric_kind_t : std::uint8_t {
100
+ unknown_k = 0,
101
+ // Classics:
102
+ ip_k = 'i',
103
+ cos_k = 'c',
104
+ l2sq_k = 'e',
105
+
106
+ // Custom:
107
+ pearson_k = 'p',
108
+ haversine_k = 'h',
109
+ divergence_k = 'd',
110
+
111
+ // Sets:
112
+ jaccard_k = 'j',
113
+ hamming_k = 'b',
114
+ tanimoto_k = 't',
115
+ sorensen_k = 's',
116
+ };
117
+
118
+ enum class scalar_kind_t : std::uint8_t {
119
+ unknown_k = 0,
120
+ // Custom:
121
+ b1x8_k = 1,
122
+ u40_k = 2,
123
+ uuid_k = 3,
124
+ // Common:
125
+ f64_k = 10,
126
+ f32_k = 11,
127
+ f16_k = 12,
128
+ f8_k = 13,
129
+ // Common Integral:
130
+ u64_k = 14,
131
+ u32_k = 15,
132
+ u16_k = 16,
133
+ u8_k = 17,
134
+ i64_k = 20,
135
+ i32_k = 21,
136
+ i16_k = 22,
137
+ i8_k = 23,
138
+ };
139
+
140
+ enum class prefetching_kind_t {
141
+ none_k,
142
+ cpu_k,
143
+ io_uring_k,
144
+ };
145
+
146
+ template <typename scalar_at> scalar_kind_t scalar_kind() noexcept {
147
+ if (std::is_same<scalar_at, b1x8_t>())
148
+ return scalar_kind_t::b1x8_k;
149
+ if (std::is_same<scalar_at, uint40_t>())
150
+ return scalar_kind_t::u40_k;
151
+ if (std::is_same<scalar_at, uuid_t>())
152
+ return scalar_kind_t::uuid_k;
153
+ if (std::is_same<scalar_at, f64_t>())
154
+ return scalar_kind_t::f64_k;
155
+ if (std::is_same<scalar_at, f32_t>())
156
+ return scalar_kind_t::f32_k;
157
+ if (std::is_same<scalar_at, f16_t>())
158
+ return scalar_kind_t::f16_k;
159
+ if (std::is_same<scalar_at, i8_t>())
160
+ return scalar_kind_t::i8_k;
161
+ if (std::is_same<scalar_at, u64_t>())
162
+ return scalar_kind_t::u64_k;
163
+ if (std::is_same<scalar_at, u32_t>())
164
+ return scalar_kind_t::u32_k;
165
+ if (std::is_same<scalar_at, u16_t>())
166
+ return scalar_kind_t::u16_k;
167
+ if (std::is_same<scalar_at, u8_t>())
168
+ return scalar_kind_t::u8_k;
169
+ if (std::is_same<scalar_at, i64_t>())
170
+ return scalar_kind_t::i64_k;
171
+ if (std::is_same<scalar_at, i32_t>())
172
+ return scalar_kind_t::i32_k;
173
+ if (std::is_same<scalar_at, i16_t>())
174
+ return scalar_kind_t::i16_k;
175
+ if (std::is_same<scalar_at, i8_t>())
176
+ return scalar_kind_t::i8_k;
177
+ return scalar_kind_t::unknown_k;
178
+ }
179
+
180
+ template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
181
+
182
+ template <typename at> at square(at value) noexcept { return value * value; }
183
+
184
+ template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp) noexcept {
185
+ return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
186
+ }
187
+ template <typename at> inline at clamp(at v, at lo, at hi) noexcept {
188
+ return usearch::clamp(v, lo, hi, std::less<at>{});
189
+ }
190
+
191
+ inline bool str_equals(char const* begin, std::size_t len, char const* other_begin) noexcept {
192
+ std::size_t other_len = std::strlen(other_begin);
193
+ return len == other_len && std::strncmp(begin, other_begin, len) == 0;
194
+ }
195
+
196
+ inline std::size_t bits_per_scalar(scalar_kind_t scalar_kind) noexcept {
197
+ switch (scalar_kind) {
198
+ case scalar_kind_t::f64_k: return 64;
199
+ case scalar_kind_t::f32_k: return 32;
200
+ case scalar_kind_t::f16_k: return 16;
201
+ case scalar_kind_t::i8_k: return 8;
202
+ case scalar_kind_t::b1x8_k: return 1;
203
+ default: return 0;
204
+ }
205
+ }
206
+
207
+ inline std::size_t bits_per_scalar_word(scalar_kind_t scalar_kind) noexcept {
208
+ switch (scalar_kind) {
209
+ case scalar_kind_t::f64_k: return 64;
210
+ case scalar_kind_t::f32_k: return 32;
211
+ case scalar_kind_t::f16_k: return 16;
212
+ case scalar_kind_t::i8_k: return 8;
213
+ case scalar_kind_t::b1x8_k: return 8;
214
+ default: return 0;
215
+ }
216
+ }
217
+
218
+ inline char const* scalar_kind_name(scalar_kind_t scalar_kind) noexcept {
219
+ switch (scalar_kind) {
220
+ case scalar_kind_t::f32_k: return "f32";
221
+ case scalar_kind_t::f16_k: return "f16";
222
+ case scalar_kind_t::f64_k: return "f64";
223
+ case scalar_kind_t::i8_k: return "i8";
224
+ case scalar_kind_t::b1x8_k: return "b1x8";
225
+ default: return "";
226
+ }
227
+ }
228
+
229
+ inline char const* metric_kind_name(metric_kind_t metric) noexcept {
230
+ switch (metric) {
231
+ case metric_kind_t::unknown_k: return "unknown";
232
+ case metric_kind_t::ip_k: return "ip";
233
+ case metric_kind_t::cos_k: return "cos";
234
+ case metric_kind_t::l2sq_k: return "l2sq";
235
+ case metric_kind_t::pearson_k: return "pearson";
236
+ case metric_kind_t::haversine_k: return "haversine";
237
+ case metric_kind_t::divergence_k: return "divergence";
238
+ case metric_kind_t::jaccard_k: return "jaccard";
239
+ case metric_kind_t::hamming_k: return "hamming";
240
+ case metric_kind_t::tanimoto_k: return "tanimoto";
241
+ case metric_kind_t::sorensen_k: return "sorensen";
242
+ }
243
+ return "";
244
+ }
245
+ inline expected_gt<scalar_kind_t> scalar_kind_from_name(char const* name, std::size_t len) {
246
+ expected_gt<scalar_kind_t> parsed;
247
+ if (str_equals(name, len, "f32"))
248
+ parsed.result = scalar_kind_t::f32_k;
249
+ else if (str_equals(name, len, "f64"))
250
+ parsed.result = scalar_kind_t::f64_k;
251
+ else if (str_equals(name, len, "f16"))
252
+ parsed.result = scalar_kind_t::f16_k;
253
+ else if (str_equals(name, len, "i8"))
254
+ parsed.result = scalar_kind_t::i8_k;
255
+ else
256
+ parsed.failed("Unknown type, choose: f32, f16, f64, i8");
257
+ return parsed;
258
+ }
259
+
260
+ inline expected_gt<scalar_kind_t> scalar_kind_from_name(char const* name) {
261
+ return scalar_kind_from_name(name, std::strlen(name));
262
+ }
263
+
264
+ inline expected_gt<metric_kind_t> metric_from_name(char const* name, std::size_t len) {
265
+ expected_gt<metric_kind_t> parsed;
266
+ if (str_equals(name, len, "l2sq") || str_equals(name, len, "euclidean_sq")) {
267
+ parsed.result = metric_kind_t::l2sq_k;
268
+ } else if (str_equals(name, len, "ip") || str_equals(name, len, "inner") || str_equals(name, len, "dot")) {
269
+ parsed.result = metric_kind_t::ip_k;
270
+ } else if (str_equals(name, len, "cos") || str_equals(name, len, "angular")) {
271
+ parsed.result = metric_kind_t::cos_k;
272
+ } else if (str_equals(name, len, "haversine")) {
273
+ parsed.result = metric_kind_t::haversine_k;
274
+ } else if (str_equals(name, len, "divergence")) {
275
+ parsed.result = metric_kind_t::divergence_k;
276
+ } else if (str_equals(name, len, "pearson")) {
277
+ parsed.result = metric_kind_t::pearson_k;
278
+ } else if (str_equals(name, len, "hamming")) {
279
+ parsed.result = metric_kind_t::hamming_k;
280
+ } else if (str_equals(name, len, "tanimoto")) {
281
+ parsed.result = metric_kind_t::tanimoto_k;
282
+ } else if (str_equals(name, len, "sorensen")) {
283
+ parsed.result = metric_kind_t::sorensen_k;
284
+ } else
285
+ parsed.failed("Unknown distance, choose: l2sq, ip, cos, haversine, divergence, jaccard, pearson, hamming, "
286
+ "tanimoto, sorensen");
287
+ return parsed;
288
+ }
289
+
290
+ inline expected_gt<metric_kind_t> metric_from_name(char const* name) {
291
+ return metric_from_name(name, std::strlen(name));
292
+ }
293
+
294
+ inline float f16_to_f32(std::uint16_t u16) noexcept {
295
+ #if !USEARCH_USE_FP16LIB
296
+ f16_native_t f16;
297
+ std::memcpy(&f16, &u16, sizeof(std::uint16_t));
298
+ return float(f16);
299
+ #else
300
+ return fp16_ieee_to_fp32_value(u16);
301
+ #endif
302
+ }
303
+
304
+ inline std::uint16_t f32_to_f16(float f32) noexcept {
305
+ #if !USEARCH_USE_FP16LIB
306
+ f16_native_t f16 = f16_native_t(f32);
307
+ std::uint16_t u16;
308
+ std::memcpy(&u16, &f16, sizeof(std::uint16_t));
309
+ return u16;
310
+ #else
311
+ return fp16_ieee_from_fp32_value(f32);
312
+ #endif
313
+ }
314
+
315
+ /**
316
+ * @brief Numeric type for the IEEE 754 half-precision floating point.
317
+ * If hardware support isn't available, falls back to a hardware
318
+ * agnostic in-software implementation.
319
+ */
320
+ class f16_bits_t {
321
+ std::uint16_t uint16_{};
322
+
323
+ public:
324
+ inline f16_bits_t() noexcept : uint16_(0) {}
325
+ inline f16_bits_t(f16_bits_t&&) = default;
326
+ inline f16_bits_t& operator=(f16_bits_t&&) = default;
327
+ inline f16_bits_t(f16_bits_t const&) = default;
328
+ inline f16_bits_t& operator=(f16_bits_t const&) = default;
329
+
330
+ inline operator float() const noexcept { return f16_to_f32(uint16_); }
331
+ inline explicit operator bool() const noexcept { return f16_to_f32(uint16_) > 0.5f; }
332
+
333
+ inline f16_bits_t(i8_converted_t) noexcept;
334
+ inline f16_bits_t(bool v) noexcept : uint16_(f32_to_f16(v)) {}
335
+ inline f16_bits_t(float v) noexcept : uint16_(f32_to_f16(v)) {}
336
+ inline f16_bits_t(double v) noexcept : uint16_(f32_to_f16(static_cast<float>(v))) {}
337
+
338
+ inline f16_bits_t operator+(f16_bits_t other) const noexcept { return {float(*this) + float(other)}; }
339
+ inline f16_bits_t operator-(f16_bits_t other) const noexcept { return {float(*this) - float(other)}; }
340
+ inline f16_bits_t operator*(f16_bits_t other) const noexcept { return {float(*this) * float(other)}; }
341
+ inline f16_bits_t operator/(f16_bits_t other) const noexcept { return {float(*this) / float(other)}; }
342
+ inline f16_bits_t operator+(float other) const noexcept { return {float(*this) + other}; }
343
+ inline f16_bits_t operator-(float other) const noexcept { return {float(*this) - other}; }
344
+ inline f16_bits_t operator*(float other) const noexcept { return {float(*this) * other}; }
345
+ inline f16_bits_t operator/(float other) const noexcept { return {float(*this) / other}; }
346
+ inline f16_bits_t operator+(double other) const noexcept { return {float(*this) + other}; }
347
+ inline f16_bits_t operator-(double other) const noexcept { return {float(*this) - other}; }
348
+ inline f16_bits_t operator*(double other) const noexcept { return {float(*this) * other}; }
349
+ inline f16_bits_t operator/(double other) const noexcept { return {float(*this) / other}; }
350
+
351
+ inline f16_bits_t& operator+=(float v) noexcept {
352
+ uint16_ = f32_to_f16(v + f16_to_f32(uint16_));
353
+ return *this;
354
+ }
355
+
356
+ inline f16_bits_t& operator-=(float v) noexcept {
357
+ uint16_ = f32_to_f16(v - f16_to_f32(uint16_));
358
+ return *this;
359
+ }
360
+
361
+ inline f16_bits_t& operator*=(float v) noexcept {
362
+ uint16_ = f32_to_f16(v * f16_to_f32(uint16_));
363
+ return *this;
364
+ }
365
+
366
+ inline f16_bits_t& operator/=(float v) noexcept {
367
+ uint16_ = f32_to_f16(v / f16_to_f32(uint16_));
368
+ return *this;
369
+ }
370
+ };
371
+
372
+ /**
373
+ * @brief An STL-based executor or a "thread-pool" for parallel execution.
374
+ * Isn't efficient for small batches, as it recreates the threads on every call.
375
+ */
376
+ class executor_stl_t {
377
+ std::size_t threads_count_{};
378
+
379
+ struct jthread_t {
380
+ std::thread native_;
381
+
382
+ jthread_t() = default;
383
+ jthread_t(jthread_t&&) = default;
384
+ jthread_t(jthread_t const&) = delete;
385
+ template <typename callable_at> jthread_t(callable_at&& func) : native_([=]() { func(); }) {}
386
+
387
+ ~jthread_t() {
388
+ if (native_.joinable())
389
+ native_.join();
390
+ }
391
+ };
392
+
393
+ public:
394
+ /**
395
+ * @param threads_count The number of threads to be used for parallel execution.
396
+ */
397
+ executor_stl_t(std::size_t threads_count = 0) noexcept
398
+ : threads_count_(threads_count ? threads_count : std::thread::hardware_concurrency()) {}
399
+
400
+ /**
401
+ * @return Maximum number of threads available to the executor.
402
+ */
403
+ std::size_t size() const noexcept { return threads_count_; }
404
+
405
+ /**
406
+ * @brief Executes a fixed number of tasks using the specified thread-aware function.
407
+ * @param tasks The total number of tasks to be executed.
408
+ * @param thread_aware_function The thread-aware function to be called for each thread index and task index.
409
+ * @throws If an exception occurs during execution of the thread-aware function.
410
+ */
411
+ template <typename thread_aware_function_at>
412
+ void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
413
+ std::vector<jthread_t> threads_pool;
414
+ std::size_t tasks_per_thread = tasks;
415
+ std::size_t threads_count = (std::min)(threads_count_, tasks);
416
+ if (threads_count > 1) {
417
+ tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
418
+ for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
419
+ threads_pool.emplace_back([=]() {
420
+ for (std::size_t task_idx = thread_idx * tasks_per_thread;
421
+ task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread); ++task_idx)
422
+ thread_aware_function(thread_idx, task_idx);
423
+ });
424
+ }
425
+ }
426
+ for (std::size_t task_idx = 0; task_idx < (std::min)(tasks, tasks_per_thread); ++task_idx)
427
+ thread_aware_function(0, task_idx);
428
+ }
429
+
430
+ /**
431
+ * @brief Executes limited number of tasks using the specified thread-aware function.
432
+ * @param tasks The upper bound on the number of tasks.
433
+ * @param thread_aware_function The thread-aware function to be called for each thread index and task index.
434
+ * @throws If an exception occurs during execution of the thread-aware function.
435
+ */
436
+ template <typename thread_aware_function_at>
437
+ void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
438
+ std::vector<jthread_t> threads_pool;
439
+ std::size_t tasks_per_thread = tasks;
440
+ std::size_t threads_count = (std::min)(threads_count_, tasks);
441
+ std::atomic_bool stop{false};
442
+ if (threads_count > 1) {
443
+ tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
444
+ for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
445
+ threads_pool.emplace_back([=, &stop]() {
446
+ for (std::size_t task_idx = thread_idx * tasks_per_thread;
447
+ task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread) &&
448
+ !stop.load(std::memory_order_relaxed);
449
+ ++task_idx)
450
+ if (!thread_aware_function(thread_idx, task_idx))
451
+ stop.store(true, std::memory_order_relaxed);
452
+ });
453
+ }
454
+ }
455
+ for (std::size_t task_idx = 0;
456
+ task_idx < (std::min)(tasks, tasks_per_thread) && !stop.load(std::memory_order_relaxed); ++task_idx)
457
+ if (!thread_aware_function(0, task_idx))
458
+ stop.store(true, std::memory_order_relaxed);
459
+ }
460
+
461
+ /**
462
+ * @brief Saturates every available thread with the given workload, until they finish.
463
+ * @param thread_aware_function The thread-aware function to be called for each thread index.
464
+ * @throws If an exception occurs during execution of the thread-aware function.
465
+ */
466
+ template <typename thread_aware_function_at>
467
+ void parallel(thread_aware_function_at&& thread_aware_function) noexcept(false) {
468
+ if (threads_count_ == 1)
469
+ return thread_aware_function(0);
470
+ std::vector<jthread_t> threads_pool;
471
+ for (std::size_t thread_idx = 1; thread_idx < threads_count_; ++thread_idx)
472
+ threads_pool.emplace_back([=]() { thread_aware_function(thread_idx); });
473
+ thread_aware_function(0);
474
+ }
475
+ };
476
+
477
+ #if USEARCH_USE_OPENMP
478
+
479
+ /**
480
+ * @brief An OpenMP-based executor or a "thread-pool" for parallel execution.
481
+ * Is the preferred implementation, when available, and maximum performance is needed.
482
+ */
483
+ class executor_openmp_t {
484
+ public:
485
+ /**
486
+ * @param threads_count The number of threads to be used for parallel execution.
487
+ */
488
+ executor_openmp_t(std::size_t threads_count = 0) noexcept {
489
+ omp_set_num_threads(static_cast<int>(threads_count ? threads_count : std::thread::hardware_concurrency()));
490
+ }
491
+
492
+ /**
493
+ * @return Maximum number of threads available to the executor.
494
+ */
495
+ std::size_t size() const noexcept { return omp_get_max_threads(); }
496
+
497
+ /**
498
+ * @brief Executes tasks in bulk using the specified thread-aware function.
499
+ * @param tasks The total number of tasks to be executed.
500
+ * @param thread_aware_function The thread-aware function to be called for each thread index and task index.
501
+ * @throws If an exception occurs during execution of the thread-aware function.
502
+ */
503
+ template <typename thread_aware_function_at>
504
+ void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
505
+ #pragma omp parallel for schedule(dynamic, 1)
506
+ for (std::size_t i = 0; i != tasks; ++i) {
507
+ thread_aware_function(omp_get_thread_num(), i);
508
+ }
509
+ }
510
+
511
+ /**
512
+ * @brief Executes tasks in bulk using the specified thread-aware function.
513
+ * @param tasks The total number of tasks to be executed.
514
+ * @param thread_aware_function The thread-aware function to be called for each thread index and task index.
515
+ * @throws If an exception occurs during execution of the thread-aware function.
516
+ */
517
+ template <typename thread_aware_function_at>
518
+ void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
519
+ // OpenMP cancellation points are not yet available on most platforms, and require
520
+ // the `OMP_CANCELLATION` environment variable to be set.
521
+ // http://jakascorner.com/blog/2016/08/omp-cancel.html
522
+ // if (omp_get_cancellation()) {
523
+ // #pragma omp parallel for schedule(dynamic, 1)
524
+ // for (std::size_t i = 0; i != tasks; ++i) {
525
+ // #pragma omp cancellation point for
526
+ // if (!thread_aware_function(omp_get_thread_num(), i)) {
527
+ // #pragma omp cancel for
528
+ // }
529
+ // }
530
+ // }
531
+ std::atomic_bool stop{false};
532
+ #pragma omp parallel for schedule(dynamic, 1) shared(stop)
533
+ for (std::size_t i = 0; i != tasks; ++i) {
534
+ if (!stop.load(std::memory_order_relaxed) && !thread_aware_function(omp_get_thread_num(), i))
535
+ stop.store(true, std::memory_order_relaxed);
536
+ }
537
+ }
538
+
539
+ /**
540
+ * @brief Saturates every available thread with the given workload, until they finish.
541
+ * @param thread_aware_function The thread-aware function to be called for each thread index.
542
+ * @throws If an exception occurs during execution of the thread-aware function.
543
+ */
544
+ template <typename thread_aware_function_at>
545
+ void parallel(thread_aware_function_at&& thread_aware_function) noexcept(false) {
546
+ #pragma omp parallel
547
+ { thread_aware_function(omp_get_thread_num()); }
548
+ }
549
+ };
550
+
551
+ using executor_default_t = executor_openmp_t;
552
+
553
+ #else
554
+
555
+ using executor_default_t = executor_stl_t;
556
+
557
+ #endif
558
+
559
+ /**
560
+ * @brief Uses OS-specific APIs for aligned memory allocations.
561
+ */
562
+ template <typename element_at = char, std::size_t alignment_ak = 64> //
563
+ class aligned_allocator_gt {
564
+ public:
565
+ using value_type = element_at;
566
+ using size_type = std::size_t;
567
+ using pointer = element_at*;
568
+ using const_pointer = element_at const*;
569
+ template <typename other_element_at> struct rebind {
570
+ using other = aligned_allocator_gt<other_element_at>;
571
+ };
572
+
573
+ constexpr std::size_t alignment() const { return alignment_ak; }
574
+
575
+ pointer allocate(size_type length) const {
576
+ std::size_t length_bytes = alignment_ak * divide_round_up<alignment_ak>(length * sizeof(value_type));
577
+ std::size_t alignment = alignment_ak;
578
+ // void* result = nullptr;
579
+ // int status = posix_memalign(&result, alignment, length_bytes);
580
+ // return status == 0 ? (pointer)result : nullptr;
581
+ #if defined(USEARCH_DEFINED_WINDOWS)
582
+ return (pointer)_aligned_malloc(length_bytes, alignment);
583
+ #else
584
+ return (pointer)aligned_alloc(alignment, length_bytes);
585
+ #endif
586
+ }
587
+
588
+ void deallocate(pointer begin, size_type) const {
589
+ #if defined(USEARCH_DEFINED_WINDOWS)
590
+ _aligned_free(begin);
591
+ #else
592
+ free(begin);
593
+ #endif
594
+ }
595
+ };
596
+
597
+ using aligned_allocator_t = aligned_allocator_gt<>;
598
+
599
+ class page_allocator_t {
600
+ public:
601
+ static constexpr std::size_t page_size() { return 4096; }
602
+
603
+ /**
604
+ * @brief Allocates an @b uninitialized block of memory of the specified size.
605
+ * @param count_bytes The number of bytes to allocate.
606
+ * @return A pointer to the allocated memory block, or `nullptr` if allocation fails.
607
+ */
608
+ byte_t* allocate(std::size_t count_bytes) const noexcept {
609
+ count_bytes = divide_round_up(count_bytes, page_size()) * page_size();
610
+ #if defined(USEARCH_DEFINED_WINDOWS)
611
+ return (byte_t*)(::VirtualAlloc(NULL, count_bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
612
+ #else
613
+ return (byte_t*)mmap(NULL, count_bytes, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
614
+ #endif
615
+ }
616
+
617
+ void deallocate(byte_t* page_pointer, std::size_t count_bytes) const noexcept {
618
+ #if defined(USEARCH_DEFINED_WINDOWS)
619
+ ::VirtualFree(page_pointer, 0, MEM_RELEASE);
620
+ #else
621
+ count_bytes = divide_round_up(count_bytes, page_size()) * page_size();
622
+ munmap(page_pointer, count_bytes);
623
+ #endif
624
+ }
625
+ };
626
+
627
+ /**
628
+ * @brief Memory-mapping allocator designed for "alloc many, free at once" usage patterns.
629
+ * @b Thread-safe, @b except constructors and destructors.
630
+ *
631
+ * Using this memory allocator won't affect your overall speed much, as that is not the bottleneck.
632
+ * However, it can drastically improve memory usage especially for huge indexes of small vectors.
633
+ */
634
+ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
635
+
636
+ static constexpr std::size_t min_capacity() { return 1024 * 1024 * 4; }
637
+ static constexpr std::size_t capacity_multiplier() { return 2; }
638
+ static constexpr std::size_t head_size() {
639
+ /// Pointer to the the previous arena and the size of the current one.
640
+ return divide_round_up<alignment_ak>(sizeof(byte_t*) + sizeof(std::size_t)) * alignment_ak;
641
+ }
642
+
643
+ std::mutex mutex_;
644
+ byte_t* last_arena_ = nullptr;
645
+ std::size_t last_usage_ = head_size();
646
+ std::size_t last_capacity_ = min_capacity();
647
+ std::size_t wasted_space_ = 0;
648
+
649
+ public:
650
+ using value_type = byte_t;
651
+ using size_type = std::size_t;
652
+ using pointer = byte_t*;
653
+ using const_pointer = byte_t const*;
654
+
655
+ memory_mapping_allocator_gt() = default;
656
+ memory_mapping_allocator_gt(memory_mapping_allocator_gt&& other) noexcept
657
+ : last_arena_(exchange(other.last_arena_, nullptr)), last_usage_(exchange(other.last_usage_, 0)),
658
+ last_capacity_(exchange(other.last_capacity_, 0)), wasted_space_(exchange(other.wasted_space_, 0)) {}
659
+
660
+ memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt&& other) noexcept {
661
+ std::swap(last_arena_, other.last_arena_);
662
+ std::swap(last_usage_, other.last_usage_);
663
+ std::swap(last_capacity_, other.last_capacity_);
664
+ std::swap(wasted_space_, other.wasted_space_);
665
+ return *this;
666
+ }
667
+
668
+ ~memory_mapping_allocator_gt() noexcept { reset(); }
669
+
670
+ /**
671
+ * @brief Discards all previously allocated memory buffers.
672
+ */
673
+ void reset() noexcept {
674
+ byte_t* last_arena = last_arena_;
675
+ while (last_arena) {
676
+ byte_t* previous_arena = nullptr;
677
+ std::memcpy(&previous_arena, last_arena, sizeof(byte_t*));
678
+ std::size_t last_cap = 0;
679
+ std::memcpy(&last_cap, last_arena + sizeof(byte_t*), sizeof(std::size_t));
680
+ page_allocator_t{}.deallocate(last_arena, last_cap);
681
+ last_arena = previous_arena;
682
+ }
683
+
684
+ // Clear the references:
685
+ last_arena_ = nullptr;
686
+ last_usage_ = head_size();
687
+ last_capacity_ = min_capacity();
688
+ wasted_space_ = 0;
689
+ }
690
+
691
+ /**
692
+ * @brief Copy constructor.
693
+ * @note This is a no-op copy constructor since the allocator is not copyable.
694
+ */
695
+ memory_mapping_allocator_gt(memory_mapping_allocator_gt const&) noexcept {}
696
+
697
+ /**
698
+ * @brief Copy assignment operator.
699
+ * @note This is a no-op copy assignment operator since the allocator is not copyable.
700
+ * @return Reference to the allocator after the assignment.
701
+ */
702
+ memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt const&) noexcept {
703
+ reset();
704
+ return *this;
705
+ }
706
+
707
+ /**
708
+ * @brief Allocates an @b uninitialized block of memory of the specified size.
709
+ * @param count_bytes The number of bytes to allocate.
710
+ * @return A pointer to the allocated memory block, or `nullptr` if allocation fails.
711
+ */
712
+ inline byte_t* allocate(std::size_t count_bytes) noexcept {
713
+ std::size_t extended_bytes = divide_round_up<alignment_ak>(count_bytes) * alignment_ak;
714
+ std::unique_lock<std::mutex> lock(mutex_);
715
+ if (!last_arena_ || (last_usage_ + extended_bytes >= last_capacity_)) {
716
+ std::size_t new_cap = (std::max)(last_capacity_, ceil2(extended_bytes)) * capacity_multiplier();
717
+ byte_t* new_arena = page_allocator_t{}.allocate(new_cap);
718
+ if (!new_arena)
719
+ return nullptr;
720
+ std::memcpy(new_arena, &last_arena_, sizeof(byte_t*));
721
+ std::memcpy(new_arena + sizeof(byte_t*), &new_cap, sizeof(std::size_t));
722
+
723
+ wasted_space_ += total_reserved();
724
+ last_arena_ = new_arena;
725
+ last_capacity_ = new_cap;
726
+ last_usage_ = head_size();
727
+ }
728
+
729
+ wasted_space_ += extended_bytes - count_bytes;
730
+ return last_arena_ + exchange(last_usage_, last_usage_ + extended_bytes);
731
+ }
732
+
733
+ /**
734
+ * @brief Returns the amount of memory used by the allocator across all arenas.
735
+ * @return The amount of space in bytes.
736
+ */
737
+ std::size_t total_allocated() const noexcept {
738
+ if (!last_arena_)
739
+ return 0;
740
+ std::size_t total_used = 0;
741
+ std::size_t last_capacity = last_capacity_;
742
+ do {
743
+ total_used += last_capacity;
744
+ last_capacity /= capacity_multiplier();
745
+ } while (last_capacity >= min_capacity());
746
+ return total_used;
747
+ }
748
+
749
+ /**
750
+ * @brief Returns the amount of wasted space due to alignment.
751
+ * @return The amount of wasted space in bytes.
752
+ */
753
+ std::size_t total_wasted() const noexcept { return wasted_space_; }
754
+
755
+ /**
756
+ * @brief Returns the amount of remaining memory already reserved but not yet used.
757
+ * @return The amount of reserved memory in bytes.
758
+ */
759
+ std::size_t total_reserved() const noexcept { return last_arena_ ? last_capacity_ - last_usage_ : 0; }
760
+
761
+ /**
762
+ * @warning The very first memory de-allocation discards all the arenas!
763
+ */
764
+ void deallocate(byte_t* = nullptr, std::size_t = 0) noexcept { reset(); }
765
+ };
766
+
767
+ using memory_mapping_allocator_t = memory_mapping_allocator_gt<>;
768
+
769
+ /**
770
+ * @brief C++11 userspace implementation of an oversimplified `std::shared_mutex`,
771
+ * that assumes rare interleaving of shared and unique locks. It's not fair,
772
+ * but requires only a single 32-bit atomic integer to work.
773
+ */
774
+ class unfair_shared_mutex_t {
775
+ /** Any positive integer describes the number of concurrent readers */
776
+ enum state_t : std::int32_t {
777
+ idle_k = 0,
778
+ writing_k = -1,
779
+ };
780
+ std::atomic<std::int32_t> state_{idle_k};
781
+
782
+ public:
783
+ inline void lock() noexcept {
784
+ std::int32_t raw;
785
+ relock:
786
+ raw = idle_k;
787
+ if (!state_.compare_exchange_weak(raw, writing_k, std::memory_order_acquire, std::memory_order_relaxed)) {
788
+ std::this_thread::yield();
789
+ goto relock;
790
+ }
791
+ }
792
+
793
+ inline void unlock() noexcept { state_.store(idle_k, std::memory_order_release); }
794
+
795
+ inline void lock_shared() noexcept {
796
+ std::int32_t raw;
797
+ relock_shared:
798
+ raw = state_.load(std::memory_order_acquire);
799
+ // Spin while it's uniquely locked
800
+ if (raw == writing_k) {
801
+ std::this_thread::yield();
802
+ goto relock_shared;
803
+ }
804
+ // Try incrementing the counter
805
+ if (!state_.compare_exchange_weak(raw, raw + 1, std::memory_order_acquire, std::memory_order_relaxed)) {
806
+ std::this_thread::yield();
807
+ goto relock_shared;
808
+ }
809
+ }
810
+
811
+ inline void unlock_shared() noexcept { state_.fetch_sub(1, std::memory_order_release); }
812
+
813
+ /**
814
+ * @brief Try upgrades the current `lock_shared()` to a unique `lock()` state.
815
+ */
816
+ inline bool try_escalate() noexcept {
817
+ std::int32_t one_read = 1;
818
+ return state_.compare_exchange_weak(one_read, writing_k, std::memory_order_acquire, std::memory_order_relaxed);
819
+ }
820
+
821
+ /**
822
+ * @brief Escalates current lock potentially loosing control in the middle.
823
+ * It's a shortcut for `try_escalate`-`unlock_shared`-`lock` trio.
824
+ */
825
+ inline void unsafe_escalate() noexcept {
826
+ if (!try_escalate()) {
827
+ unlock_shared();
828
+ lock();
829
+ }
830
+ }
831
+
832
+ /**
833
+ * @brief Upgrades the current `lock_shared()` to a unique `lock()` state.
834
+ */
835
+ inline void escalate() noexcept {
836
+ while (!try_escalate())
837
+ std::this_thread::yield();
838
+ }
839
+
840
+ /**
841
+ * @brief De-escalation of a previously escalated state.
842
+ */
843
+ inline void de_escalate() noexcept {
844
+ std::int32_t one_read = 1;
845
+ state_.store(one_read, std::memory_order_release);
846
+ }
847
+ };
848
+
849
+ template <typename mutex_at = unfair_shared_mutex_t> class shared_lock_gt {
850
+ mutex_at& mutex_;
851
+
852
+ public:
853
+ inline explicit shared_lock_gt(mutex_at& m) noexcept : mutex_(m) { mutex_.lock_shared(); }
854
+ inline ~shared_lock_gt() noexcept { mutex_.unlock_shared(); }
855
+ };
856
+
857
+ /**
858
+ * @brief Utility class used to cast arrays of one scalar type to another,
859
+ * avoiding unnecessary conversions.
860
+ */
861
+ template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
862
+ inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
863
+ from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
864
+ to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
865
+ auto converter = [](from_scalar_at from) { return to_scalar_at(from); };
866
+ std::transform(typed_input, typed_input + dim, typed_output, converter);
867
+ return true;
868
+ }
869
+ };
870
+
871
+ template <> struct cast_gt<f32_t, f32_t> {
872
+ bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
873
+ };
874
+
875
+ template <> struct cast_gt<f64_t, f64_t> {
876
+ bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
877
+ };
878
+
879
+ template <> struct cast_gt<f16_bits_t, f16_bits_t> {
880
+ bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
881
+ };
882
+
883
+ template <> struct cast_gt<i8_t, i8_t> {
884
+ bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
885
+ };
886
+
887
+ template <> struct cast_gt<b1x8_t, b1x8_t> {
888
+ bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
889
+ };
890
+
891
+ template <typename from_scalar_at> struct cast_gt<from_scalar_at, b1x8_t> {
892
+ inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
893
+ from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
894
+ unsigned char* typed_output = reinterpret_cast<unsigned char*>(output);
895
+ for (std::size_t i = 0; i != dim; ++i)
896
+ // Converting from scalar types to boolean isn't trivial and depends on the type.
897
+ // The most common case is to consider all positive values as `true` and all others as `false`.
898
+ // - `bool(0.00001f)` converts to 1
899
+ // - `bool(-0.00001f)` converts to 1
900
+ // - `bool(0)` converts to 0
901
+ // - `bool(-0)` converts to 0
902
+ // - `bool(std::numeric_limits<float>::infinity())` converts to 1
903
+ // - `bool(std::numeric_limits<float>::epsilon())` converts to 1
904
+ // - `bool(std::numeric_limits<float>::signaling_NaN())` converts to 1
905
+ // - `bool(std::numeric_limits<float>::denorm_min())` converts to 1
906
+ typed_output[i / CHAR_BIT] |= bool(typed_input[i] > 0) ? (128 >> (i & (CHAR_BIT - 1))) : 0;
907
+ return true;
908
+ }
909
+ };
910
+
911
+ template <typename to_scalar_at> struct cast_gt<b1x8_t, to_scalar_at> {
912
+ inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
913
+ unsigned char const* typed_input = reinterpret_cast<unsigned char const*>(input);
914
+ to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
915
+ for (std::size_t i = 0; i != dim; ++i)
916
+ // We can't entirely reconstruct the original scalar type from a boolean.
917
+ // The simplest variant would be to map set bits to ones, and unset bits to zeros.
918
+ typed_output[i] = bool(typed_input[i / CHAR_BIT] & (128 >> (i & (CHAR_BIT - 1))));
919
+ return true;
920
+ }
921
+ };
922
+
923
+ /**
924
+ * @brief Numeric type for uniformly-distributed floating point
925
+ * values within [-1,1] range, quantized to integers [-100,100].
926
+ */
927
+ class i8_converted_t {
928
+ std::int8_t int8_{};
929
+
930
+ public:
931
+ constexpr static f32_t divisor_k = 100.f;
932
+ constexpr static std::int8_t min_k = -100;
933
+ constexpr static std::int8_t max_k = 100;
934
+
935
+ inline i8_converted_t() noexcept : int8_(0) {}
936
+ inline i8_converted_t(bool v) noexcept : int8_(v ? max_k : 0) {}
937
+
938
+ inline i8_converted_t(i8_converted_t&&) = default;
939
+ inline i8_converted_t& operator=(i8_converted_t&&) = default;
940
+ inline i8_converted_t(i8_converted_t const&) = default;
941
+ inline i8_converted_t& operator=(i8_converted_t const&) = default;
942
+
943
+ inline operator f16_t() const noexcept { return static_cast<f16_t>(f32_t(int8_) / divisor_k); }
944
+ inline operator f32_t() const noexcept { return f32_t(int8_) / divisor_k; }
945
+ inline operator f64_t() const noexcept { return f64_t(int8_) / divisor_k; }
946
+ inline explicit operator bool() const noexcept { return int8_ > (max_k / 2); }
947
+ inline explicit operator std::int8_t() const noexcept { return int8_; }
948
+ inline explicit operator std::int16_t() const noexcept { return int8_; }
949
+ inline explicit operator std::int32_t() const noexcept { return int8_; }
950
+ inline explicit operator std::int64_t() const noexcept { return int8_; }
951
+
952
+ inline i8_converted_t(f16_t v)
953
+ : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
954
+ inline i8_converted_t(f32_t v)
955
+ : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
956
+ inline i8_converted_t(f64_t v)
957
+ : int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
958
+ };
959
+
960
+ f16_bits_t::f16_bits_t(i8_converted_t v) noexcept : uint16_(f32_to_f16(v)) {}
961
+
962
+ template <> struct cast_gt<i8_t, f16_t> : public cast_gt<i8_converted_t, f16_t> {};
963
+ template <> struct cast_gt<i8_t, f32_t> : public cast_gt<i8_converted_t, f32_t> {};
964
+ template <> struct cast_gt<i8_t, f64_t> : public cast_gt<i8_converted_t, f64_t> {};
965
+
966
+ template <> struct cast_gt<f16_t, i8_t> : public cast_gt<f16_t, i8_converted_t> {};
967
+ template <> struct cast_gt<f32_t, i8_t> : public cast_gt<f32_t, i8_converted_t> {};
968
+ template <> struct cast_gt<f64_t, i8_t> : public cast_gt<f64_t, i8_converted_t> {};
969
+
970
+ /**
971
+ * @brief Inner (Dot) Product distance.
972
+ */
973
+ template <typename scalar_at = float, typename result_at = scalar_at> struct metric_ip_gt {
974
+ using scalar_t = scalar_at;
975
+ using result_t = result_at;
976
+
977
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
978
+ result_t ab{};
979
+ #if USEARCH_USE_OPENMP
980
+ #pragma omp simd reduction(+ : ab)
981
+ #elif defined(USEARCH_DEFINED_CLANG)
982
+ #pragma clang loop vectorize(enable)
983
+ #elif defined(USEARCH_DEFINED_GCC)
984
+ #pragma GCC ivdep
985
+ #endif
986
+ for (std::size_t i = 0; i != dim; ++i)
987
+ ab += result_t(a[i]) * result_t(b[i]);
988
+ return 1 - ab;
989
+ }
990
+ };
991
+
992
+ /**
993
+ * @brief Cosine (Angular) distance.
994
+ * Identical to the Inner Product of normalized vectors.
995
+ * Unless you are running on an tiny embedded platform, this metric
996
+ * is recommended over `::metric_ip_gt` for low-precision scalars.
997
+ */
998
+ template <typename scalar_at = float, typename result_at = scalar_at> struct metric_cos_gt {
999
+ using scalar_t = scalar_at;
1000
+ using result_t = result_at;
1001
+
1002
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
1003
+ result_t ab{}, a2{}, b2{};
1004
+ #if USEARCH_USE_OPENMP
1005
+ #pragma omp simd reduction(+ : ab, a2, b2)
1006
+ #elif defined(USEARCH_DEFINED_CLANG)
1007
+ #pragma clang loop vectorize(enable)
1008
+ #elif defined(USEARCH_DEFINED_GCC)
1009
+ #pragma GCC ivdep
1010
+ #endif
1011
+ for (std::size_t i = 0; i != dim; ++i) {
1012
+ result_t ai = static_cast<result_t>(a[i]);
1013
+ result_t bi = static_cast<result_t>(b[i]);
1014
+ ab += ai * bi, a2 += square(ai), b2 += square(bi);
1015
+ }
1016
+
1017
+ result_t result_if_zero[2][2];
1018
+ result_if_zero[0][0] = 1 - ab / (std::sqrt(a2) * std::sqrt(b2));
1019
+ result_if_zero[0][1] = result_if_zero[1][0] = 1;
1020
+ result_if_zero[1][1] = 0;
1021
+ return result_if_zero[a2 == 0][b2 == 0];
1022
+ }
1023
+ };
1024
+
1025
+ /**
1026
+ * @brief Squared Euclidean (L2) distance.
1027
+ * Square root is avoided at the end, as it won't affect the ordering.
1028
+ */
1029
+ template <typename scalar_at = float, typename result_at = scalar_at> struct metric_l2sq_gt {
1030
+ using scalar_t = scalar_at;
1031
+ using result_t = result_at;
1032
+
1033
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
1034
+ result_t ab_deltas_sq{};
1035
+ #if USEARCH_USE_OPENMP
1036
+ #pragma omp simd reduction(+ : ab_deltas_sq)
1037
+ #elif defined(USEARCH_DEFINED_CLANG)
1038
+ #pragma clang loop vectorize(enable)
1039
+ #elif defined(USEARCH_DEFINED_GCC)
1040
+ #pragma GCC ivdep
1041
+ #endif
1042
+ for (std::size_t i = 0; i != dim; ++i) {
1043
+ result_t ai = static_cast<result_t>(a[i]);
1044
+ result_t bi = static_cast<result_t>(b[i]);
1045
+ ab_deltas_sq += square(ai - bi);
1046
+ }
1047
+ return ab_deltas_sq;
1048
+ }
1049
+ };
1050
+
1051
+ /**
1052
+ * @brief Hamming distance computes the number of differing bits in
1053
+ * two arrays of integers. An example would be a textual document,
1054
+ * tokenized and hashed into a fixed-capacity bitset.
1055
+ */
1056
+ template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct metric_hamming_gt {
1057
+ using scalar_t = scalar_at;
1058
+ using result_t = result_at;
1059
+ static_assert( //
1060
+ std::is_unsigned<scalar_t>::value ||
1061
+ (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
1062
+ "Hamming distance requires unsigned integral words");
1063
+
1064
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
1065
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
1066
+ result_t matches{};
1067
+ #if USEARCH_USE_OPENMP
1068
+ #pragma omp simd reduction(+ : matches)
1069
+ #elif defined(USEARCH_DEFINED_CLANG)
1070
+ #pragma clang loop vectorize(enable)
1071
+ #elif defined(USEARCH_DEFINED_GCC)
1072
+ #pragma GCC ivdep
1073
+ #endif
1074
+ for (std::size_t i = 0; i != words; ++i)
1075
+ matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
1076
+ return matches;
1077
+ }
1078
+ };
1079
+
1080
+ /**
1081
+ * @brief Tanimoto distance is the intersection over bitwise union.
1082
+ * Often used in chemistry and biology to compare molecular fingerprints.
1083
+ */
1084
+ template <typename scalar_at = std::uint64_t, typename result_at = float> struct metric_tanimoto_gt {
1085
+ using scalar_t = scalar_at;
1086
+ using result_t = result_at;
1087
+ static_assert( //
1088
+ std::is_unsigned<scalar_t>::value ||
1089
+ (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
1090
+ "Tanimoto distance requires unsigned integral words");
1091
+ static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
1092
+
1093
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
1094
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
1095
+ result_t and_count{};
1096
+ result_t or_count{};
1097
+ #if USEARCH_USE_OPENMP
1098
+ #pragma omp simd reduction(+ : and_count, or_count)
1099
+ #elif defined(USEARCH_DEFINED_CLANG)
1100
+ #pragma clang loop vectorize(enable)
1101
+ #elif defined(USEARCH_DEFINED_GCC)
1102
+ #pragma GCC ivdep
1103
+ #endif
1104
+ for (std::size_t i = 0; i != words; ++i) {
1105
+ and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count();
1106
+ or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
1107
+ }
1108
+ return 1 - result_t(and_count) / or_count;
1109
+ }
1110
+ };
1111
+
1112
+ /**
1113
+ * @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
1114
+ * Often used in chemistry and biology to compare molecular fingerprints.
1115
+ */
1116
+ template <typename scalar_at = std::uint64_t, typename result_at = float> struct metric_sorensen_gt {
1117
+ using scalar_t = scalar_at;
1118
+ using result_t = result_at;
1119
+ static_assert( //
1120
+ std::is_unsigned<scalar_t>::value ||
1121
+ (std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
1122
+ "Sorensen-Dice distance requires unsigned integral words");
1123
+ static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
1124
+
1125
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
1126
+ constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
1127
+ result_t and_count{};
1128
+ result_t any_count{};
1129
+ #if USEARCH_USE_OPENMP
1130
+ #pragma omp simd reduction(+ : and_count, any_count)
1131
+ #elif defined(USEARCH_DEFINED_CLANG)
1132
+ #pragma clang loop vectorize(enable)
1133
+ #elif defined(USEARCH_DEFINED_GCC)
1134
+ #pragma GCC ivdep
1135
+ #endif
1136
+ for (std::size_t i = 0; i != words; ++i) {
1137
+ and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count();
1138
+ any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
1139
+ }
1140
+ return 1 - 2 * result_t(and_count) / any_count;
1141
+ }
1142
+ };
1143
+
1144
+ /**
1145
+ * @brief Counts the number of matching elements in two unique sorted sets.
1146
+ * Can be used to compute the similarity between two textual documents
1147
+ * using the IDs of tokens present in them.
1148
+ * Similar to `metric_tanimoto_gt` for dense representations.
1149
+ */
1150
+ template <typename scalar_at = std::int32_t, typename result_at = float> struct metric_jaccard_gt {
1151
+ using scalar_t = scalar_at;
1152
+ using result_t = result_at;
1153
+ static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
1154
+
1155
+ inline result_t operator()( //
1156
+ scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
1157
+ result_t intersection{};
1158
+ std::size_t i{};
1159
+ std::size_t j{};
1160
+ while (i != a_length && j != b_length) {
1161
+ intersection += a[i] == b[j];
1162
+ i += a[i] < b[j];
1163
+ j += a[i] >= b[j];
1164
+ }
1165
+ return 1 - intersection / (a_length + b_length - intersection);
1166
+ }
1167
+ };
1168
+
1169
+ /**
1170
+ * @brief Measures Pearson Correlation between two sequences in a single pass.
1171
+ */
1172
+ template <typename scalar_at = float, typename result_at = float> struct metric_pearson_gt {
1173
+ using scalar_t = scalar_at;
1174
+ using result_t = result_at;
1175
+
1176
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
1177
+ // The correlation coefficient can't be defined for one or zero-dimensional data.
1178
+ if (dim <= 1)
1179
+ return 0;
1180
+ // Conventional Pearson Correlation Coefficient definiton subtracts the mean value of each
1181
+ // sequence from each element, before dividing them. WikiPedia article suggests a convenient
1182
+ // single-pass algorithm for calculating sample correlations, though depending on the numbers
1183
+ // involved, it can sometimes be numerically unstable.
1184
+ result_t a_sum{}, b_sum{}, ab_sum{};
1185
+ result_t a_sq_sum{}, b_sq_sum{};
1186
+ #if USEARCH_USE_OPENMP
1187
+ #pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
1188
+ #elif defined(USEARCH_DEFINED_CLANG)
1189
+ #pragma clang loop vectorize(enable)
1190
+ #elif defined(USEARCH_DEFINED_GCC)
1191
+ #pragma GCC ivdep
1192
+ #endif
1193
+ for (std::size_t i = 0; i != dim; ++i) {
1194
+ result_t ai = static_cast<result_t>(a[i]);
1195
+ result_t bi = static_cast<result_t>(b[i]);
1196
+ a_sum += ai;
1197
+ b_sum += bi;
1198
+ ab_sum += ai * bi;
1199
+ a_sq_sum += ai * ai;
1200
+ b_sq_sum += bi * bi;
1201
+ }
1202
+ result_t denom = (dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum);
1203
+ if (denom == 0)
1204
+ return 0;
1205
+ result_t corr = dim * ab_sum - a_sum * b_sum;
1206
+ denom = std::sqrt(denom);
1207
+ // The normal Pearson correlation value is between -1 and 1, but we are looking for a distance.
1208
+ // So instead of returning `corr / denom`, we return `1 - corr / denom`.
1209
+ return 1 - corr / denom;
1210
+ }
1211
+ };
1212
+
1213
+ /**
1214
+ * @brief Measures Jensen-Shannon Divergence between two probability distributions.
1215
+ */
1216
+ template <typename scalar_at = float, typename result_at = float> struct metric_divergence_gt {
1217
+ using scalar_t = scalar_at;
1218
+ using result_t = result_at;
1219
+
1220
+ inline result_t operator()(scalar_t const* p, scalar_t const* q, std::size_t dim) const noexcept {
1221
+ result_t kld_pm{}, kld_qm{};
1222
+ result_t epsilon = std::numeric_limits<result_t>::epsilon();
1223
+ #if USEARCH_USE_OPENMP
1224
+ #pragma omp simd reduction(+ : kld_pm, kld_qm)
1225
+ #elif defined(USEARCH_DEFINED_CLANG)
1226
+ #pragma clang loop vectorize(enable)
1227
+ #elif defined(USEARCH_DEFINED_GCC)
1228
+ #pragma GCC ivdep
1229
+ #endif
1230
+ for (std::size_t i = 0; i != dim; ++i) {
1231
+ result_t pi = static_cast<result_t>(p[i]);
1232
+ result_t qi = static_cast<result_t>(q[i]);
1233
+ result_t mi = (pi + qi) / 2 + epsilon;
1234
+ kld_pm += pi * std::log((pi + epsilon) / mi);
1235
+ kld_qm += qi * std::log((qi + epsilon) / mi);
1236
+ }
1237
+ return (kld_pm + kld_qm) / 2;
1238
+ }
1239
+ };
1240
+
1241
+ struct cos_i8_t {
1242
+ using scalar_t = i8_t;
1243
+ using result_t = f32_t;
1244
+
1245
+ inline result_t operator()(i8_t const* a, i8_t const* b, std::size_t dim) const noexcept {
1246
+ std::int32_t ab{}, a2{}, b2{};
1247
+ #if USEARCH_USE_OPENMP
1248
+ #pragma omp simd reduction(+ : ab, a2, b2)
1249
+ #elif defined(USEARCH_DEFINED_CLANG)
1250
+ #pragma clang loop vectorize(enable)
1251
+ #elif defined(USEARCH_DEFINED_GCC)
1252
+ #pragma GCC ivdep
1253
+ #endif
1254
+ for (std::size_t i = 0; i != dim; i++) {
1255
+ std::int16_t ai{a[i]};
1256
+ std::int16_t bi{b[i]};
1257
+ ab += ai * bi;
1258
+ a2 += square(ai);
1259
+ b2 += square(bi);
1260
+ }
1261
+ result_t a2f = std::sqrt(static_cast<result_t>(a2));
1262
+ result_t b2f = std::sqrt(static_cast<result_t>(b2));
1263
+ return (ab != 0) ? (1.f - ab / (a2f * b2f)) : 0;
1264
+ }
1265
+ };
1266
+
1267
+ struct l2sq_i8_t {
1268
+ using scalar_t = i8_t;
1269
+ using result_t = f32_t;
1270
+
1271
+ inline result_t operator()(i8_t const* a, i8_t const* b, std::size_t dim) const noexcept {
1272
+ std::int32_t ab_deltas_sq{};
1273
+ #if USEARCH_USE_OPENMP
1274
+ #pragma omp simd reduction(+ : ab_deltas_sq)
1275
+ #elif defined(USEARCH_DEFINED_CLANG)
1276
+ #pragma clang loop vectorize(enable)
1277
+ #elif defined(USEARCH_DEFINED_GCC)
1278
+ #pragma GCC ivdep
1279
+ #endif
1280
+ for (std::size_t i = 0; i != dim; i++)
1281
+ ab_deltas_sq += square(std::int16_t(a[i]) - std::int16_t(b[i]));
1282
+ return static_cast<result_t>(ab_deltas_sq);
1283
+ }
1284
+ };
1285
+
1286
+ /**
1287
+ * @brief Haversine distance for the shortest distance between two nodes on
1288
+ * the surface of a 3D sphere, defined with latitude and longitude.
1289
+ */
1290
+ template <typename scalar_at = float, typename result_at = scalar_at> struct metric_haversine_gt {
1291
+ using scalar_t = scalar_at;
1292
+ using result_t = result_at;
1293
+ static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
1294
+
1295
+ inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t = 2) const noexcept {
1296
+ result_t lat_a = a[0], lon_a = a[1];
1297
+ result_t lat_b = b[0], lon_b = b[1];
1298
+
1299
+ result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a) / 2;
1300
+ result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a) / 2;
1301
+
1302
+ result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
1303
+ result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
1304
+
1305
+ result_t x = square(std::sin(lat_delta)) + //
1306
+ std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta));
1307
+
1308
+ return 2 * std::asin(std::sqrt(x));
1309
+ }
1310
+ };
1311
+
1312
+ using distance_punned_t = float;
1313
+ using span_punned_t = span_gt<byte_t const>;
1314
+
1315
+ /**
1316
+ * @brief The signature of the user-defined function.
1317
+ * Can be just two array pointers, precompiled for a specific array length,
1318
+ * or include one or two array sizes as 64-bit unsigned integers.
1319
+ */
1320
+ enum class metric_punned_signature_t {
1321
+ array_array_k = 0,
1322
+ array_array_size_k,
1323
+ };
1324
+
1325
+ /**
1326
+ * @brief Type-punned metric class, which unlike STL's `std::function` avoids any memory allocations.
1327
+ * It also provides additional APIs to check, if SIMD hardware-acceleration is available.
1328
+ * Wraps the `simsimd_metric_punned_t` when available. The auto-vectorized backend otherwise.
1329
+ */
1330
+ class metric_punned_t {
1331
+ public:
1332
+ using scalar_t = byte_t;
1333
+ using result_t = distance_punned_t;
1334
+
1335
+ private:
1336
+ /// In the generalized function API all the are arguments are pointer-sized.
1337
+ using uptr_t = std::size_t;
1338
+ /// Distance function that takes two arrays and returns a scalar.
1339
+ using metric_array_array_t = result_t (*)(uptr_t, uptr_t);
1340
+ /// Distance function that takes two arrays and their length and returns a scalar.
1341
+ using metric_array_array_size_t = result_t (*)(uptr_t, uptr_t, uptr_t);
1342
+ /// Distance function callback, like `metric_array_array_size_t`, but depends on member variables.
1343
+ using metric_rounted_t = result_t (metric_punned_t::*)(uptr_t, uptr_t) const;
1344
+
1345
+ metric_rounted_t metric_routed_ = nullptr;
1346
+ uptr_t metric_ptr_ = 0;
1347
+ uptr_t metric_size_arg_ = 0;
1348
+
1349
+ std::size_t dimensions_ = 0;
1350
+ metric_kind_t metric_kind_ = metric_kind_t::unknown_k;
1351
+ scalar_kind_t scalar_kind_ = scalar_kind_t::unknown_k;
1352
+
1353
+ #if USEARCH_USE_SIMSIMD
1354
+ simsimd_capability_t isa_kind_ = simsimd_cap_serial_k;
1355
+ #endif
1356
+
1357
+ public:
1358
+ /**
1359
+ * @brief Computes the distance between two vectors of fixed length.
1360
+ *
1361
+ * ! This is the only relevant function in the object. Everything else is just dynamic dispatch logic.
1362
+ */
1363
+ inline result_t operator()(byte_t const* a, byte_t const* b) const noexcept {
1364
+ return (this->*metric_routed_)(reinterpret_cast<uptr_t>(a), reinterpret_cast<uptr_t>(b));
1365
+ }
1366
+
1367
+ inline metric_punned_t() noexcept = default;
1368
+ inline metric_punned_t(metric_punned_t const&) noexcept = default;
1369
+ inline metric_punned_t& operator=(metric_punned_t const&) noexcept = default;
1370
+ inline metric_punned_t( //
1371
+ std::size_t dimensions, //
1372
+ metric_kind_t metric_kind = metric_kind_t::l2sq_k, //
1373
+ scalar_kind_t scalar_kind = scalar_kind_t::f32_k) noexcept
1374
+ : metric_routed_(&metric_punned_t::invoke_array_array_size), metric_size_arg_(dimensions),
1375
+ dimensions_(dimensions), metric_kind_(metric_kind), scalar_kind_(scalar_kind) {
1376
+
1377
+ #if USEARCH_USE_SIMSIMD
1378
+ if (!configure_with_simsimd())
1379
+ configure_with_autovec();
1380
+ #else
1381
+ configure_with_autovec();
1382
+ #endif
1383
+
1384
+ if (scalar_kind == scalar_kind_t::b1x8_k)
1385
+ metric_size_arg_ = divide_round_up<CHAR_BIT>(dimensions_);
1386
+ }
1387
+
1388
+ inline metric_punned_t( //
1389
+ std::size_t dimensions, //
1390
+ std::uintptr_t metric_uintptr, metric_punned_signature_t signature, //
1391
+ metric_kind_t metric_kind, //
1392
+ scalar_kind_t scalar_kind) noexcept
1393
+ : metric_routed_(signature == metric_punned_signature_t::array_array_k
1394
+ ? &metric_punned_t::invoke_array_array
1395
+ : &metric_punned_t::invoke_array_array_size),
1396
+ metric_ptr_(metric_uintptr), metric_size_arg_(dimensions), dimensions_(dimensions), metric_kind_(metric_kind),
1397
+ scalar_kind_(scalar_kind) {
1398
+
1399
+ if (scalar_kind == scalar_kind_t::b1x8_k)
1400
+ metric_size_arg_ = divide_round_up<CHAR_BIT>(dimensions_);
1401
+ }
1402
+
1403
+ inline std::size_t dimensions() const noexcept { return dimensions_; }
1404
+ inline metric_kind_t metric_kind() const noexcept { return metric_kind_; }
1405
+ inline scalar_kind_t scalar_kind() const noexcept { return scalar_kind_; }
1406
+ inline explicit operator bool() const noexcept { return metric_routed_ && metric_ptr_; }
1407
+
1408
+ /**
1409
+ * @brief Checks fi we've failed to initialized the metric with provided arguments.
1410
+ *
1411
+ * It's different from `operator bool()` when it comes to explicitly uninitialized metrics.
1412
+ * It's a common case, where a NULL state is created only to be overwritten later, when
1413
+ * we recover an old index state from a file or a network.
1414
+ */
1415
+ inline bool missing() const noexcept { return !bool(*this) && metric_kind_ != metric_kind_t::unknown_k; }
1416
+
1417
+ inline char const* isa_name() const noexcept {
1418
+ if (!*this)
1419
+ return "uninitialized";
1420
+
1421
+ #if USEARCH_USE_SIMSIMD
1422
+ switch (isa_kind_) {
1423
+ case simsimd_cap_serial_k: return "serial";
1424
+ case simsimd_cap_neon_k: return "neon";
1425
+ case simsimd_cap_sve_k: return "sve";
1426
+ case simsimd_cap_haswell_k: return "haswell";
1427
+ case simsimd_cap_skylake_k: return "skylake";
1428
+ case simsimd_cap_ice_k: return "ice";
1429
+ case simsimd_cap_sapphire_k: return "sapphire";
1430
+ default: return "unknown";
1431
+ }
1432
+ #endif
1433
+ return "serial";
1434
+ }
1435
+
1436
+ inline std::size_t bytes_per_vector() const noexcept {
1437
+ return divide_round_up<CHAR_BIT>(dimensions_ * bits_per_scalar(scalar_kind_));
1438
+ }
1439
+
1440
+ inline std::size_t scalar_words() const noexcept {
1441
+ return divide_round_up(dimensions_ * bits_per_scalar(scalar_kind_), bits_per_scalar_word(scalar_kind_));
1442
+ }
1443
+
1444
+ private:
1445
+ #if USEARCH_USE_SIMSIMD
1446
+ bool configure_with_simsimd(simsimd_capability_t simd_caps) noexcept {
1447
+ simsimd_metric_kind_t kind = simsimd_metric_unknown_k;
1448
+ simsimd_datatype_t datatype = simsimd_datatype_unknown_k;
1449
+ simsimd_capability_t allowed = simsimd_cap_any_k;
1450
+ switch (metric_kind_) {
1451
+ case metric_kind_t::ip_k: kind = simsimd_metric_dot_k; break;
1452
+ case metric_kind_t::cos_k: kind = simsimd_metric_cos_k; break;
1453
+ case metric_kind_t::l2sq_k: kind = simsimd_metric_l2sq_k; break;
1454
+ case metric_kind_t::hamming_k: kind = simsimd_metric_hamming_k; break;
1455
+ case metric_kind_t::tanimoto_k: kind = simsimd_metric_jaccard_k; break;
1456
+ case metric_kind_t::jaccard_k: kind = simsimd_metric_jaccard_k; break;
1457
+ default: break;
1458
+ }
1459
+ switch (scalar_kind_) {
1460
+ case scalar_kind_t::f32_k: datatype = simsimd_datatype_f32_k; break;
1461
+ case scalar_kind_t::f64_k: datatype = simsimd_datatype_f64_k; break;
1462
+ case scalar_kind_t::f16_k: datatype = simsimd_datatype_f16_k; break;
1463
+ case scalar_kind_t::i8_k: datatype = simsimd_datatype_i8_k; break;
1464
+ case scalar_kind_t::b1x8_k: datatype = simsimd_datatype_b8_k; break;
1465
+ default: break;
1466
+ }
1467
+ simsimd_metric_punned_t simd_metric = NULL;
1468
+ simsimd_capability_t simd_kind = simsimd_cap_any_k;
1469
+ simsimd_find_metric_punned(kind, datatype, simd_caps, allowed, &simd_metric, &simd_kind);
1470
+ if (simd_metric == nullptr)
1471
+ return false;
1472
+
1473
+ std::memcpy(&metric_ptr_, &simd_metric, sizeof(simd_metric));
1474
+ metric_routed_ = metric_kind_ == metric_kind_t::ip_k
1475
+ ? reinterpret_cast<metric_rounted_t>(&metric_punned_t::invoke_simsimd_reverse)
1476
+ : reinterpret_cast<metric_rounted_t>(&metric_punned_t::invoke_simsimd);
1477
+ isa_kind_ = simd_kind;
1478
+ return true;
1479
+ }
1480
+ bool configure_with_simsimd() noexcept {
1481
+ static simsimd_capability_t static_capabilities = simsimd_capabilities();
1482
+ return configure_with_simsimd(static_capabilities);
1483
+ }
1484
+ result_t invoke_simsimd(uptr_t a, uptr_t b) const noexcept {
1485
+ simsimd_distance_t result;
1486
+ // Here `reinterpret_cast` raises warning... we know what we are doing!
1487
+ auto function_pointer = (simsimd_metric_punned_t)(metric_ptr_);
1488
+ function_pointer(reinterpret_cast<void const*>(a), reinterpret_cast<void const*>(b), metric_size_arg_, &result);
1489
+ return (result_t)result;
1490
+ }
1491
+ result_t invoke_simsimd_reverse(uptr_t a, uptr_t b) const noexcept { return 1 - invoke_simsimd(a, b); }
1492
+ #else
1493
+ bool configure_with_simsimd() noexcept { return false; }
1494
+ #endif
1495
+ result_t invoke_array_array_size(uptr_t a, uptr_t b) const noexcept {
1496
+ auto function_pointer = (metric_array_array_size_t)(metric_ptr_);
1497
+ result_t result = function_pointer(a, b, metric_size_arg_);
1498
+ return result;
1499
+ }
1500
+ result_t invoke_array_array(uptr_t a, uptr_t b) const noexcept {
1501
+ auto function_pointer = (metric_array_array_t)(metric_ptr_);
1502
+ result_t result = function_pointer(a, b);
1503
+ return result;
1504
+ }
1505
+ void configure_with_autovec() noexcept {
1506
+ switch (metric_kind_) {
1507
+ case metric_kind_t::ip_k: {
1508
+ switch (scalar_kind_) {
1509
+ case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f32_t>>; break;
1510
+ case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f16_t, f32_t>>; break;
1511
+ case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<i8_t, f32_t>>; break;
1512
+ case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f64_t>>; break;
1513
+ default: metric_ptr_ = 0; break;
1514
+ }
1515
+ break;
1516
+ }
1517
+ case metric_kind_t::cos_k: {
1518
+ switch (scalar_kind_) {
1519
+ case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f32_t>>; break;
1520
+ case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f16_t, f32_t>>; break;
1521
+ case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<i8_t, f32_t>>; break;
1522
+ case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f64_t>>; break;
1523
+ default: metric_ptr_ = 0; break;
1524
+ }
1525
+ break;
1526
+ }
1527
+ case metric_kind_t::l2sq_k: {
1528
+ switch (scalar_kind_) {
1529
+ case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f32_t>>; break;
1530
+ case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f16_t, f32_t>>; break;
1531
+ case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<i8_t, f32_t>>; break;
1532
+ case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f64_t>>; break;
1533
+ default: metric_ptr_ = 0; break;
1534
+ }
1535
+ break;
1536
+ }
1537
+ case metric_kind_t::pearson_k: {
1538
+ switch (scalar_kind_) {
1539
+ case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<i8_t, f32_t>>; break;
1540
+ case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f16_t, f32_t>>; break;
1541
+ case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f32_t>>; break;
1542
+ case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f64_t>>; break;
1543
+ default: metric_ptr_ = 0; break;
1544
+ }
1545
+ break;
1546
+ }
1547
+ case metric_kind_t::haversine_k: {
1548
+ switch (scalar_kind_) {
1549
+ case scalar_kind_t::f16_k:
1550
+ metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f16_t, f32_t>>;
1551
+ break;
1552
+ case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f32_t>>; break;
1553
+ case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f64_t>>; break;
1554
+ default: metric_ptr_ = 0; break;
1555
+ }
1556
+ break;
1557
+ }
1558
+ case metric_kind_t::divergence_k: {
1559
+ switch (scalar_kind_) {
1560
+ case scalar_kind_t::f16_k:
1561
+ metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f16_t, f32_t>>;
1562
+ break;
1563
+ case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f32_t>>; break;
1564
+ case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f64_t>>; break;
1565
+ default: metric_ptr_ = 0; break;
1566
+ }
1567
+ break;
1568
+ }
1569
+ case metric_kind_t::jaccard_k: // Equivalent to Tanimoto
1570
+ case metric_kind_t::tanimoto_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_tanimoto_gt<b1x8_t>>; break;
1571
+ case metric_kind_t::hamming_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_hamming_gt<b1x8_t>>; break;
1572
+ case metric_kind_t::sorensen_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_sorensen_gt<b1x8_t>>; break;
1573
+ default: return;
1574
+ }
1575
+ }
1576
+
1577
+ template <typename typed_at>
1578
+ inline static result_t equidimensional_(uptr_t a, uptr_t b, uptr_t a_dimensions) noexcept {
1579
+ using scalar_t = typename typed_at::scalar_t;
1580
+ return static_cast<result_t>(typed_at{}((scalar_t const*)a, (scalar_t const*)b, a_dimensions));
1581
+ }
1582
+ };
1583
+
1584
+ /**
1585
+ * @brief View over a potentially-strided memory buffer, containing a row-major matrix.
1586
+ */
1587
+ template <typename scalar_at> //
1588
+ class vectors_view_gt {
1589
+ using scalar_t = scalar_at;
1590
+
1591
+ scalar_t const* begin_{};
1592
+ std::size_t dimensions_{};
1593
+ std::size_t count_{};
1594
+ std::size_t stride_bytes_{};
1595
+
1596
+ public:
1597
+ vectors_view_gt() noexcept = default;
1598
+ vectors_view_gt(vectors_view_gt const&) noexcept = default;
1599
+ vectors_view_gt& operator=(vectors_view_gt const&) noexcept = default;
1600
+
1601
+ vectors_view_gt(scalar_t const* begin, std::size_t dimensions, std::size_t count = 1) noexcept
1602
+ : vectors_view_gt(begin, dimensions, count, dimensions * sizeof(scalar_at)) {}
1603
+
1604
+ vectors_view_gt(scalar_t const* begin, std::size_t dimensions, std::size_t count, std::size_t stride_bytes) noexcept
1605
+ : begin_(begin), dimensions_(dimensions), count_(count), stride_bytes_(stride_bytes) {}
1606
+
1607
+ explicit operator bool() const noexcept { return begin_; }
1608
+ std::size_t size() const noexcept { return count_; }
1609
+ std::size_t dimensions() const noexcept { return dimensions_; }
1610
+ std::size_t stride() const noexcept { return stride_bytes_; }
1611
+ scalar_t const* data() const noexcept { return begin_; }
1612
+ scalar_t const* at(std::size_t i) const noexcept {
1613
+ return reinterpret_cast<scalar_t const*>(reinterpret_cast<byte_t const*>(begin_) + i * stride_bytes_);
1614
+ }
1615
+ };
1616
+
1617
+ struct exact_offset_and_distance_t {
1618
+ u32_t offset;
1619
+ f32_t distance;
1620
+ };
1621
+
1622
+ using exact_search_results_t = vectors_view_gt<exact_offset_and_distance_t>;
1623
+
1624
+ /**
1625
+ * @brief Helper-structure for exact search operations.
1626
+ * Perfect if you have @b <1M vectors and @b <100 queries per call.
1627
+ *
1628
+ * Uses a 3-step procedure to minimize:
1629
+ * - cache-misses on vector lookups,
1630
+ * - multi-threaded contention on concurrent writes.
1631
+ */
1632
+ class exact_search_t {
1633
+
1634
+ inline static bool smaller_distance(exact_offset_and_distance_t a, exact_offset_and_distance_t b) noexcept {
1635
+ return a.distance < b.distance;
1636
+ }
1637
+
1638
+ using keys_and_distances_t = buffer_gt<exact_offset_and_distance_t>;
1639
+ keys_and_distances_t keys_and_distances;
1640
+
1641
+ public:
1642
+ template <typename scalar_at, typename executor_at = dummy_executor_t, typename progress_at = dummy_progress_t>
1643
+ exact_search_results_t operator()( //
1644
+ vectors_view_gt<scalar_at> dataset, vectors_view_gt<scalar_at> queries, //
1645
+ std::size_t wanted, metric_punned_t const& metric, //
1646
+ executor_at&& executor = executor_at{}, progress_at&& progress = progress_at{}) {
1647
+ return operator()( //
1648
+ metric, //
1649
+ reinterpret_cast<byte_t const*>(dataset.data()), dataset.size(), dataset.stride(), //
1650
+ reinterpret_cast<byte_t const*>(queries.data()), queries.size(), queries.stride(), //
1651
+ wanted, executor, progress);
1652
+ }
1653
+
1654
+ template <typename executor_at = dummy_executor_t, typename progress_at = dummy_progress_t>
1655
+ exact_search_results_t operator()( //
1656
+ byte_t const* dataset_data, std::size_t dataset_count, std::size_t dataset_stride, //
1657
+ byte_t const* queries_data, std::size_t queries_count, std::size_t queries_stride, //
1658
+ std::size_t wanted, metric_punned_t const& metric, executor_at&& executor = executor_at{},
1659
+ progress_at&& progress = progress_at{}) {
1660
+
1661
+ // Allocate temporary memory to store the distance matrix
1662
+ // Previous version didn't need temporary memory, but the performance was much lower.
1663
+ // In the new design we keep two buffers - original and transposed, as in-place transpositions
1664
+ // of non-rectangular matrixes is expensive.
1665
+ std::size_t tasks_count = dataset_count * queries_count;
1666
+ if (keys_and_distances.size() < tasks_count * 2)
1667
+ keys_and_distances = keys_and_distances_t(tasks_count * 2);
1668
+ if (keys_and_distances.size() < tasks_count * 2)
1669
+ return {};
1670
+
1671
+ exact_offset_and_distance_t* keys_and_distances_per_dataset = keys_and_distances.data();
1672
+ exact_offset_and_distance_t* keys_and_distances_per_query = keys_and_distances_per_dataset + tasks_count;
1673
+
1674
+ // §1. Compute distances in a data-parallel fashion
1675
+ std::atomic<std::size_t> processed{0};
1676
+ executor.dynamic(dataset_count, [&](std::size_t thread_idx, std::size_t dataset_idx) {
1677
+ byte_t const* dataset = dataset_data + dataset_idx * dataset_stride;
1678
+ for (std::size_t query_idx = 0; query_idx != queries_count; ++query_idx) {
1679
+ byte_t const* query = queries_data + query_idx * queries_stride;
1680
+ auto distance = metric(dataset, query);
1681
+ std::size_t task_idx = queries_count * dataset_idx + query_idx;
1682
+ keys_and_distances_per_dataset[task_idx].offset = static_cast<u32_t>(dataset_idx);
1683
+ keys_and_distances_per_dataset[task_idx].distance = static_cast<f32_t>(distance);
1684
+ }
1685
+
1686
+ // It's more efficient in this case to report progress from a single thread
1687
+ processed += queries_count;
1688
+ if (thread_idx == 0)
1689
+ if (!progress(processed.load(), tasks_count))
1690
+ return false;
1691
+ return true;
1692
+ });
1693
+ if (processed.load() != tasks_count)
1694
+ return {};
1695
+
1696
+ // §2. Transpose in a single thread to avoid contention writing into the same memory buffers
1697
+ for (std::size_t query_idx = 0; query_idx != queries_count; ++query_idx) {
1698
+ for (std::size_t dataset_idx = 0; dataset_idx != dataset_count; ++dataset_idx) {
1699
+ std::size_t from_idx = queries_count * dataset_idx + query_idx;
1700
+ std::size_t to_idx = dataset_count * query_idx + dataset_idx;
1701
+ keys_and_distances_per_query[to_idx] = keys_and_distances_per_dataset[from_idx];
1702
+ }
1703
+ }
1704
+
1705
+ // §3. Partial-sort every query result
1706
+ executor.fixed(queries_count, [&](std::size_t, std::size_t query_idx) {
1707
+ auto start = keys_and_distances_per_query + dataset_count * query_idx;
1708
+ if (wanted > 1) {
1709
+ // TODO: Consider alternative sorting approaches
1710
+ // radix_sort(start, start + dataset_count, wanted);
1711
+ // std::sort(start, start + dataset_count, &smaller_distance);
1712
+ std::partial_sort(start, start + wanted, start + dataset_count, &smaller_distance);
1713
+ } else {
1714
+ auto min_it = std::min_element(start, start + dataset_count, &smaller_distance);
1715
+ if (min_it != start)
1716
+ std::swap(*min_it, *start);
1717
+ }
1718
+ });
1719
+
1720
+ // At the end report the latest numbers, because the reporter thread may be finished earlier
1721
+ progress(tasks_count, tasks_count);
1722
+ return {keys_and_distances_per_query, wanted, queries_count,
1723
+ dataset_count * sizeof(exact_offset_and_distance_t)};
1724
+ }
1725
+ };
1726
+
1727
+ /**
1728
+ * @brief C++11 Multi-Hash-Set with Linear Probing.
1729
+ *
1730
+ * - Allows multiple equivalent values,
1731
+ * - Supports transparent hashing and equality operator.
1732
+ * - Doesn't throw exceptions, if forbidden.
1733
+ * - Doesn't need reserving a value for deletions.
1734
+ *
1735
+ * @section Layout
1736
+ *
1737
+ * For every slot we store 2 extra bits for 3 possible states: empty, populated, or deleted.
1738
+ * With linear probing the hashes at the end of the populated region will spill into its first half.
1739
+ */
1740
+ template <typename element_at, typename hash_at, typename equals_at, typename allocator_at = std::allocator<char>>
1741
+ class flat_hash_multi_set_gt {
1742
+ public:
1743
+ using element_t = element_at;
1744
+ using hash_t = hash_at;
1745
+ using equals_t = equals_at;
1746
+ using allocator_t = allocator_at;
1747
+
1748
+ static constexpr std::size_t slots_per_bucket() { return 64; }
1749
+ static constexpr std::size_t bytes_per_bucket() {
1750
+ return slots_per_bucket() * sizeof(element_t) + sizeof(bucket_header_t);
1751
+ }
1752
+
1753
+ private:
1754
+ struct bucket_header_t {
1755
+ std::uint64_t populated{};
1756
+ std::uint64_t deleted{};
1757
+ };
1758
+ char* data_ = nullptr;
1759
+ std::size_t buckets_ = 0;
1760
+ std::size_t populated_slots_ = 0;
1761
+ /// @brief Number of slots
1762
+ std::size_t capacity_slots_ = 0;
1763
+
1764
+ struct slot_ref_t {
1765
+ bucket_header_t& header;
1766
+ std::uint64_t mask;
1767
+ element_t& element;
1768
+ };
1769
+
1770
+ slot_ref_t slot_ref(char* data, std::size_t slot_index) const noexcept {
1771
+ std::size_t bucket_index = slot_index / slots_per_bucket();
1772
+ std::size_t in_bucket_index = slot_index % slots_per_bucket();
1773
+ auto bucket_pointer = data + bytes_per_bucket() * bucket_index;
1774
+ auto slot_pointer = bucket_pointer + sizeof(bucket_header_t) + sizeof(element_t) * in_bucket_index;
1775
+ return {
1776
+ *reinterpret_cast<bucket_header_t*>(bucket_pointer),
1777
+ static_cast<std::uint64_t>(1ull) << in_bucket_index,
1778
+ *reinterpret_cast<element_t*>(slot_pointer),
1779
+ };
1780
+ }
1781
+
1782
+ slot_ref_t slot_ref(std::size_t slot_index) const noexcept { return slot_ref(data_, slot_index); }
1783
+
1784
+ bool populate_slot(slot_ref_t slot, element_t const& new_element) {
1785
+ if (slot.header.populated & slot.mask) {
1786
+ slot.element = new_element;
1787
+ slot.header.deleted &= ~slot.mask;
1788
+ return false;
1789
+ } else {
1790
+ new (&slot.element) element_t(new_element);
1791
+ slot.header.populated |= slot.mask;
1792
+ return true;
1793
+ }
1794
+ }
1795
+
1796
+ public:
1797
+ std::size_t size() const noexcept { return populated_slots_; }
1798
+ std::size_t capacity() const noexcept { return capacity_slots_; }
1799
+
1800
+ flat_hash_multi_set_gt() noexcept {}
1801
+ ~flat_hash_multi_set_gt() noexcept { reset(); }
1802
+
1803
+ flat_hash_multi_set_gt(flat_hash_multi_set_gt const& other) {
1804
+
1805
+ // On Windows allocating a zero-size array would fail
1806
+ if (!other.buckets_) {
1807
+ reset();
1808
+ return;
1809
+ }
1810
+
1811
+ // Allocate new memory
1812
+ data_ = (char*)allocator_t{}.allocate(other.buckets_ * bytes_per_bucket());
1813
+ if (!data_)
1814
+ throw std::bad_alloc();
1815
+
1816
+ // Copy metadata
1817
+ buckets_ = other.buckets_;
1818
+ populated_slots_ = other.populated_slots_;
1819
+ capacity_slots_ = other.capacity_slots_;
1820
+
1821
+ // Initialize new buckets to empty
1822
+ std::memset(data_, 0, buckets_ * bytes_per_bucket());
1823
+
1824
+ // Copy elements and bucket headers
1825
+ for (std::size_t i = 0; i < capacity_slots_; ++i) {
1826
+ slot_ref_t old_slot = other.slot_ref(i);
1827
+ if ((old_slot.header.populated & old_slot.mask) && !(old_slot.header.deleted & old_slot.mask)) {
1828
+ slot_ref_t new_slot = slot_ref(i);
1829
+ populate_slot(new_slot, old_slot.element);
1830
+ }
1831
+ }
1832
+ }
1833
+
1834
+ flat_hash_multi_set_gt& operator=(flat_hash_multi_set_gt const& other) {
1835
+
1836
+ // On Windows allocating a zero-size array would fail
1837
+ if (!other.buckets_) {
1838
+ reset();
1839
+ return *this;
1840
+ }
1841
+
1842
+ // Handle self-assignment
1843
+ if (this == &other)
1844
+ return *this;
1845
+
1846
+ // Clear existing data
1847
+ clear();
1848
+ if (data_)
1849
+ allocator_t{}.deallocate(data_, buckets_ * bytes_per_bucket());
1850
+
1851
+ // Allocate new memory
1852
+ data_ = (char*)allocator_t{}.allocate(other.buckets_ * bytes_per_bucket());
1853
+ if (!data_)
1854
+ throw std::bad_alloc();
1855
+
1856
+ // Copy metadata
1857
+ buckets_ = other.buckets_;
1858
+ populated_slots_ = other.populated_slots_;
1859
+ capacity_slots_ = other.capacity_slots_;
1860
+
1861
+ // Initialize new buckets to empty
1862
+ std::memset(data_, 0, buckets_ * bytes_per_bucket());
1863
+
1864
+ // Copy elements and bucket headers
1865
+ for (std::size_t i = 0; i < capacity_slots_; ++i) {
1866
+ slot_ref_t old_slot = other.slot_ref(i);
1867
+ if ((old_slot.header.populated & old_slot.mask) && !(old_slot.header.deleted & old_slot.mask)) {
1868
+ slot_ref_t new_slot = slot_ref(i);
1869
+ populate_slot(new_slot, old_slot.element);
1870
+ }
1871
+ }
1872
+
1873
+ return *this;
1874
+ }
1875
+
1876
+ void clear() noexcept {
1877
+ // Call the destructors
1878
+ for (std::size_t i = 0; i < capacity_slots_; ++i) {
1879
+ slot_ref_t slot = slot_ref(i);
1880
+ if ((slot.header.populated & slot.mask) & (~slot.header.deleted & slot.mask))
1881
+ slot.element.~element_t();
1882
+ }
1883
+
1884
+ // Reset populated slots count
1885
+ if (data_)
1886
+ std::memset(data_, 0, buckets_ * bytes_per_bucket());
1887
+ populated_slots_ = 0;
1888
+ }
1889
+
1890
+ void reset() noexcept {
1891
+ clear(); // Clear all elements
1892
+ if (data_)
1893
+ allocator_t{}.deallocate(data_, buckets_ * bytes_per_bucket());
1894
+ buckets_ = 0;
1895
+ populated_slots_ = 0;
1896
+ capacity_slots_ = 0;
1897
+ }
1898
+
1899
+ bool try_reserve(std::size_t capacity) noexcept {
1900
+ if (capacity * 3u <= capacity_slots_ * 2u)
1901
+ return true;
1902
+
1903
+ // Calculate new sizes
1904
+ std::size_t new_slots = ceil2((capacity * 3ul) / 2ul);
1905
+ std::size_t new_buckets = divide_round_up<slots_per_bucket()>(new_slots);
1906
+ new_slots = new_buckets * slots_per_bucket(); // This must be a power of two!
1907
+ std::size_t new_bytes = new_buckets * bytes_per_bucket();
1908
+
1909
+ // Allocate new memory
1910
+ char* new_data = (char*)allocator_t{}.allocate(new_bytes);
1911
+ if (!new_data)
1912
+ return false;
1913
+
1914
+ // Initialize new buckets to empty
1915
+ std::memset(new_data, 0, new_bytes);
1916
+
1917
+ // Rehash and copy existing elements to new_data
1918
+ hash_t hasher;
1919
+ for (std::size_t i = 0; i < capacity_slots_; ++i) {
1920
+ slot_ref_t old_slot = slot_ref(i);
1921
+ if ((~old_slot.header.populated & old_slot.mask) | (old_slot.header.deleted & old_slot.mask))
1922
+ continue;
1923
+
1924
+ // Rehash
1925
+ std::size_t hash_value = hasher(old_slot.element);
1926
+ std::size_t new_slot_index = hash_value & (new_slots - 1);
1927
+
1928
+ // Linear probing to find an empty slot in new_data
1929
+ while (true) {
1930
+ slot_ref_t new_slot = slot_ref(new_data, new_slot_index);
1931
+ if (!(new_slot.header.populated & new_slot.mask) || (new_slot.header.deleted & new_slot.mask)) {
1932
+ populate_slot(new_slot, std::move(old_slot.element));
1933
+ new_slot.header.populated |= new_slot.mask;
1934
+ break;
1935
+ }
1936
+ new_slot_index = (new_slot_index + 1) & (new_slots - 1);
1937
+ }
1938
+ }
1939
+
1940
+ // Deallocate old data and update pointers and sizes
1941
+ if (data_)
1942
+ allocator_t{}.deallocate(data_, buckets_ * bytes_per_bucket());
1943
+ data_ = new_data;
1944
+ buckets_ = new_buckets;
1945
+ capacity_slots_ = new_slots;
1946
+
1947
+ return true;
1948
+ }
1949
+
1950
+ template <typename query_at> class equal_iterator_gt {
1951
+ public:
1952
+ using iterator_category = std::forward_iterator_tag;
1953
+ using value_type = element_t;
1954
+ using difference_type = std::ptrdiff_t;
1955
+ using pointer = element_t*;
1956
+ using reference = element_t&;
1957
+
1958
+ equal_iterator_gt(std::size_t index, flat_hash_multi_set_gt* parent, query_at const& query,
1959
+ equals_t const& equals)
1960
+ : index_(index), parent_(parent), query_(query), equals_(equals) {}
1961
+
1962
+ // Pre-increment
1963
+ equal_iterator_gt& operator++() {
1964
+ do {
1965
+ index_ = (index_ + 1) & (parent_->capacity_slots_ - 1);
1966
+ } while (!equals_(parent_->slot_ref(index_).element, query_) &&
1967
+ (parent_->slot_ref(index_).header.populated & parent_->slot_ref(index_).mask));
1968
+ return *this;
1969
+ }
1970
+
1971
+ equal_iterator_gt operator++(int) {
1972
+ equal_iterator_gt temp = *this;
1973
+ ++(*this);
1974
+ return temp;
1975
+ }
1976
+
1977
+ reference operator*() { return parent_->slot_ref(index_).element; }
1978
+ pointer operator->() { return &parent_->slot_ref(index_).element; }
1979
+ bool operator!=(equal_iterator_gt const& other) const { return !(*this == other); }
1980
+ bool operator==(equal_iterator_gt const& other) const {
1981
+ return index_ == other.index_ && parent_ == other.parent_;
1982
+ }
1983
+
1984
+ private:
1985
+ std::size_t index_;
1986
+ flat_hash_multi_set_gt* parent_;
1987
+ query_at query_; // Store the query object
1988
+ equals_t equals_; // Store the equals functor
1989
+ };
1990
+
1991
+ /**
1992
+ * @brief Returns an iterator range of all elements matching the given query.
1993
+ *
1994
+ * Technically, the second iterator points to the first empty slot after a
1995
+ * range of equal values and non-equal values with similar hashes.
1996
+ */
1997
+ template <typename query_at>
1998
+ std::pair<equal_iterator_gt<query_at>, equal_iterator_gt<query_at>>
1999
+ equal_range(query_at const& query) const noexcept {
2000
+
2001
+ equals_t equals;
2002
+ auto this_ptr = const_cast<flat_hash_multi_set_gt*>(this);
2003
+ auto end = equal_iterator_gt<query_at>(capacity_slots_, this_ptr, query, equals);
2004
+ if (!capacity_slots_)
2005
+ return {end, end};
2006
+
2007
+ hash_t hasher;
2008
+ std::size_t hash_value = hasher(query);
2009
+ std::size_t first_equal_index = hash_value & (capacity_slots_ - 1);
2010
+ std::size_t const start_index = first_equal_index;
2011
+
2012
+ // Linear probing to find the first equal element
2013
+ do {
2014
+ slot_ref_t slot = slot_ref(first_equal_index);
2015
+ if (slot.header.populated & ~slot.header.deleted & slot.mask) {
2016
+ if (equals(slot.element, query))
2017
+ break;
2018
+ }
2019
+ // Stop if we find an empty slot
2020
+ else if (~slot.header.populated & slot.mask)
2021
+ return {end, end};
2022
+
2023
+ // Move to the next slot
2024
+ first_equal_index = (first_equal_index + 1) & (capacity_slots_ - 1);
2025
+ } while (first_equal_index != start_index);
2026
+
2027
+ // If no matching element was found, return end iterators
2028
+ if (first_equal_index == capacity_slots_)
2029
+ return {end, end};
2030
+
2031
+ // Start from the first matching element and find the end of the populated range
2032
+ std::size_t first_empty_index = first_equal_index;
2033
+ do {
2034
+ first_empty_index = (first_empty_index + 1) & (capacity_slots_ - 1);
2035
+ slot_ref_t slot = slot_ref(first_empty_index);
2036
+
2037
+ // If we find an empty slot, this is our end
2038
+ if (~slot.header.populated & slot.mask)
2039
+ break;
2040
+ } while (first_empty_index != start_index);
2041
+
2042
+ return {equal_iterator_gt<query_at>(first_equal_index, this_ptr, query, equals),
2043
+ equal_iterator_gt<query_at>(first_empty_index, this_ptr, query, equals)};
2044
+ }
2045
+
2046
+ template <typename similar_at> bool pop_first(similar_at&& query, element_t& popped_value) noexcept {
2047
+
2048
+ if (!capacity_slots_)
2049
+ return false;
2050
+
2051
+ hash_t hasher;
2052
+ equals_t equals;
2053
+ std::size_t hash_value = hasher(query);
2054
+ std::size_t slot_index = hash_value & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
2055
+ std::size_t start_index = slot_index; // To detect loop in probing
2056
+
2057
+ // Linear probing to find the first match
2058
+ do {
2059
+ slot_ref_t slot = slot_ref(slot_index);
2060
+ if (slot.header.populated & slot.mask) {
2061
+ if ((~slot.header.deleted & slot.mask) && equals(slot.element, query)) {
2062
+ // Found a match, mark as deleted
2063
+ slot.header.deleted |= slot.mask;
2064
+ --populated_slots_;
2065
+ popped_value = slot.element;
2066
+ return true; // Successfully removed
2067
+ }
2068
+ } else {
2069
+ // Stop if we find an empty slot
2070
+ break;
2071
+ }
2072
+
2073
+ // Move to the next slot
2074
+ slot_index = (slot_index + 1) & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
2075
+ } while (slot_index != start_index);
2076
+
2077
+ return false; // No match found
2078
+ }
2079
+
2080
+ template <typename similar_at> std::size_t erase(similar_at&& query) noexcept {
2081
+
2082
+ if (!capacity_slots_)
2083
+ return 0;
2084
+
2085
+ hash_t hasher;
2086
+ equals_t equals;
2087
+ std::size_t hash_value = hasher(query);
2088
+ std::size_t slot_index = hash_value & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
2089
+ std::size_t const start_index = slot_index; // To detect loop in probing
2090
+ std::size_t count = 0; // Count of elements removed
2091
+
2092
+ // Linear probing to find all matches
2093
+ do {
2094
+ slot_ref_t slot = slot_ref(slot_index);
2095
+ if (slot.header.populated & slot.mask) {
2096
+ if ((~slot.header.deleted & slot.mask) && equals(slot.element, query)) {
2097
+ // Found a match, mark as deleted
2098
+ slot.header.deleted |= slot.mask;
2099
+ --populated_slots_;
2100
+ ++count; // Increment count of elements removed
2101
+ }
2102
+ } else {
2103
+ // Stop if we find an empty slot
2104
+ break;
2105
+ }
2106
+
2107
+ // Move to the next slot
2108
+ slot_index = (slot_index + 1) & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
2109
+ } while (slot_index != start_index);
2110
+
2111
+ return count; // Return the number of elements removed
2112
+ }
2113
+
2114
+ template <typename similar_at> element_t const* find(similar_at&& query) const noexcept {
2115
+
2116
+ if (!capacity_slots_)
2117
+ return nullptr;
2118
+
2119
+ hash_t hasher;
2120
+ equals_t equals;
2121
+ std::size_t hash_value = hasher(query);
2122
+ std::size_t slot_index = hash_value & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
2123
+ std::size_t start_index = slot_index; // To detect loop in probing
2124
+
2125
+ // Linear probing to find the first match
2126
+ do {
2127
+ slot_ref_t slot = slot_ref(slot_index);
2128
+ if (slot.header.populated & slot.mask) {
2129
+ if ((~slot.header.deleted & slot.mask) && equals(slot.element, query))
2130
+ return &slot.element; // Found a match, return pointer to the element
2131
+ } else {
2132
+ // Stop if we find an empty slot
2133
+ break;
2134
+ }
2135
+
2136
+ // Move to the next slot
2137
+ slot_index = (slot_index + 1) & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
2138
+ } while (slot_index != start_index);
2139
+
2140
+ return nullptr; // No match found
2141
+ }
2142
+
2143
+ element_t const* end() const noexcept { return nullptr; }
2144
+
2145
+ template <typename func_at> void for_each(func_at&& func) const {
2146
+ for (std::size_t bucket_index = 0; bucket_index < buckets_; ++bucket_index) {
2147
+ auto bucket_pointer = data_ + bytes_per_bucket() * bucket_index;
2148
+ bucket_header_t& header = *reinterpret_cast<bucket_header_t*>(bucket_pointer);
2149
+ std::uint64_t populated = header.populated;
2150
+ std::uint64_t deleted = header.deleted;
2151
+
2152
+ // Iterate through slots in the bucket
2153
+ for (std::size_t in_bucket_index = 0; in_bucket_index < slots_per_bucket(); ++in_bucket_index) {
2154
+ std::uint64_t mask = std::uint64_t(1ull) << in_bucket_index;
2155
+
2156
+ // Check if the slot is populated and not deleted
2157
+ if ((populated & ~deleted) & mask) {
2158
+ auto slot_pointer = bucket_pointer + sizeof(bucket_header_t) + sizeof(element_t) * in_bucket_index;
2159
+ element_t const& element = *reinterpret_cast<element_t const*>(slot_pointer);
2160
+ func(element);
2161
+ }
2162
+ }
2163
+ }
2164
+ }
2165
+
2166
+ template <typename similar_at> std::size_t count(similar_at&& query) const noexcept {
2167
+
2168
+ if (!capacity_slots_)
2169
+ return 0;
2170
+
2171
+ hash_t hasher;
2172
+ equals_t equals;
2173
+ std::size_t hash_value = hasher(query);
2174
+ std::size_t slot_index = hash_value & (capacity_slots_ - 1);
2175
+ std::size_t start_index = slot_index; // To detect loop in probing
2176
+ std::size_t count = 0;
2177
+
2178
+ // Linear probing to find the range
2179
+ do {
2180
+ slot_ref_t slot = slot_ref(slot_index);
2181
+ if ((slot.header.populated & slot.mask) && (~slot.header.deleted & slot.mask)) {
2182
+ if (equals(slot.element, query))
2183
+ ++count;
2184
+ } else if (~slot.header.populated & slot.mask) {
2185
+ // Stop if we find an empty slot
2186
+ break;
2187
+ }
2188
+
2189
+ // Move to the next slot
2190
+ slot_index = (slot_index + 1) & (capacity_slots_ - 1);
2191
+ } while (slot_index != start_index);
2192
+
2193
+ return count;
2194
+ }
2195
+
2196
+ template <typename similar_at> bool contains(similar_at&& query) const noexcept {
2197
+
2198
+ if (!capacity_slots_)
2199
+ return false;
2200
+
2201
+ hash_t hasher;
2202
+ equals_t equals;
2203
+ std::size_t hash_value = hasher(query);
2204
+ std::size_t slot_index = hash_value & (capacity_slots_ - 1);
2205
+ std::size_t start_index = slot_index; // To detect loop in probing
2206
+
2207
+ // Linear probing to find the first match
2208
+ do {
2209
+ slot_ref_t slot = slot_ref(slot_index);
2210
+ if (slot.header.populated & slot.mask) {
2211
+ if ((~slot.header.deleted & slot.mask) && equals(slot.element, query))
2212
+ return true; // Found a match, exit early
2213
+ } else
2214
+ // Stop if we find an empty slot
2215
+ break;
2216
+
2217
+ // Move to the next slot
2218
+ slot_index = (slot_index + 1) & (capacity_slots_ - 1);
2219
+ } while (slot_index != start_index);
2220
+
2221
+ return false; // No match found
2222
+ }
2223
+
2224
+ void reserve(std::size_t capacity) {
2225
+ if (!try_reserve(capacity))
2226
+ throw std::bad_alloc();
2227
+ }
2228
+
2229
+ bool try_emplace(element_t const& element) noexcept {
2230
+ // Check if we need to resize
2231
+ if (populated_slots_ * 3u >= capacity_slots_ * 2u)
2232
+ if (!try_reserve(populated_slots_ + 1))
2233
+ return false;
2234
+
2235
+ hash_t hasher;
2236
+ std::size_t hash_value = hasher(element);
2237
+ std::size_t slot_index = hash_value & (capacity_slots_ - 1);
2238
+
2239
+ // Linear probing
2240
+ while (true) {
2241
+ slot_ref_t slot = slot_ref(slot_index);
2242
+ if ((~slot.header.populated & slot.mask) | (slot.header.deleted & slot.mask)) {
2243
+ // Found an empty or deleted slot
2244
+ populate_slot(slot, element);
2245
+ ++populated_slots_;
2246
+ return true;
2247
+ }
2248
+ // Move to the next slot
2249
+ slot_index = (slot_index + 1) & (capacity_slots_ - 1);
2250
+ }
2251
+ }
2252
+ };
2253
+
2254
+ } // namespace usearch
2255
+ } // namespace unum