usearch 2.10.3 → 2.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.gyp +101 -0
- package/c/CMakeLists.txt +27 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +362 -0
- package/c/test.c +382 -0
- package/c/usearch.h +356 -0
- package/cmake/clipp.cmake +7 -0
- package/cmake/config.cmake.in +15 -0
- package/cmake/emscripten.cmake +8 -0
- package/cmake/jemalloc.cmake +62 -0
- package/cmake/pkg-config.pc.in +4 -0
- package/cmake/usearchConfigVersion.cmake.in +20 -0
- package/cpp/CMakeLists.txt +14 -0
- package/cpp/README.md +161 -0
- package/cpp/bench.cpp +616 -0
- package/cpp/test.cpp +324 -0
- package/fp16/CMakeLists.txt +176 -0
- package/fp16/LICENSE +11 -0
- package/fp16/README.md +20 -0
- package/fp16/bench/alt-element.cc +89 -0
- package/fp16/bench/from-alt-array.cc +175 -0
- package/fp16/bench/from-ieee-array.cc +399 -0
- package/fp16/bench/ieee-element.cc +272 -0
- package/fp16/bench/to-alt-array.cc +91 -0
- package/fp16/bench/to-ieee-array.cc +303 -0
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +15 -0
- package/fp16/cmake/DownloadGoogleTest.cmake +15 -0
- package/fp16/cmake/DownloadPSimd.cmake +15 -0
- package/fp16/configure.py +65 -0
- package/fp16/confu.yaml +8 -0
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +50 -0
- package/fp16/include/fp16/avx2.py +53 -0
- package/fp16/include/fp16/bitcasts.h +92 -0
- package/fp16/include/fp16/fp16.h +451 -0
- package/fp16/include/fp16/psimd.h +131 -0
- package/fp16/include/fp16.h +11 -0
- package/fp16/jni/Android.mk +6 -0
- package/fp16/jni/Application.mk +4 -0
- package/fp16/test/alt-from-fp32-value.cc +498 -0
- package/fp16/test/alt-to-fp32-bits.cc +263 -0
- package/fp16/test/alt-to-fp32-psimd.cc +145 -0
- package/fp16/test/alt-to-fp32-value.cc +348 -0
- package/fp16/test/alt-to-fp32x2-psimd.cc +245 -0
- package/fp16/test/bitcasts.cc +56 -0
- package/fp16/test/ieee-from-fp32-value.cc +523 -0
- package/fp16/test/ieee-to-fp32-bits.cc +333 -0
- package/fp16/test/ieee-to-fp32-psimd.cc +312 -0
- package/fp16/test/ieee-to-fp32-value.cc +420 -0
- package/fp16/test/ieee-to-fp32x2-psimd.cc +541 -0
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +96 -0
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +96 -0
- package/fp16/test/peachpy/stubs.py +38 -0
- package/fp16/test/tables.cc +530 -0
- package/fp16/test/tables.h +13 -0
- package/fp16/third-party/THHalf.h +92 -0
- package/fp16/third-party/eigen-half.h +170 -0
- package/fp16/third-party/float16-compressor.h +80 -0
- package/fp16/third-party/half.hpp +2916 -0
- package/fp16/third-party/npy-halffloat.h +328 -0
- package/include/usearch/index.hpp +3838 -0
- package/include/usearch/index_dense.hpp +2004 -0
- package/include/usearch/index_plugins.hpp +2255 -0
- package/javascript/dist-package-cjs.json +3 -0
- package/javascript/dist-package-esm.json +3 -0
- package/javascript/lib.cpp +350 -0
- package/javascript/node-gyp-build.d.ts +1 -0
- package/javascript/tsconfig-base.json +25 -0
- package/javascript/tsconfig-cjs.json +8 -0
- package/javascript/tsconfig-esm.json +8 -0
- package/javascript/usearch.test.js +84 -0
- package/javascript/usearch.ts +640 -0
- package/package.json +10 -3
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/prebuilds/win32-ia32/usearch.node +0 -0
- package/prebuilds/win32-x64/usearch.node +0 -0
- package/simsimd/.releaserc +87 -0
- package/simsimd/CONTRIBUTING.md +114 -0
- package/simsimd/Cargo.lock +709 -0
- package/simsimd/Cargo.toml +44 -0
- package/simsimd/LICENSE +201 -0
- package/simsimd/README.md +708 -0
- package/simsimd/binding.gyp +36 -0
- package/simsimd/build.rs +51 -0
- package/simsimd/c/lib.c +110 -0
- package/simsimd/include/simsimd/binary.h +258 -0
- package/simsimd/include/simsimd/dot.h +1237 -0
- package/simsimd/include/simsimd/geospatial.h +37 -0
- package/simsimd/include/simsimd/probability.h +563 -0
- package/simsimd/include/simsimd/simsimd.h +1129 -0
- package/simsimd/include/simsimd/spatial.h +1019 -0
- package/simsimd/include/simsimd/types.h +309 -0
- package/simsimd/javascript/bench.js +127 -0
- package/simsimd/javascript/dist-package-cjs.json +3 -0
- package/simsimd/javascript/dist-package-esm.json +3 -0
- package/simsimd/javascript/fallback.ts +203 -0
- package/simsimd/javascript/lib.c +107 -0
- package/simsimd/javascript/node-gyp-build.d.ts +1 -0
- package/simsimd/javascript/simsimd.ts +135 -0
- package/simsimd/javascript/test.mjs +179 -0
- package/simsimd/javascript/tsconfig-base.json +37 -0
- package/simsimd/javascript/tsconfig-cjs.json +8 -0
- package/simsimd/javascript/tsconfig-esm.json +8 -0
- package/simsimd/package-lock.json +1709 -0
- package/simsimd/package.json +61 -0
- package/simsimd/rust/benches/cosine.rs +27 -0
- package/simsimd/rust/benches/native.rs +30 -0
- package/simsimd/rust/benches/sqeuclidean.rs +27 -0
- package/simsimd/rust/lib.rs +778 -0
|
@@ -0,0 +1,2255 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
#define __STDC_WANT_IEC_60559_TYPES_EXT__
|
|
3
|
+
#include <float.h> // `_Float16`
|
|
4
|
+
#include <stdlib.h> // `aligned_alloc`
|
|
5
|
+
|
|
6
|
+
#include <cstring> // `std::strncmp`
|
|
7
|
+
#include <numeric> // `std::iota`
|
|
8
|
+
#include <thread> // `std::thread`
|
|
9
|
+
#include <vector> // `std::vector`
|
|
10
|
+
|
|
11
|
+
#include <atomic> // `std::atomic`
|
|
12
|
+
#include <thread> // `std::thread`
|
|
13
|
+
|
|
14
|
+
#include <usearch/index.hpp> // `expected_gt` and macros
|
|
15
|
+
|
|
16
|
+
#if !defined(USEARCH_USE_OPENMP)
|
|
17
|
+
#define USEARCH_USE_OPENMP 0
|
|
18
|
+
#endif
|
|
19
|
+
|
|
20
|
+
#if USEARCH_USE_OPENMP
|
|
21
|
+
#include <omp.h> // `omp_get_num_threads()`
|
|
22
|
+
#endif
|
|
23
|
+
|
|
24
|
+
#if defined(USEARCH_DEFINED_LINUX)
|
|
25
|
+
#include <sys/auxv.h> // `getauxval()`
|
|
26
|
+
#endif
|
|
27
|
+
|
|
28
|
+
#if !defined(USEARCH_USE_FP16LIB)
|
|
29
|
+
#if defined(__AVX512F__)
|
|
30
|
+
#define USEARCH_USE_FP16LIB 0
|
|
31
|
+
#elif defined(USEARCH_DEFINED_ARM)
|
|
32
|
+
#include <arm_fp16.h> // `__fp16`
|
|
33
|
+
#define USEARCH_USE_FP16LIB 0
|
|
34
|
+
#else
|
|
35
|
+
#define USEARCH_USE_FP16LIB 1
|
|
36
|
+
#endif
|
|
37
|
+
#endif
|
|
38
|
+
|
|
39
|
+
#if USEARCH_USE_FP16LIB
|
|
40
|
+
#include <fp16/fp16.h>
|
|
41
|
+
#endif
|
|
42
|
+
|
|
43
|
+
#if !defined(USEARCH_USE_SIMSIMD)
|
|
44
|
+
#define USEARCH_USE_SIMSIMD 0
|
|
45
|
+
#endif
|
|
46
|
+
|
|
47
|
+
#if USEARCH_USE_SIMSIMD
|
|
48
|
+
// Propagate the `f16` settings
|
|
49
|
+
#define SIMSIMD_NATIVE_F16 !USEARCH_USE_FP16LIB
|
|
50
|
+
#define SIMSIMD_DYNAMIC_DISPATCH 0
|
|
51
|
+
// No problem, if some of the functions are unused or undefined
|
|
52
|
+
#pragma GCC diagnostic push
|
|
53
|
+
#pragma GCC diagnostic ignored "-Wunused-function"
|
|
54
|
+
#pragma warning(push)
|
|
55
|
+
#pragma warning(disable : 4101)
|
|
56
|
+
#include <simsimd/simsimd.h>
|
|
57
|
+
#pragma warning(pop)
|
|
58
|
+
#pragma GCC diagnostic pop
|
|
59
|
+
#endif
|
|
60
|
+
|
|
61
|
+
namespace unum {
|
|
62
|
+
namespace usearch {
|
|
63
|
+
|
|
64
|
+
using u40_t = uint40_t;
|
|
65
|
+
enum b1x8_t : unsigned char {};
|
|
66
|
+
|
|
67
|
+
struct uuid_t {
|
|
68
|
+
std::uint8_t octets[16];
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
class f16_bits_t;
|
|
72
|
+
class i8_converted_t;
|
|
73
|
+
|
|
74
|
+
#if !USEARCH_USE_FP16LIB
|
|
75
|
+
#if defined(USEARCH_DEFINED_ARM)
|
|
76
|
+
using f16_native_t = __fp16;
|
|
77
|
+
#else
|
|
78
|
+
using f16_native_t = _Float16;
|
|
79
|
+
#endif
|
|
80
|
+
using f16_t = f16_native_t;
|
|
81
|
+
#else
|
|
82
|
+
using f16_native_t = void;
|
|
83
|
+
using f16_t = f16_bits_t;
|
|
84
|
+
#endif
|
|
85
|
+
|
|
86
|
+
using f64_t = double;
|
|
87
|
+
using f32_t = float;
|
|
88
|
+
|
|
89
|
+
using u64_t = std::uint64_t;
|
|
90
|
+
using u32_t = std::uint32_t;
|
|
91
|
+
using u16_t = std::uint16_t;
|
|
92
|
+
using u8_t = std::uint8_t;
|
|
93
|
+
|
|
94
|
+
using i64_t = std::int64_t;
|
|
95
|
+
using i32_t = std::int32_t;
|
|
96
|
+
using i16_t = std::int16_t;
|
|
97
|
+
using i8_t = std::int8_t;
|
|
98
|
+
|
|
99
|
+
enum class metric_kind_t : std::uint8_t {
|
|
100
|
+
unknown_k = 0,
|
|
101
|
+
// Classics:
|
|
102
|
+
ip_k = 'i',
|
|
103
|
+
cos_k = 'c',
|
|
104
|
+
l2sq_k = 'e',
|
|
105
|
+
|
|
106
|
+
// Custom:
|
|
107
|
+
pearson_k = 'p',
|
|
108
|
+
haversine_k = 'h',
|
|
109
|
+
divergence_k = 'd',
|
|
110
|
+
|
|
111
|
+
// Sets:
|
|
112
|
+
jaccard_k = 'j',
|
|
113
|
+
hamming_k = 'b',
|
|
114
|
+
tanimoto_k = 't',
|
|
115
|
+
sorensen_k = 's',
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
enum class scalar_kind_t : std::uint8_t {
|
|
119
|
+
unknown_k = 0,
|
|
120
|
+
// Custom:
|
|
121
|
+
b1x8_k = 1,
|
|
122
|
+
u40_k = 2,
|
|
123
|
+
uuid_k = 3,
|
|
124
|
+
// Common:
|
|
125
|
+
f64_k = 10,
|
|
126
|
+
f32_k = 11,
|
|
127
|
+
f16_k = 12,
|
|
128
|
+
f8_k = 13,
|
|
129
|
+
// Common Integral:
|
|
130
|
+
u64_k = 14,
|
|
131
|
+
u32_k = 15,
|
|
132
|
+
u16_k = 16,
|
|
133
|
+
u8_k = 17,
|
|
134
|
+
i64_k = 20,
|
|
135
|
+
i32_k = 21,
|
|
136
|
+
i16_k = 22,
|
|
137
|
+
i8_k = 23,
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
enum class prefetching_kind_t {
|
|
141
|
+
none_k,
|
|
142
|
+
cpu_k,
|
|
143
|
+
io_uring_k,
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
template <typename scalar_at> scalar_kind_t scalar_kind() noexcept {
|
|
147
|
+
if (std::is_same<scalar_at, b1x8_t>())
|
|
148
|
+
return scalar_kind_t::b1x8_k;
|
|
149
|
+
if (std::is_same<scalar_at, uint40_t>())
|
|
150
|
+
return scalar_kind_t::u40_k;
|
|
151
|
+
if (std::is_same<scalar_at, uuid_t>())
|
|
152
|
+
return scalar_kind_t::uuid_k;
|
|
153
|
+
if (std::is_same<scalar_at, f64_t>())
|
|
154
|
+
return scalar_kind_t::f64_k;
|
|
155
|
+
if (std::is_same<scalar_at, f32_t>())
|
|
156
|
+
return scalar_kind_t::f32_k;
|
|
157
|
+
if (std::is_same<scalar_at, f16_t>())
|
|
158
|
+
return scalar_kind_t::f16_k;
|
|
159
|
+
if (std::is_same<scalar_at, i8_t>())
|
|
160
|
+
return scalar_kind_t::i8_k;
|
|
161
|
+
if (std::is_same<scalar_at, u64_t>())
|
|
162
|
+
return scalar_kind_t::u64_k;
|
|
163
|
+
if (std::is_same<scalar_at, u32_t>())
|
|
164
|
+
return scalar_kind_t::u32_k;
|
|
165
|
+
if (std::is_same<scalar_at, u16_t>())
|
|
166
|
+
return scalar_kind_t::u16_k;
|
|
167
|
+
if (std::is_same<scalar_at, u8_t>())
|
|
168
|
+
return scalar_kind_t::u8_k;
|
|
169
|
+
if (std::is_same<scalar_at, i64_t>())
|
|
170
|
+
return scalar_kind_t::i64_k;
|
|
171
|
+
if (std::is_same<scalar_at, i32_t>())
|
|
172
|
+
return scalar_kind_t::i32_k;
|
|
173
|
+
if (std::is_same<scalar_at, i16_t>())
|
|
174
|
+
return scalar_kind_t::i16_k;
|
|
175
|
+
if (std::is_same<scalar_at, i8_t>())
|
|
176
|
+
return scalar_kind_t::i8_k;
|
|
177
|
+
return scalar_kind_t::unknown_k;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
|
|
181
|
+
|
|
182
|
+
template <typename at> at square(at value) noexcept { return value * value; }
|
|
183
|
+
|
|
184
|
+
template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp) noexcept {
|
|
185
|
+
return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
|
|
186
|
+
}
|
|
187
|
+
template <typename at> inline at clamp(at v, at lo, at hi) noexcept {
|
|
188
|
+
return usearch::clamp(v, lo, hi, std::less<at>{});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
inline bool str_equals(char const* begin, std::size_t len, char const* other_begin) noexcept {
|
|
192
|
+
std::size_t other_len = std::strlen(other_begin);
|
|
193
|
+
return len == other_len && std::strncmp(begin, other_begin, len) == 0;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
inline std::size_t bits_per_scalar(scalar_kind_t scalar_kind) noexcept {
|
|
197
|
+
switch (scalar_kind) {
|
|
198
|
+
case scalar_kind_t::f64_k: return 64;
|
|
199
|
+
case scalar_kind_t::f32_k: return 32;
|
|
200
|
+
case scalar_kind_t::f16_k: return 16;
|
|
201
|
+
case scalar_kind_t::i8_k: return 8;
|
|
202
|
+
case scalar_kind_t::b1x8_k: return 1;
|
|
203
|
+
default: return 0;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
inline std::size_t bits_per_scalar_word(scalar_kind_t scalar_kind) noexcept {
|
|
208
|
+
switch (scalar_kind) {
|
|
209
|
+
case scalar_kind_t::f64_k: return 64;
|
|
210
|
+
case scalar_kind_t::f32_k: return 32;
|
|
211
|
+
case scalar_kind_t::f16_k: return 16;
|
|
212
|
+
case scalar_kind_t::i8_k: return 8;
|
|
213
|
+
case scalar_kind_t::b1x8_k: return 8;
|
|
214
|
+
default: return 0;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
inline char const* scalar_kind_name(scalar_kind_t scalar_kind) noexcept {
|
|
219
|
+
switch (scalar_kind) {
|
|
220
|
+
case scalar_kind_t::f32_k: return "f32";
|
|
221
|
+
case scalar_kind_t::f16_k: return "f16";
|
|
222
|
+
case scalar_kind_t::f64_k: return "f64";
|
|
223
|
+
case scalar_kind_t::i8_k: return "i8";
|
|
224
|
+
case scalar_kind_t::b1x8_k: return "b1x8";
|
|
225
|
+
default: return "";
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
inline char const* metric_kind_name(metric_kind_t metric) noexcept {
|
|
230
|
+
switch (metric) {
|
|
231
|
+
case metric_kind_t::unknown_k: return "unknown";
|
|
232
|
+
case metric_kind_t::ip_k: return "ip";
|
|
233
|
+
case metric_kind_t::cos_k: return "cos";
|
|
234
|
+
case metric_kind_t::l2sq_k: return "l2sq";
|
|
235
|
+
case metric_kind_t::pearson_k: return "pearson";
|
|
236
|
+
case metric_kind_t::haversine_k: return "haversine";
|
|
237
|
+
case metric_kind_t::divergence_k: return "divergence";
|
|
238
|
+
case metric_kind_t::jaccard_k: return "jaccard";
|
|
239
|
+
case metric_kind_t::hamming_k: return "hamming";
|
|
240
|
+
case metric_kind_t::tanimoto_k: return "tanimoto";
|
|
241
|
+
case metric_kind_t::sorensen_k: return "sorensen";
|
|
242
|
+
}
|
|
243
|
+
return "";
|
|
244
|
+
}
|
|
245
|
+
inline expected_gt<scalar_kind_t> scalar_kind_from_name(char const* name, std::size_t len) {
|
|
246
|
+
expected_gt<scalar_kind_t> parsed;
|
|
247
|
+
if (str_equals(name, len, "f32"))
|
|
248
|
+
parsed.result = scalar_kind_t::f32_k;
|
|
249
|
+
else if (str_equals(name, len, "f64"))
|
|
250
|
+
parsed.result = scalar_kind_t::f64_k;
|
|
251
|
+
else if (str_equals(name, len, "f16"))
|
|
252
|
+
parsed.result = scalar_kind_t::f16_k;
|
|
253
|
+
else if (str_equals(name, len, "i8"))
|
|
254
|
+
parsed.result = scalar_kind_t::i8_k;
|
|
255
|
+
else
|
|
256
|
+
parsed.failed("Unknown type, choose: f32, f16, f64, i8");
|
|
257
|
+
return parsed;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
inline expected_gt<scalar_kind_t> scalar_kind_from_name(char const* name) {
|
|
261
|
+
return scalar_kind_from_name(name, std::strlen(name));
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
inline expected_gt<metric_kind_t> metric_from_name(char const* name, std::size_t len) {
|
|
265
|
+
expected_gt<metric_kind_t> parsed;
|
|
266
|
+
if (str_equals(name, len, "l2sq") || str_equals(name, len, "euclidean_sq")) {
|
|
267
|
+
parsed.result = metric_kind_t::l2sq_k;
|
|
268
|
+
} else if (str_equals(name, len, "ip") || str_equals(name, len, "inner") || str_equals(name, len, "dot")) {
|
|
269
|
+
parsed.result = metric_kind_t::ip_k;
|
|
270
|
+
} else if (str_equals(name, len, "cos") || str_equals(name, len, "angular")) {
|
|
271
|
+
parsed.result = metric_kind_t::cos_k;
|
|
272
|
+
} else if (str_equals(name, len, "haversine")) {
|
|
273
|
+
parsed.result = metric_kind_t::haversine_k;
|
|
274
|
+
} else if (str_equals(name, len, "divergence")) {
|
|
275
|
+
parsed.result = metric_kind_t::divergence_k;
|
|
276
|
+
} else if (str_equals(name, len, "pearson")) {
|
|
277
|
+
parsed.result = metric_kind_t::pearson_k;
|
|
278
|
+
} else if (str_equals(name, len, "hamming")) {
|
|
279
|
+
parsed.result = metric_kind_t::hamming_k;
|
|
280
|
+
} else if (str_equals(name, len, "tanimoto")) {
|
|
281
|
+
parsed.result = metric_kind_t::tanimoto_k;
|
|
282
|
+
} else if (str_equals(name, len, "sorensen")) {
|
|
283
|
+
parsed.result = metric_kind_t::sorensen_k;
|
|
284
|
+
} else
|
|
285
|
+
parsed.failed("Unknown distance, choose: l2sq, ip, cos, haversine, divergence, jaccard, pearson, hamming, "
|
|
286
|
+
"tanimoto, sorensen");
|
|
287
|
+
return parsed;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
inline expected_gt<metric_kind_t> metric_from_name(char const* name) {
|
|
291
|
+
return metric_from_name(name, std::strlen(name));
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
inline float f16_to_f32(std::uint16_t u16) noexcept {
|
|
295
|
+
#if !USEARCH_USE_FP16LIB
|
|
296
|
+
f16_native_t f16;
|
|
297
|
+
std::memcpy(&f16, &u16, sizeof(std::uint16_t));
|
|
298
|
+
return float(f16);
|
|
299
|
+
#else
|
|
300
|
+
return fp16_ieee_to_fp32_value(u16);
|
|
301
|
+
#endif
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
inline std::uint16_t f32_to_f16(float f32) noexcept {
|
|
305
|
+
#if !USEARCH_USE_FP16LIB
|
|
306
|
+
f16_native_t f16 = f16_native_t(f32);
|
|
307
|
+
std::uint16_t u16;
|
|
308
|
+
std::memcpy(&u16, &f16, sizeof(std::uint16_t));
|
|
309
|
+
return u16;
|
|
310
|
+
#else
|
|
311
|
+
return fp16_ieee_from_fp32_value(f32);
|
|
312
|
+
#endif
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* @brief Numeric type for the IEEE 754 half-precision floating point.
|
|
317
|
+
* If hardware support isn't available, falls back to a hardware
|
|
318
|
+
* agnostic in-software implementation.
|
|
319
|
+
*/
|
|
320
|
+
class f16_bits_t {
|
|
321
|
+
std::uint16_t uint16_{};
|
|
322
|
+
|
|
323
|
+
public:
|
|
324
|
+
inline f16_bits_t() noexcept : uint16_(0) {}
|
|
325
|
+
inline f16_bits_t(f16_bits_t&&) = default;
|
|
326
|
+
inline f16_bits_t& operator=(f16_bits_t&&) = default;
|
|
327
|
+
inline f16_bits_t(f16_bits_t const&) = default;
|
|
328
|
+
inline f16_bits_t& operator=(f16_bits_t const&) = default;
|
|
329
|
+
|
|
330
|
+
inline operator float() const noexcept { return f16_to_f32(uint16_); }
|
|
331
|
+
inline explicit operator bool() const noexcept { return f16_to_f32(uint16_) > 0.5f; }
|
|
332
|
+
|
|
333
|
+
inline f16_bits_t(i8_converted_t) noexcept;
|
|
334
|
+
inline f16_bits_t(bool v) noexcept : uint16_(f32_to_f16(v)) {}
|
|
335
|
+
inline f16_bits_t(float v) noexcept : uint16_(f32_to_f16(v)) {}
|
|
336
|
+
inline f16_bits_t(double v) noexcept : uint16_(f32_to_f16(static_cast<float>(v))) {}
|
|
337
|
+
|
|
338
|
+
inline f16_bits_t operator+(f16_bits_t other) const noexcept { return {float(*this) + float(other)}; }
|
|
339
|
+
inline f16_bits_t operator-(f16_bits_t other) const noexcept { return {float(*this) - float(other)}; }
|
|
340
|
+
inline f16_bits_t operator*(f16_bits_t other) const noexcept { return {float(*this) * float(other)}; }
|
|
341
|
+
inline f16_bits_t operator/(f16_bits_t other) const noexcept { return {float(*this) / float(other)}; }
|
|
342
|
+
inline f16_bits_t operator+(float other) const noexcept { return {float(*this) + other}; }
|
|
343
|
+
inline f16_bits_t operator-(float other) const noexcept { return {float(*this) - other}; }
|
|
344
|
+
inline f16_bits_t operator*(float other) const noexcept { return {float(*this) * other}; }
|
|
345
|
+
inline f16_bits_t operator/(float other) const noexcept { return {float(*this) / other}; }
|
|
346
|
+
inline f16_bits_t operator+(double other) const noexcept { return {float(*this) + other}; }
|
|
347
|
+
inline f16_bits_t operator-(double other) const noexcept { return {float(*this) - other}; }
|
|
348
|
+
inline f16_bits_t operator*(double other) const noexcept { return {float(*this) * other}; }
|
|
349
|
+
inline f16_bits_t operator/(double other) const noexcept { return {float(*this) / other}; }
|
|
350
|
+
|
|
351
|
+
inline f16_bits_t& operator+=(float v) noexcept {
|
|
352
|
+
uint16_ = f32_to_f16(v + f16_to_f32(uint16_));
|
|
353
|
+
return *this;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
inline f16_bits_t& operator-=(float v) noexcept {
|
|
357
|
+
uint16_ = f32_to_f16(v - f16_to_f32(uint16_));
|
|
358
|
+
return *this;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
inline f16_bits_t& operator*=(float v) noexcept {
|
|
362
|
+
uint16_ = f32_to_f16(v * f16_to_f32(uint16_));
|
|
363
|
+
return *this;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
inline f16_bits_t& operator/=(float v) noexcept {
|
|
367
|
+
uint16_ = f32_to_f16(v / f16_to_f32(uint16_));
|
|
368
|
+
return *this;
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* @brief An STL-based executor or a "thread-pool" for parallel execution.
|
|
374
|
+
* Isn't efficient for small batches, as it recreates the threads on every call.
|
|
375
|
+
*/
|
|
376
|
+
class executor_stl_t {
|
|
377
|
+
std::size_t threads_count_{};
|
|
378
|
+
|
|
379
|
+
struct jthread_t {
|
|
380
|
+
std::thread native_;
|
|
381
|
+
|
|
382
|
+
jthread_t() = default;
|
|
383
|
+
jthread_t(jthread_t&&) = default;
|
|
384
|
+
jthread_t(jthread_t const&) = delete;
|
|
385
|
+
template <typename callable_at> jthread_t(callable_at&& func) : native_([=]() { func(); }) {}
|
|
386
|
+
|
|
387
|
+
~jthread_t() {
|
|
388
|
+
if (native_.joinable())
|
|
389
|
+
native_.join();
|
|
390
|
+
}
|
|
391
|
+
};
|
|
392
|
+
|
|
393
|
+
public:
|
|
394
|
+
/**
|
|
395
|
+
* @param threads_count The number of threads to be used for parallel execution.
|
|
396
|
+
*/
|
|
397
|
+
executor_stl_t(std::size_t threads_count = 0) noexcept
|
|
398
|
+
: threads_count_(threads_count ? threads_count : std::thread::hardware_concurrency()) {}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* @return Maximum number of threads available to the executor.
|
|
402
|
+
*/
|
|
403
|
+
std::size_t size() const noexcept { return threads_count_; }
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* @brief Executes a fixed number of tasks using the specified thread-aware function.
|
|
407
|
+
* @param tasks The total number of tasks to be executed.
|
|
408
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
409
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
410
|
+
*/
|
|
411
|
+
template <typename thread_aware_function_at>
|
|
412
|
+
void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
413
|
+
std::vector<jthread_t> threads_pool;
|
|
414
|
+
std::size_t tasks_per_thread = tasks;
|
|
415
|
+
std::size_t threads_count = (std::min)(threads_count_, tasks);
|
|
416
|
+
if (threads_count > 1) {
|
|
417
|
+
tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
|
|
418
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
|
|
419
|
+
threads_pool.emplace_back([=]() {
|
|
420
|
+
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
421
|
+
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread); ++task_idx)
|
|
422
|
+
thread_aware_function(thread_idx, task_idx);
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
for (std::size_t task_idx = 0; task_idx < (std::min)(tasks, tasks_per_thread); ++task_idx)
|
|
427
|
+
thread_aware_function(0, task_idx);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* @brief Executes limited number of tasks using the specified thread-aware function.
|
|
432
|
+
* @param tasks The upper bound on the number of tasks.
|
|
433
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
434
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
435
|
+
*/
|
|
436
|
+
template <typename thread_aware_function_at>
|
|
437
|
+
void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
438
|
+
std::vector<jthread_t> threads_pool;
|
|
439
|
+
std::size_t tasks_per_thread = tasks;
|
|
440
|
+
std::size_t threads_count = (std::min)(threads_count_, tasks);
|
|
441
|
+
std::atomic_bool stop{false};
|
|
442
|
+
if (threads_count > 1) {
|
|
443
|
+
tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
|
|
444
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
|
|
445
|
+
threads_pool.emplace_back([=, &stop]() {
|
|
446
|
+
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
447
|
+
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread) &&
|
|
448
|
+
!stop.load(std::memory_order_relaxed);
|
|
449
|
+
++task_idx)
|
|
450
|
+
if (!thread_aware_function(thread_idx, task_idx))
|
|
451
|
+
stop.store(true, std::memory_order_relaxed);
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
for (std::size_t task_idx = 0;
|
|
456
|
+
task_idx < (std::min)(tasks, tasks_per_thread) && !stop.load(std::memory_order_relaxed); ++task_idx)
|
|
457
|
+
if (!thread_aware_function(0, task_idx))
|
|
458
|
+
stop.store(true, std::memory_order_relaxed);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* @brief Saturates every available thread with the given workload, until they finish.
|
|
463
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index.
|
|
464
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
465
|
+
*/
|
|
466
|
+
template <typename thread_aware_function_at>
|
|
467
|
+
void parallel(thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
468
|
+
if (threads_count_ == 1)
|
|
469
|
+
return thread_aware_function(0);
|
|
470
|
+
std::vector<jthread_t> threads_pool;
|
|
471
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count_; ++thread_idx)
|
|
472
|
+
threads_pool.emplace_back([=]() { thread_aware_function(thread_idx); });
|
|
473
|
+
thread_aware_function(0);
|
|
474
|
+
}
|
|
475
|
+
};
|
|
476
|
+
|
|
477
|
+
#if USEARCH_USE_OPENMP
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* @brief An OpenMP-based executor or a "thread-pool" for parallel execution.
|
|
481
|
+
* Is the preferred implementation, when available, and maximum performance is needed.
|
|
482
|
+
*/
|
|
483
|
+
class executor_openmp_t {
|
|
484
|
+
public:
|
|
485
|
+
/**
|
|
486
|
+
* @param threads_count The number of threads to be used for parallel execution.
|
|
487
|
+
*/
|
|
488
|
+
executor_openmp_t(std::size_t threads_count = 0) noexcept {
|
|
489
|
+
omp_set_num_threads(static_cast<int>(threads_count ? threads_count : std::thread::hardware_concurrency()));
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* @return Maximum number of threads available to the executor.
|
|
494
|
+
*/
|
|
495
|
+
std::size_t size() const noexcept { return omp_get_max_threads(); }
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* @brief Executes tasks in bulk using the specified thread-aware function.
|
|
499
|
+
* @param tasks The total number of tasks to be executed.
|
|
500
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
501
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
502
|
+
*/
|
|
503
|
+
template <typename thread_aware_function_at>
|
|
504
|
+
void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
505
|
+
#pragma omp parallel for schedule(dynamic, 1)
|
|
506
|
+
for (std::size_t i = 0; i != tasks; ++i) {
|
|
507
|
+
thread_aware_function(omp_get_thread_num(), i);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* @brief Executes tasks in bulk using the specified thread-aware function.
|
|
513
|
+
* @param tasks The total number of tasks to be executed.
|
|
514
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
515
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
516
|
+
*/
|
|
517
|
+
template <typename thread_aware_function_at>
|
|
518
|
+
void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
519
|
+
// OpenMP cancellation points are not yet available on most platforms, and require
|
|
520
|
+
// the `OMP_CANCELLATION` environment variable to be set.
|
|
521
|
+
// http://jakascorner.com/blog/2016/08/omp-cancel.html
|
|
522
|
+
// if (omp_get_cancellation()) {
|
|
523
|
+
// #pragma omp parallel for schedule(dynamic, 1)
|
|
524
|
+
// for (std::size_t i = 0; i != tasks; ++i) {
|
|
525
|
+
// #pragma omp cancellation point for
|
|
526
|
+
// if (!thread_aware_function(omp_get_thread_num(), i)) {
|
|
527
|
+
// #pragma omp cancel for
|
|
528
|
+
// }
|
|
529
|
+
// }
|
|
530
|
+
// }
|
|
531
|
+
std::atomic_bool stop{false};
|
|
532
|
+
#pragma omp parallel for schedule(dynamic, 1) shared(stop)
|
|
533
|
+
for (std::size_t i = 0; i != tasks; ++i) {
|
|
534
|
+
if (!stop.load(std::memory_order_relaxed) && !thread_aware_function(omp_get_thread_num(), i))
|
|
535
|
+
stop.store(true, std::memory_order_relaxed);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* @brief Saturates every available thread with the given workload, until they finish.
|
|
541
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index.
|
|
542
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
543
|
+
*/
|
|
544
|
+
template <typename thread_aware_function_at>
|
|
545
|
+
void parallel(thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
546
|
+
#pragma omp parallel
|
|
547
|
+
{ thread_aware_function(omp_get_thread_num()); }
|
|
548
|
+
}
|
|
549
|
+
};
|
|
550
|
+
|
|
551
|
+
using executor_default_t = executor_openmp_t;
|
|
552
|
+
|
|
553
|
+
#else
|
|
554
|
+
|
|
555
|
+
using executor_default_t = executor_stl_t;
|
|
556
|
+
|
|
557
|
+
#endif
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* @brief Uses OS-specific APIs for aligned memory allocations.
|
|
561
|
+
*/
|
|
562
|
+
template <typename element_at = char, std::size_t alignment_ak = 64> //
|
|
563
|
+
class aligned_allocator_gt {
|
|
564
|
+
public:
|
|
565
|
+
using value_type = element_at;
|
|
566
|
+
using size_type = std::size_t;
|
|
567
|
+
using pointer = element_at*;
|
|
568
|
+
using const_pointer = element_at const*;
|
|
569
|
+
template <typename other_element_at> struct rebind {
|
|
570
|
+
using other = aligned_allocator_gt<other_element_at>;
|
|
571
|
+
};
|
|
572
|
+
|
|
573
|
+
constexpr std::size_t alignment() const { return alignment_ak; }
|
|
574
|
+
|
|
575
|
+
pointer allocate(size_type length) const {
|
|
576
|
+
std::size_t length_bytes = alignment_ak * divide_round_up<alignment_ak>(length * sizeof(value_type));
|
|
577
|
+
std::size_t alignment = alignment_ak;
|
|
578
|
+
// void* result = nullptr;
|
|
579
|
+
// int status = posix_memalign(&result, alignment, length_bytes);
|
|
580
|
+
// return status == 0 ? (pointer)result : nullptr;
|
|
581
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
582
|
+
return (pointer)_aligned_malloc(length_bytes, alignment);
|
|
583
|
+
#else
|
|
584
|
+
return (pointer)aligned_alloc(alignment, length_bytes);
|
|
585
|
+
#endif
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
void deallocate(pointer begin, size_type) const {
|
|
589
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
590
|
+
_aligned_free(begin);
|
|
591
|
+
#else
|
|
592
|
+
free(begin);
|
|
593
|
+
#endif
|
|
594
|
+
}
|
|
595
|
+
};
|
|
596
|
+
|
|
597
|
+
using aligned_allocator_t = aligned_allocator_gt<>;
|
|
598
|
+
|
|
599
|
+
class page_allocator_t {
|
|
600
|
+
public:
|
|
601
|
+
static constexpr std::size_t page_size() { return 4096; }
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* @brief Allocates an @b uninitialized block of memory of the specified size.
|
|
605
|
+
* @param count_bytes The number of bytes to allocate.
|
|
606
|
+
* @return A pointer to the allocated memory block, or `nullptr` if allocation fails.
|
|
607
|
+
*/
|
|
608
|
+
byte_t* allocate(std::size_t count_bytes) const noexcept {
|
|
609
|
+
count_bytes = divide_round_up(count_bytes, page_size()) * page_size();
|
|
610
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
611
|
+
return (byte_t*)(::VirtualAlloc(NULL, count_bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
|
|
612
|
+
#else
|
|
613
|
+
return (byte_t*)mmap(NULL, count_bytes, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
|
|
614
|
+
#endif
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
void deallocate(byte_t* page_pointer, std::size_t count_bytes) const noexcept {
|
|
618
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
619
|
+
::VirtualFree(page_pointer, 0, MEM_RELEASE);
|
|
620
|
+
#else
|
|
621
|
+
count_bytes = divide_round_up(count_bytes, page_size()) * page_size();
|
|
622
|
+
munmap(page_pointer, count_bytes);
|
|
623
|
+
#endif
|
|
624
|
+
}
|
|
625
|
+
};
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* @brief Memory-mapping allocator designed for "alloc many, free at once" usage patterns.
|
|
629
|
+
* @b Thread-safe, @b except constructors and destructors.
|
|
630
|
+
*
|
|
631
|
+
* Using this memory allocator won't affect your overall speed much, as that is not the bottleneck.
|
|
632
|
+
* However, it can drastically improve memory usage especially for huge indexes of small vectors.
|
|
633
|
+
*/
|
|
634
|
+
template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
635
|
+
|
|
636
|
+
static constexpr std::size_t min_capacity() { return 1024 * 1024 * 4; }
|
|
637
|
+
static constexpr std::size_t capacity_multiplier() { return 2; }
|
|
638
|
+
static constexpr std::size_t head_size() {
|
|
639
|
+
/// Pointer to the the previous arena and the size of the current one.
|
|
640
|
+
return divide_round_up<alignment_ak>(sizeof(byte_t*) + sizeof(std::size_t)) * alignment_ak;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
std::mutex mutex_;
|
|
644
|
+
byte_t* last_arena_ = nullptr;
|
|
645
|
+
std::size_t last_usage_ = head_size();
|
|
646
|
+
std::size_t last_capacity_ = min_capacity();
|
|
647
|
+
std::size_t wasted_space_ = 0;
|
|
648
|
+
|
|
649
|
+
public:
|
|
650
|
+
using value_type = byte_t;
|
|
651
|
+
using size_type = std::size_t;
|
|
652
|
+
using pointer = byte_t*;
|
|
653
|
+
using const_pointer = byte_t const*;
|
|
654
|
+
|
|
655
|
+
memory_mapping_allocator_gt() = default;
|
|
656
|
+
memory_mapping_allocator_gt(memory_mapping_allocator_gt&& other) noexcept
|
|
657
|
+
: last_arena_(exchange(other.last_arena_, nullptr)), last_usage_(exchange(other.last_usage_, 0)),
|
|
658
|
+
last_capacity_(exchange(other.last_capacity_, 0)), wasted_space_(exchange(other.wasted_space_, 0)) {}
|
|
659
|
+
|
|
660
|
+
memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt&& other) noexcept {
|
|
661
|
+
std::swap(last_arena_, other.last_arena_);
|
|
662
|
+
std::swap(last_usage_, other.last_usage_);
|
|
663
|
+
std::swap(last_capacity_, other.last_capacity_);
|
|
664
|
+
std::swap(wasted_space_, other.wasted_space_);
|
|
665
|
+
return *this;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
~memory_mapping_allocator_gt() noexcept { reset(); }
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* @brief Discards all previously allocated memory buffers.
|
|
672
|
+
*/
|
|
673
|
+
void reset() noexcept {
|
|
674
|
+
byte_t* last_arena = last_arena_;
|
|
675
|
+
while (last_arena) {
|
|
676
|
+
byte_t* previous_arena = nullptr;
|
|
677
|
+
std::memcpy(&previous_arena, last_arena, sizeof(byte_t*));
|
|
678
|
+
std::size_t last_cap = 0;
|
|
679
|
+
std::memcpy(&last_cap, last_arena + sizeof(byte_t*), sizeof(std::size_t));
|
|
680
|
+
page_allocator_t{}.deallocate(last_arena, last_cap);
|
|
681
|
+
last_arena = previous_arena;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
// Clear the references:
|
|
685
|
+
last_arena_ = nullptr;
|
|
686
|
+
last_usage_ = head_size();
|
|
687
|
+
last_capacity_ = min_capacity();
|
|
688
|
+
wasted_space_ = 0;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* @brief Copy constructor.
|
|
693
|
+
* @note This is a no-op copy constructor since the allocator is not copyable.
|
|
694
|
+
*/
|
|
695
|
+
memory_mapping_allocator_gt(memory_mapping_allocator_gt const&) noexcept {}
|
|
696
|
+
|
|
697
|
+
/**
|
|
698
|
+
* @brief Copy assignment operator.
|
|
699
|
+
* @note This is a no-op copy assignment operator since the allocator is not copyable.
|
|
700
|
+
* @return Reference to the allocator after the assignment.
|
|
701
|
+
*/
|
|
702
|
+
memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt const&) noexcept {
|
|
703
|
+
reset();
|
|
704
|
+
return *this;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/**
|
|
708
|
+
* @brief Allocates an @b uninitialized block of memory of the specified size.
|
|
709
|
+
* @param count_bytes The number of bytes to allocate.
|
|
710
|
+
* @return A pointer to the allocated memory block, or `nullptr` if allocation fails.
|
|
711
|
+
*/
|
|
712
|
+
inline byte_t* allocate(std::size_t count_bytes) noexcept {
|
|
713
|
+
std::size_t extended_bytes = divide_round_up<alignment_ak>(count_bytes) * alignment_ak;
|
|
714
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
715
|
+
if (!last_arena_ || (last_usage_ + extended_bytes >= last_capacity_)) {
|
|
716
|
+
std::size_t new_cap = (std::max)(last_capacity_, ceil2(extended_bytes)) * capacity_multiplier();
|
|
717
|
+
byte_t* new_arena = page_allocator_t{}.allocate(new_cap);
|
|
718
|
+
if (!new_arena)
|
|
719
|
+
return nullptr;
|
|
720
|
+
std::memcpy(new_arena, &last_arena_, sizeof(byte_t*));
|
|
721
|
+
std::memcpy(new_arena + sizeof(byte_t*), &new_cap, sizeof(std::size_t));
|
|
722
|
+
|
|
723
|
+
wasted_space_ += total_reserved();
|
|
724
|
+
last_arena_ = new_arena;
|
|
725
|
+
last_capacity_ = new_cap;
|
|
726
|
+
last_usage_ = head_size();
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
wasted_space_ += extended_bytes - count_bytes;
|
|
730
|
+
return last_arena_ + exchange(last_usage_, last_usage_ + extended_bytes);
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
/**
|
|
734
|
+
* @brief Returns the amount of memory used by the allocator across all arenas.
|
|
735
|
+
* @return The amount of space in bytes.
|
|
736
|
+
*/
|
|
737
|
+
std::size_t total_allocated() const noexcept {
|
|
738
|
+
if (!last_arena_)
|
|
739
|
+
return 0;
|
|
740
|
+
std::size_t total_used = 0;
|
|
741
|
+
std::size_t last_capacity = last_capacity_;
|
|
742
|
+
do {
|
|
743
|
+
total_used += last_capacity;
|
|
744
|
+
last_capacity /= capacity_multiplier();
|
|
745
|
+
} while (last_capacity >= min_capacity());
|
|
746
|
+
return total_used;
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/**
|
|
750
|
+
* @brief Returns the amount of wasted space due to alignment.
|
|
751
|
+
* @return The amount of wasted space in bytes.
|
|
752
|
+
*/
|
|
753
|
+
std::size_t total_wasted() const noexcept { return wasted_space_; }
|
|
754
|
+
|
|
755
|
+
/**
|
|
756
|
+
* @brief Returns the amount of remaining memory already reserved but not yet used.
|
|
757
|
+
* @return The amount of reserved memory in bytes.
|
|
758
|
+
*/
|
|
759
|
+
std::size_t total_reserved() const noexcept { return last_arena_ ? last_capacity_ - last_usage_ : 0; }
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* @warning The very first memory de-allocation discards all the arenas!
|
|
763
|
+
*/
|
|
764
|
+
void deallocate(byte_t* = nullptr, std::size_t = 0) noexcept { reset(); }
|
|
765
|
+
};
|
|
766
|
+
|
|
767
|
+
using memory_mapping_allocator_t = memory_mapping_allocator_gt<>;
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* @brief C++11 userspace implementation of an oversimplified `std::shared_mutex`,
|
|
771
|
+
* that assumes rare interleaving of shared and unique locks. It's not fair,
|
|
772
|
+
* but requires only a single 32-bit atomic integer to work.
|
|
773
|
+
*/
|
|
774
|
+
class unfair_shared_mutex_t {
|
|
775
|
+
/** Any positive integer describes the number of concurrent readers */
|
|
776
|
+
enum state_t : std::int32_t {
|
|
777
|
+
idle_k = 0,
|
|
778
|
+
writing_k = -1,
|
|
779
|
+
};
|
|
780
|
+
std::atomic<std::int32_t> state_{idle_k};
|
|
781
|
+
|
|
782
|
+
public:
|
|
783
|
+
inline void lock() noexcept {
|
|
784
|
+
std::int32_t raw;
|
|
785
|
+
relock:
|
|
786
|
+
raw = idle_k;
|
|
787
|
+
if (!state_.compare_exchange_weak(raw, writing_k, std::memory_order_acquire, std::memory_order_relaxed)) {
|
|
788
|
+
std::this_thread::yield();
|
|
789
|
+
goto relock;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
inline void unlock() noexcept { state_.store(idle_k, std::memory_order_release); }
|
|
794
|
+
|
|
795
|
+
inline void lock_shared() noexcept {
|
|
796
|
+
std::int32_t raw;
|
|
797
|
+
relock_shared:
|
|
798
|
+
raw = state_.load(std::memory_order_acquire);
|
|
799
|
+
// Spin while it's uniquely locked
|
|
800
|
+
if (raw == writing_k) {
|
|
801
|
+
std::this_thread::yield();
|
|
802
|
+
goto relock_shared;
|
|
803
|
+
}
|
|
804
|
+
// Try incrementing the counter
|
|
805
|
+
if (!state_.compare_exchange_weak(raw, raw + 1, std::memory_order_acquire, std::memory_order_relaxed)) {
|
|
806
|
+
std::this_thread::yield();
|
|
807
|
+
goto relock_shared;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
inline void unlock_shared() noexcept { state_.fetch_sub(1, std::memory_order_release); }
|
|
812
|
+
|
|
813
|
+
/**
|
|
814
|
+
* @brief Try upgrades the current `lock_shared()` to a unique `lock()` state.
|
|
815
|
+
*/
|
|
816
|
+
inline bool try_escalate() noexcept {
|
|
817
|
+
std::int32_t one_read = 1;
|
|
818
|
+
return state_.compare_exchange_weak(one_read, writing_k, std::memory_order_acquire, std::memory_order_relaxed);
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
/**
|
|
822
|
+
* @brief Escalates current lock potentially loosing control in the middle.
|
|
823
|
+
* It's a shortcut for `try_escalate`-`unlock_shared`-`lock` trio.
|
|
824
|
+
*/
|
|
825
|
+
inline void unsafe_escalate() noexcept {
|
|
826
|
+
if (!try_escalate()) {
|
|
827
|
+
unlock_shared();
|
|
828
|
+
lock();
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* @brief Upgrades the current `lock_shared()` to a unique `lock()` state.
|
|
834
|
+
*/
|
|
835
|
+
inline void escalate() noexcept {
|
|
836
|
+
while (!try_escalate())
|
|
837
|
+
std::this_thread::yield();
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* @brief De-escalation of a previously escalated state.
|
|
842
|
+
*/
|
|
843
|
+
inline void de_escalate() noexcept {
|
|
844
|
+
std::int32_t one_read = 1;
|
|
845
|
+
state_.store(one_read, std::memory_order_release);
|
|
846
|
+
}
|
|
847
|
+
};
|
|
848
|
+
|
|
849
|
+
template <typename mutex_at = unfair_shared_mutex_t> class shared_lock_gt {
|
|
850
|
+
mutex_at& mutex_;
|
|
851
|
+
|
|
852
|
+
public:
|
|
853
|
+
inline explicit shared_lock_gt(mutex_at& m) noexcept : mutex_(m) { mutex_.lock_shared(); }
|
|
854
|
+
inline ~shared_lock_gt() noexcept { mutex_.unlock_shared(); }
|
|
855
|
+
};
|
|
856
|
+
|
|
857
|
+
/**
|
|
858
|
+
* @brief Utility class used to cast arrays of one scalar type to another,
|
|
859
|
+
* avoiding unnecessary conversions.
|
|
860
|
+
*/
|
|
861
|
+
template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
|
|
862
|
+
inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
|
|
863
|
+
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
864
|
+
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
865
|
+
auto converter = [](from_scalar_at from) { return to_scalar_at(from); };
|
|
866
|
+
std::transform(typed_input, typed_input + dim, typed_output, converter);
|
|
867
|
+
return true;
|
|
868
|
+
}
|
|
869
|
+
};
|
|
870
|
+
|
|
871
|
+
template <> struct cast_gt<f32_t, f32_t> {
|
|
872
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
873
|
+
};
|
|
874
|
+
|
|
875
|
+
template <> struct cast_gt<f64_t, f64_t> {
|
|
876
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
877
|
+
};
|
|
878
|
+
|
|
879
|
+
template <> struct cast_gt<f16_bits_t, f16_bits_t> {
|
|
880
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
881
|
+
};
|
|
882
|
+
|
|
883
|
+
template <> struct cast_gt<i8_t, i8_t> {
|
|
884
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
885
|
+
};
|
|
886
|
+
|
|
887
|
+
template <> struct cast_gt<b1x8_t, b1x8_t> {
|
|
888
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
889
|
+
};
|
|
890
|
+
|
|
891
|
+
template <typename from_scalar_at> struct cast_gt<from_scalar_at, b1x8_t> {
|
|
892
|
+
inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
|
|
893
|
+
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
894
|
+
unsigned char* typed_output = reinterpret_cast<unsigned char*>(output);
|
|
895
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
896
|
+
// Converting from scalar types to boolean isn't trivial and depends on the type.
|
|
897
|
+
// The most common case is to consider all positive values as `true` and all others as `false`.
|
|
898
|
+
// - `bool(0.00001f)` converts to 1
|
|
899
|
+
// - `bool(-0.00001f)` converts to 1
|
|
900
|
+
// - `bool(0)` converts to 0
|
|
901
|
+
// - `bool(-0)` converts to 0
|
|
902
|
+
// - `bool(std::numeric_limits<float>::infinity())` converts to 1
|
|
903
|
+
// - `bool(std::numeric_limits<float>::epsilon())` converts to 1
|
|
904
|
+
// - `bool(std::numeric_limits<float>::signaling_NaN())` converts to 1
|
|
905
|
+
// - `bool(std::numeric_limits<float>::denorm_min())` converts to 1
|
|
906
|
+
typed_output[i / CHAR_BIT] |= bool(typed_input[i] > 0) ? (128 >> (i & (CHAR_BIT - 1))) : 0;
|
|
907
|
+
return true;
|
|
908
|
+
}
|
|
909
|
+
};
|
|
910
|
+
|
|
911
|
+
template <typename to_scalar_at> struct cast_gt<b1x8_t, to_scalar_at> {
|
|
912
|
+
inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
|
|
913
|
+
unsigned char const* typed_input = reinterpret_cast<unsigned char const*>(input);
|
|
914
|
+
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
915
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
916
|
+
// We can't entirely reconstruct the original scalar type from a boolean.
|
|
917
|
+
// The simplest variant would be to map set bits to ones, and unset bits to zeros.
|
|
918
|
+
typed_output[i] = bool(typed_input[i / CHAR_BIT] & (128 >> (i & (CHAR_BIT - 1))));
|
|
919
|
+
return true;
|
|
920
|
+
}
|
|
921
|
+
};
|
|
922
|
+
|
|
923
|
+
/**
|
|
924
|
+
* @brief Numeric type for uniformly-distributed floating point
|
|
925
|
+
* values within [-1,1] range, quantized to integers [-100,100].
|
|
926
|
+
*/
|
|
927
|
+
class i8_converted_t {
|
|
928
|
+
std::int8_t int8_{};
|
|
929
|
+
|
|
930
|
+
public:
|
|
931
|
+
constexpr static f32_t divisor_k = 100.f;
|
|
932
|
+
constexpr static std::int8_t min_k = -100;
|
|
933
|
+
constexpr static std::int8_t max_k = 100;
|
|
934
|
+
|
|
935
|
+
inline i8_converted_t() noexcept : int8_(0) {}
|
|
936
|
+
inline i8_converted_t(bool v) noexcept : int8_(v ? max_k : 0) {}
|
|
937
|
+
|
|
938
|
+
inline i8_converted_t(i8_converted_t&&) = default;
|
|
939
|
+
inline i8_converted_t& operator=(i8_converted_t&&) = default;
|
|
940
|
+
inline i8_converted_t(i8_converted_t const&) = default;
|
|
941
|
+
inline i8_converted_t& operator=(i8_converted_t const&) = default;
|
|
942
|
+
|
|
943
|
+
inline operator f16_t() const noexcept { return static_cast<f16_t>(f32_t(int8_) / divisor_k); }
|
|
944
|
+
inline operator f32_t() const noexcept { return f32_t(int8_) / divisor_k; }
|
|
945
|
+
inline operator f64_t() const noexcept { return f64_t(int8_) / divisor_k; }
|
|
946
|
+
inline explicit operator bool() const noexcept { return int8_ > (max_k / 2); }
|
|
947
|
+
inline explicit operator std::int8_t() const noexcept { return int8_; }
|
|
948
|
+
inline explicit operator std::int16_t() const noexcept { return int8_; }
|
|
949
|
+
inline explicit operator std::int32_t() const noexcept { return int8_; }
|
|
950
|
+
inline explicit operator std::int64_t() const noexcept { return int8_; }
|
|
951
|
+
|
|
952
|
+
inline i8_converted_t(f16_t v)
|
|
953
|
+
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
954
|
+
inline i8_converted_t(f32_t v)
|
|
955
|
+
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
956
|
+
inline i8_converted_t(f64_t v)
|
|
957
|
+
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
958
|
+
};
|
|
959
|
+
|
|
960
|
+
f16_bits_t::f16_bits_t(i8_converted_t v) noexcept : uint16_(f32_to_f16(v)) {}
|
|
961
|
+
|
|
962
|
+
template <> struct cast_gt<i8_t, f16_t> : public cast_gt<i8_converted_t, f16_t> {};
|
|
963
|
+
template <> struct cast_gt<i8_t, f32_t> : public cast_gt<i8_converted_t, f32_t> {};
|
|
964
|
+
template <> struct cast_gt<i8_t, f64_t> : public cast_gt<i8_converted_t, f64_t> {};
|
|
965
|
+
|
|
966
|
+
template <> struct cast_gt<f16_t, i8_t> : public cast_gt<f16_t, i8_converted_t> {};
|
|
967
|
+
template <> struct cast_gt<f32_t, i8_t> : public cast_gt<f32_t, i8_converted_t> {};
|
|
968
|
+
template <> struct cast_gt<f64_t, i8_t> : public cast_gt<f64_t, i8_converted_t> {};
|
|
969
|
+
|
|
970
|
+
/**
|
|
971
|
+
* @brief Inner (Dot) Product distance.
|
|
972
|
+
*/
|
|
973
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_ip_gt {
|
|
974
|
+
using scalar_t = scalar_at;
|
|
975
|
+
using result_t = result_at;
|
|
976
|
+
|
|
977
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
978
|
+
result_t ab{};
|
|
979
|
+
#if USEARCH_USE_OPENMP
|
|
980
|
+
#pragma omp simd reduction(+ : ab)
|
|
981
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
982
|
+
#pragma clang loop vectorize(enable)
|
|
983
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
984
|
+
#pragma GCC ivdep
|
|
985
|
+
#endif
|
|
986
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
987
|
+
ab += result_t(a[i]) * result_t(b[i]);
|
|
988
|
+
return 1 - ab;
|
|
989
|
+
}
|
|
990
|
+
};
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* @brief Cosine (Angular) distance.
|
|
994
|
+
* Identical to the Inner Product of normalized vectors.
|
|
995
|
+
* Unless you are running on an tiny embedded platform, this metric
|
|
996
|
+
* is recommended over `::metric_ip_gt` for low-precision scalars.
|
|
997
|
+
*/
|
|
998
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_cos_gt {
|
|
999
|
+
using scalar_t = scalar_at;
|
|
1000
|
+
using result_t = result_at;
|
|
1001
|
+
|
|
1002
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
1003
|
+
result_t ab{}, a2{}, b2{};
|
|
1004
|
+
#if USEARCH_USE_OPENMP
|
|
1005
|
+
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
1006
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1007
|
+
#pragma clang loop vectorize(enable)
|
|
1008
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1009
|
+
#pragma GCC ivdep
|
|
1010
|
+
#endif
|
|
1011
|
+
for (std::size_t i = 0; i != dim; ++i) {
|
|
1012
|
+
result_t ai = static_cast<result_t>(a[i]);
|
|
1013
|
+
result_t bi = static_cast<result_t>(b[i]);
|
|
1014
|
+
ab += ai * bi, a2 += square(ai), b2 += square(bi);
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
result_t result_if_zero[2][2];
|
|
1018
|
+
result_if_zero[0][0] = 1 - ab / (std::sqrt(a2) * std::sqrt(b2));
|
|
1019
|
+
result_if_zero[0][1] = result_if_zero[1][0] = 1;
|
|
1020
|
+
result_if_zero[1][1] = 0;
|
|
1021
|
+
return result_if_zero[a2 == 0][b2 == 0];
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* @brief Squared Euclidean (L2) distance.
|
|
1027
|
+
* Square root is avoided at the end, as it won't affect the ordering.
|
|
1028
|
+
*/
|
|
1029
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_l2sq_gt {
|
|
1030
|
+
using scalar_t = scalar_at;
|
|
1031
|
+
using result_t = result_at;
|
|
1032
|
+
|
|
1033
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
1034
|
+
result_t ab_deltas_sq{};
|
|
1035
|
+
#if USEARCH_USE_OPENMP
|
|
1036
|
+
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
1037
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1038
|
+
#pragma clang loop vectorize(enable)
|
|
1039
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1040
|
+
#pragma GCC ivdep
|
|
1041
|
+
#endif
|
|
1042
|
+
for (std::size_t i = 0; i != dim; ++i) {
|
|
1043
|
+
result_t ai = static_cast<result_t>(a[i]);
|
|
1044
|
+
result_t bi = static_cast<result_t>(b[i]);
|
|
1045
|
+
ab_deltas_sq += square(ai - bi);
|
|
1046
|
+
}
|
|
1047
|
+
return ab_deltas_sq;
|
|
1048
|
+
}
|
|
1049
|
+
};
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* @brief Hamming distance computes the number of differing bits in
|
|
1053
|
+
* two arrays of integers. An example would be a textual document,
|
|
1054
|
+
* tokenized and hashed into a fixed-capacity bitset.
|
|
1055
|
+
*/
|
|
1056
|
+
template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct metric_hamming_gt {
|
|
1057
|
+
using scalar_t = scalar_at;
|
|
1058
|
+
using result_t = result_at;
|
|
1059
|
+
static_assert( //
|
|
1060
|
+
std::is_unsigned<scalar_t>::value ||
|
|
1061
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
1062
|
+
"Hamming distance requires unsigned integral words");
|
|
1063
|
+
|
|
1064
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
1065
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
1066
|
+
result_t matches{};
|
|
1067
|
+
#if USEARCH_USE_OPENMP
|
|
1068
|
+
#pragma omp simd reduction(+ : matches)
|
|
1069
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1070
|
+
#pragma clang loop vectorize(enable)
|
|
1071
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1072
|
+
#pragma GCC ivdep
|
|
1073
|
+
#endif
|
|
1074
|
+
for (std::size_t i = 0; i != words; ++i)
|
|
1075
|
+
matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
|
|
1076
|
+
return matches;
|
|
1077
|
+
}
|
|
1078
|
+
};
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* @brief Tanimoto distance is the intersection over bitwise union.
|
|
1082
|
+
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
1083
|
+
*/
|
|
1084
|
+
template <typename scalar_at = std::uint64_t, typename result_at = float> struct metric_tanimoto_gt {
|
|
1085
|
+
using scalar_t = scalar_at;
|
|
1086
|
+
using result_t = result_at;
|
|
1087
|
+
static_assert( //
|
|
1088
|
+
std::is_unsigned<scalar_t>::value ||
|
|
1089
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
1090
|
+
"Tanimoto distance requires unsigned integral words");
|
|
1091
|
+
static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
|
|
1092
|
+
|
|
1093
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
1094
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
1095
|
+
result_t and_count{};
|
|
1096
|
+
result_t or_count{};
|
|
1097
|
+
#if USEARCH_USE_OPENMP
|
|
1098
|
+
#pragma omp simd reduction(+ : and_count, or_count)
|
|
1099
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1100
|
+
#pragma clang loop vectorize(enable)
|
|
1101
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1102
|
+
#pragma GCC ivdep
|
|
1103
|
+
#endif
|
|
1104
|
+
for (std::size_t i = 0; i != words; ++i) {
|
|
1105
|
+
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count();
|
|
1106
|
+
or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
|
|
1107
|
+
}
|
|
1108
|
+
return 1 - result_t(and_count) / or_count;
|
|
1109
|
+
}
|
|
1110
|
+
};
|
|
1111
|
+
|
|
1112
|
+
/**
|
|
1113
|
+
* @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
|
|
1114
|
+
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
1115
|
+
*/
|
|
1116
|
+
template <typename scalar_at = std::uint64_t, typename result_at = float> struct metric_sorensen_gt {
|
|
1117
|
+
using scalar_t = scalar_at;
|
|
1118
|
+
using result_t = result_at;
|
|
1119
|
+
static_assert( //
|
|
1120
|
+
std::is_unsigned<scalar_t>::value ||
|
|
1121
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
1122
|
+
"Sorensen-Dice distance requires unsigned integral words");
|
|
1123
|
+
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
1124
|
+
|
|
1125
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
1126
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
1127
|
+
result_t and_count{};
|
|
1128
|
+
result_t any_count{};
|
|
1129
|
+
#if USEARCH_USE_OPENMP
|
|
1130
|
+
#pragma omp simd reduction(+ : and_count, any_count)
|
|
1131
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1132
|
+
#pragma clang loop vectorize(enable)
|
|
1133
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1134
|
+
#pragma GCC ivdep
|
|
1135
|
+
#endif
|
|
1136
|
+
for (std::size_t i = 0; i != words; ++i) {
|
|
1137
|
+
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count();
|
|
1138
|
+
any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
|
|
1139
|
+
}
|
|
1140
|
+
return 1 - 2 * result_t(and_count) / any_count;
|
|
1141
|
+
}
|
|
1142
|
+
};
|
|
1143
|
+
|
|
1144
|
+
/**
|
|
1145
|
+
* @brief Counts the number of matching elements in two unique sorted sets.
|
|
1146
|
+
* Can be used to compute the similarity between two textual documents
|
|
1147
|
+
* using the IDs of tokens present in them.
|
|
1148
|
+
* Similar to `metric_tanimoto_gt` for dense representations.
|
|
1149
|
+
*/
|
|
1150
|
+
template <typename scalar_at = std::int32_t, typename result_at = float> struct metric_jaccard_gt {
|
|
1151
|
+
using scalar_t = scalar_at;
|
|
1152
|
+
using result_t = result_at;
|
|
1153
|
+
static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
|
|
1154
|
+
|
|
1155
|
+
inline result_t operator()( //
|
|
1156
|
+
scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
|
|
1157
|
+
result_t intersection{};
|
|
1158
|
+
std::size_t i{};
|
|
1159
|
+
std::size_t j{};
|
|
1160
|
+
while (i != a_length && j != b_length) {
|
|
1161
|
+
intersection += a[i] == b[j];
|
|
1162
|
+
i += a[i] < b[j];
|
|
1163
|
+
j += a[i] >= b[j];
|
|
1164
|
+
}
|
|
1165
|
+
return 1 - intersection / (a_length + b_length - intersection);
|
|
1166
|
+
}
|
|
1167
|
+
};
|
|
1168
|
+
|
|
1169
|
+
/**
|
|
1170
|
+
* @brief Measures Pearson Correlation between two sequences in a single pass.
|
|
1171
|
+
*/
|
|
1172
|
+
template <typename scalar_at = float, typename result_at = float> struct metric_pearson_gt {
|
|
1173
|
+
using scalar_t = scalar_at;
|
|
1174
|
+
using result_t = result_at;
|
|
1175
|
+
|
|
1176
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
1177
|
+
// The correlation coefficient can't be defined for one or zero-dimensional data.
|
|
1178
|
+
if (dim <= 1)
|
|
1179
|
+
return 0;
|
|
1180
|
+
// Conventional Pearson Correlation Coefficient definiton subtracts the mean value of each
|
|
1181
|
+
// sequence from each element, before dividing them. WikiPedia article suggests a convenient
|
|
1182
|
+
// single-pass algorithm for calculating sample correlations, though depending on the numbers
|
|
1183
|
+
// involved, it can sometimes be numerically unstable.
|
|
1184
|
+
result_t a_sum{}, b_sum{}, ab_sum{};
|
|
1185
|
+
result_t a_sq_sum{}, b_sq_sum{};
|
|
1186
|
+
#if USEARCH_USE_OPENMP
|
|
1187
|
+
#pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
|
|
1188
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1189
|
+
#pragma clang loop vectorize(enable)
|
|
1190
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1191
|
+
#pragma GCC ivdep
|
|
1192
|
+
#endif
|
|
1193
|
+
for (std::size_t i = 0; i != dim; ++i) {
|
|
1194
|
+
result_t ai = static_cast<result_t>(a[i]);
|
|
1195
|
+
result_t bi = static_cast<result_t>(b[i]);
|
|
1196
|
+
a_sum += ai;
|
|
1197
|
+
b_sum += bi;
|
|
1198
|
+
ab_sum += ai * bi;
|
|
1199
|
+
a_sq_sum += ai * ai;
|
|
1200
|
+
b_sq_sum += bi * bi;
|
|
1201
|
+
}
|
|
1202
|
+
result_t denom = (dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum);
|
|
1203
|
+
if (denom == 0)
|
|
1204
|
+
return 0;
|
|
1205
|
+
result_t corr = dim * ab_sum - a_sum * b_sum;
|
|
1206
|
+
denom = std::sqrt(denom);
|
|
1207
|
+
// The normal Pearson correlation value is between -1 and 1, but we are looking for a distance.
|
|
1208
|
+
// So instead of returning `corr / denom`, we return `1 - corr / denom`.
|
|
1209
|
+
return 1 - corr / denom;
|
|
1210
|
+
}
|
|
1211
|
+
};
|
|
1212
|
+
|
|
1213
|
+
/**
|
|
1214
|
+
* @brief Measures Jensen-Shannon Divergence between two probability distributions.
|
|
1215
|
+
*/
|
|
1216
|
+
template <typename scalar_at = float, typename result_at = float> struct metric_divergence_gt {
|
|
1217
|
+
using scalar_t = scalar_at;
|
|
1218
|
+
using result_t = result_at;
|
|
1219
|
+
|
|
1220
|
+
inline result_t operator()(scalar_t const* p, scalar_t const* q, std::size_t dim) const noexcept {
|
|
1221
|
+
result_t kld_pm{}, kld_qm{};
|
|
1222
|
+
result_t epsilon = std::numeric_limits<result_t>::epsilon();
|
|
1223
|
+
#if USEARCH_USE_OPENMP
|
|
1224
|
+
#pragma omp simd reduction(+ : kld_pm, kld_qm)
|
|
1225
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1226
|
+
#pragma clang loop vectorize(enable)
|
|
1227
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1228
|
+
#pragma GCC ivdep
|
|
1229
|
+
#endif
|
|
1230
|
+
for (std::size_t i = 0; i != dim; ++i) {
|
|
1231
|
+
result_t pi = static_cast<result_t>(p[i]);
|
|
1232
|
+
result_t qi = static_cast<result_t>(q[i]);
|
|
1233
|
+
result_t mi = (pi + qi) / 2 + epsilon;
|
|
1234
|
+
kld_pm += pi * std::log((pi + epsilon) / mi);
|
|
1235
|
+
kld_qm += qi * std::log((qi + epsilon) / mi);
|
|
1236
|
+
}
|
|
1237
|
+
return (kld_pm + kld_qm) / 2;
|
|
1238
|
+
}
|
|
1239
|
+
};
|
|
1240
|
+
|
|
1241
|
+
struct cos_i8_t {
|
|
1242
|
+
using scalar_t = i8_t;
|
|
1243
|
+
using result_t = f32_t;
|
|
1244
|
+
|
|
1245
|
+
inline result_t operator()(i8_t const* a, i8_t const* b, std::size_t dim) const noexcept {
|
|
1246
|
+
std::int32_t ab{}, a2{}, b2{};
|
|
1247
|
+
#if USEARCH_USE_OPENMP
|
|
1248
|
+
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
1249
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1250
|
+
#pragma clang loop vectorize(enable)
|
|
1251
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1252
|
+
#pragma GCC ivdep
|
|
1253
|
+
#endif
|
|
1254
|
+
for (std::size_t i = 0; i != dim; i++) {
|
|
1255
|
+
std::int16_t ai{a[i]};
|
|
1256
|
+
std::int16_t bi{b[i]};
|
|
1257
|
+
ab += ai * bi;
|
|
1258
|
+
a2 += square(ai);
|
|
1259
|
+
b2 += square(bi);
|
|
1260
|
+
}
|
|
1261
|
+
result_t a2f = std::sqrt(static_cast<result_t>(a2));
|
|
1262
|
+
result_t b2f = std::sqrt(static_cast<result_t>(b2));
|
|
1263
|
+
return (ab != 0) ? (1.f - ab / (a2f * b2f)) : 0;
|
|
1264
|
+
}
|
|
1265
|
+
};
|
|
1266
|
+
|
|
1267
|
+
struct l2sq_i8_t {
|
|
1268
|
+
using scalar_t = i8_t;
|
|
1269
|
+
using result_t = f32_t;
|
|
1270
|
+
|
|
1271
|
+
inline result_t operator()(i8_t const* a, i8_t const* b, std::size_t dim) const noexcept {
|
|
1272
|
+
std::int32_t ab_deltas_sq{};
|
|
1273
|
+
#if USEARCH_USE_OPENMP
|
|
1274
|
+
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
1275
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1276
|
+
#pragma clang loop vectorize(enable)
|
|
1277
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1278
|
+
#pragma GCC ivdep
|
|
1279
|
+
#endif
|
|
1280
|
+
for (std::size_t i = 0; i != dim; i++)
|
|
1281
|
+
ab_deltas_sq += square(std::int16_t(a[i]) - std::int16_t(b[i]));
|
|
1282
|
+
return static_cast<result_t>(ab_deltas_sq);
|
|
1283
|
+
}
|
|
1284
|
+
};
|
|
1285
|
+
|
|
1286
|
+
/**
|
|
1287
|
+
* @brief Haversine distance for the shortest distance between two nodes on
|
|
1288
|
+
* the surface of a 3D sphere, defined with latitude and longitude.
|
|
1289
|
+
*/
|
|
1290
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_haversine_gt {
|
|
1291
|
+
using scalar_t = scalar_at;
|
|
1292
|
+
using result_t = result_at;
|
|
1293
|
+
static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
|
|
1294
|
+
|
|
1295
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t = 2) const noexcept {
|
|
1296
|
+
result_t lat_a = a[0], lon_a = a[1];
|
|
1297
|
+
result_t lat_b = b[0], lon_b = b[1];
|
|
1298
|
+
|
|
1299
|
+
result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a) / 2;
|
|
1300
|
+
result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a) / 2;
|
|
1301
|
+
|
|
1302
|
+
result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
|
|
1303
|
+
result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
|
|
1304
|
+
|
|
1305
|
+
result_t x = square(std::sin(lat_delta)) + //
|
|
1306
|
+
std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta));
|
|
1307
|
+
|
|
1308
|
+
return 2 * std::asin(std::sqrt(x));
|
|
1309
|
+
}
|
|
1310
|
+
};
|
|
1311
|
+
|
|
1312
|
+
using distance_punned_t = float;
|
|
1313
|
+
using span_punned_t = span_gt<byte_t const>;
|
|
1314
|
+
|
|
1315
|
+
/**
|
|
1316
|
+
* @brief The signature of the user-defined function.
|
|
1317
|
+
* Can be just two array pointers, precompiled for a specific array length,
|
|
1318
|
+
* or include one or two array sizes as 64-bit unsigned integers.
|
|
1319
|
+
*/
|
|
1320
|
+
enum class metric_punned_signature_t {
|
|
1321
|
+
array_array_k = 0,
|
|
1322
|
+
array_array_size_k,
|
|
1323
|
+
};
|
|
1324
|
+
|
|
1325
|
+
/**
|
|
1326
|
+
* @brief Type-punned metric class, which unlike STL's `std::function` avoids any memory allocations.
|
|
1327
|
+
* It also provides additional APIs to check, if SIMD hardware-acceleration is available.
|
|
1328
|
+
* Wraps the `simsimd_metric_punned_t` when available. The auto-vectorized backend otherwise.
|
|
1329
|
+
*/
|
|
1330
|
+
class metric_punned_t {
|
|
1331
|
+
public:
|
|
1332
|
+
using scalar_t = byte_t;
|
|
1333
|
+
using result_t = distance_punned_t;
|
|
1334
|
+
|
|
1335
|
+
private:
|
|
1336
|
+
/// In the generalized function API all the are arguments are pointer-sized.
|
|
1337
|
+
using uptr_t = std::size_t;
|
|
1338
|
+
/// Distance function that takes two arrays and returns a scalar.
|
|
1339
|
+
using metric_array_array_t = result_t (*)(uptr_t, uptr_t);
|
|
1340
|
+
/// Distance function that takes two arrays and their length and returns a scalar.
|
|
1341
|
+
using metric_array_array_size_t = result_t (*)(uptr_t, uptr_t, uptr_t);
|
|
1342
|
+
/// Distance function callback, like `metric_array_array_size_t`, but depends on member variables.
|
|
1343
|
+
using metric_rounted_t = result_t (metric_punned_t::*)(uptr_t, uptr_t) const;
|
|
1344
|
+
|
|
1345
|
+
metric_rounted_t metric_routed_ = nullptr;
|
|
1346
|
+
uptr_t metric_ptr_ = 0;
|
|
1347
|
+
uptr_t metric_size_arg_ = 0;
|
|
1348
|
+
|
|
1349
|
+
std::size_t dimensions_ = 0;
|
|
1350
|
+
metric_kind_t metric_kind_ = metric_kind_t::unknown_k;
|
|
1351
|
+
scalar_kind_t scalar_kind_ = scalar_kind_t::unknown_k;
|
|
1352
|
+
|
|
1353
|
+
#if USEARCH_USE_SIMSIMD
|
|
1354
|
+
simsimd_capability_t isa_kind_ = simsimd_cap_serial_k;
|
|
1355
|
+
#endif
|
|
1356
|
+
|
|
1357
|
+
public:
|
|
1358
|
+
/**
|
|
1359
|
+
* @brief Computes the distance between two vectors of fixed length.
|
|
1360
|
+
*
|
|
1361
|
+
* ! This is the only relevant function in the object. Everything else is just dynamic dispatch logic.
|
|
1362
|
+
*/
|
|
1363
|
+
inline result_t operator()(byte_t const* a, byte_t const* b) const noexcept {
|
|
1364
|
+
return (this->*metric_routed_)(reinterpret_cast<uptr_t>(a), reinterpret_cast<uptr_t>(b));
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
inline metric_punned_t() noexcept = default;
|
|
1368
|
+
inline metric_punned_t(metric_punned_t const&) noexcept = default;
|
|
1369
|
+
inline metric_punned_t& operator=(metric_punned_t const&) noexcept = default;
|
|
1370
|
+
inline metric_punned_t( //
|
|
1371
|
+
std::size_t dimensions, //
|
|
1372
|
+
metric_kind_t metric_kind = metric_kind_t::l2sq_k, //
|
|
1373
|
+
scalar_kind_t scalar_kind = scalar_kind_t::f32_k) noexcept
|
|
1374
|
+
: metric_routed_(&metric_punned_t::invoke_array_array_size), metric_size_arg_(dimensions),
|
|
1375
|
+
dimensions_(dimensions), metric_kind_(metric_kind), scalar_kind_(scalar_kind) {
|
|
1376
|
+
|
|
1377
|
+
#if USEARCH_USE_SIMSIMD
|
|
1378
|
+
if (!configure_with_simsimd())
|
|
1379
|
+
configure_with_autovec();
|
|
1380
|
+
#else
|
|
1381
|
+
configure_with_autovec();
|
|
1382
|
+
#endif
|
|
1383
|
+
|
|
1384
|
+
if (scalar_kind == scalar_kind_t::b1x8_k)
|
|
1385
|
+
metric_size_arg_ = divide_round_up<CHAR_BIT>(dimensions_);
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
inline metric_punned_t( //
|
|
1389
|
+
std::size_t dimensions, //
|
|
1390
|
+
std::uintptr_t metric_uintptr, metric_punned_signature_t signature, //
|
|
1391
|
+
metric_kind_t metric_kind, //
|
|
1392
|
+
scalar_kind_t scalar_kind) noexcept
|
|
1393
|
+
: metric_routed_(signature == metric_punned_signature_t::array_array_k
|
|
1394
|
+
? &metric_punned_t::invoke_array_array
|
|
1395
|
+
: &metric_punned_t::invoke_array_array_size),
|
|
1396
|
+
metric_ptr_(metric_uintptr), metric_size_arg_(dimensions), dimensions_(dimensions), metric_kind_(metric_kind),
|
|
1397
|
+
scalar_kind_(scalar_kind) {
|
|
1398
|
+
|
|
1399
|
+
if (scalar_kind == scalar_kind_t::b1x8_k)
|
|
1400
|
+
metric_size_arg_ = divide_round_up<CHAR_BIT>(dimensions_);
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
inline std::size_t dimensions() const noexcept { return dimensions_; }
|
|
1404
|
+
inline metric_kind_t metric_kind() const noexcept { return metric_kind_; }
|
|
1405
|
+
inline scalar_kind_t scalar_kind() const noexcept { return scalar_kind_; }
|
|
1406
|
+
inline explicit operator bool() const noexcept { return metric_routed_ && metric_ptr_; }
|
|
1407
|
+
|
|
1408
|
+
/**
|
|
1409
|
+
* @brief Checks fi we've failed to initialized the metric with provided arguments.
|
|
1410
|
+
*
|
|
1411
|
+
* It's different from `operator bool()` when it comes to explicitly uninitialized metrics.
|
|
1412
|
+
* It's a common case, where a NULL state is created only to be overwritten later, when
|
|
1413
|
+
* we recover an old index state from a file or a network.
|
|
1414
|
+
*/
|
|
1415
|
+
inline bool missing() const noexcept { return !bool(*this) && metric_kind_ != metric_kind_t::unknown_k; }
|
|
1416
|
+
|
|
1417
|
+
inline char const* isa_name() const noexcept {
|
|
1418
|
+
if (!*this)
|
|
1419
|
+
return "uninitialized";
|
|
1420
|
+
|
|
1421
|
+
#if USEARCH_USE_SIMSIMD
|
|
1422
|
+
switch (isa_kind_) {
|
|
1423
|
+
case simsimd_cap_serial_k: return "serial";
|
|
1424
|
+
case simsimd_cap_neon_k: return "neon";
|
|
1425
|
+
case simsimd_cap_sve_k: return "sve";
|
|
1426
|
+
case simsimd_cap_haswell_k: return "haswell";
|
|
1427
|
+
case simsimd_cap_skylake_k: return "skylake";
|
|
1428
|
+
case simsimd_cap_ice_k: return "ice";
|
|
1429
|
+
case simsimd_cap_sapphire_k: return "sapphire";
|
|
1430
|
+
default: return "unknown";
|
|
1431
|
+
}
|
|
1432
|
+
#endif
|
|
1433
|
+
return "serial";
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
inline std::size_t bytes_per_vector() const noexcept {
|
|
1437
|
+
return divide_round_up<CHAR_BIT>(dimensions_ * bits_per_scalar(scalar_kind_));
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
inline std::size_t scalar_words() const noexcept {
|
|
1441
|
+
return divide_round_up(dimensions_ * bits_per_scalar(scalar_kind_), bits_per_scalar_word(scalar_kind_));
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
private:
|
|
1445
|
+
#if USEARCH_USE_SIMSIMD
|
|
1446
|
+
bool configure_with_simsimd(simsimd_capability_t simd_caps) noexcept {
|
|
1447
|
+
simsimd_metric_kind_t kind = simsimd_metric_unknown_k;
|
|
1448
|
+
simsimd_datatype_t datatype = simsimd_datatype_unknown_k;
|
|
1449
|
+
simsimd_capability_t allowed = simsimd_cap_any_k;
|
|
1450
|
+
switch (metric_kind_) {
|
|
1451
|
+
case metric_kind_t::ip_k: kind = simsimd_metric_dot_k; break;
|
|
1452
|
+
case metric_kind_t::cos_k: kind = simsimd_metric_cos_k; break;
|
|
1453
|
+
case metric_kind_t::l2sq_k: kind = simsimd_metric_l2sq_k; break;
|
|
1454
|
+
case metric_kind_t::hamming_k: kind = simsimd_metric_hamming_k; break;
|
|
1455
|
+
case metric_kind_t::tanimoto_k: kind = simsimd_metric_jaccard_k; break;
|
|
1456
|
+
case metric_kind_t::jaccard_k: kind = simsimd_metric_jaccard_k; break;
|
|
1457
|
+
default: break;
|
|
1458
|
+
}
|
|
1459
|
+
switch (scalar_kind_) {
|
|
1460
|
+
case scalar_kind_t::f32_k: datatype = simsimd_datatype_f32_k; break;
|
|
1461
|
+
case scalar_kind_t::f64_k: datatype = simsimd_datatype_f64_k; break;
|
|
1462
|
+
case scalar_kind_t::f16_k: datatype = simsimd_datatype_f16_k; break;
|
|
1463
|
+
case scalar_kind_t::i8_k: datatype = simsimd_datatype_i8_k; break;
|
|
1464
|
+
case scalar_kind_t::b1x8_k: datatype = simsimd_datatype_b8_k; break;
|
|
1465
|
+
default: break;
|
|
1466
|
+
}
|
|
1467
|
+
simsimd_metric_punned_t simd_metric = NULL;
|
|
1468
|
+
simsimd_capability_t simd_kind = simsimd_cap_any_k;
|
|
1469
|
+
simsimd_find_metric_punned(kind, datatype, simd_caps, allowed, &simd_metric, &simd_kind);
|
|
1470
|
+
if (simd_metric == nullptr)
|
|
1471
|
+
return false;
|
|
1472
|
+
|
|
1473
|
+
std::memcpy(&metric_ptr_, &simd_metric, sizeof(simd_metric));
|
|
1474
|
+
metric_routed_ = metric_kind_ == metric_kind_t::ip_k
|
|
1475
|
+
? reinterpret_cast<metric_rounted_t>(&metric_punned_t::invoke_simsimd_reverse)
|
|
1476
|
+
: reinterpret_cast<metric_rounted_t>(&metric_punned_t::invoke_simsimd);
|
|
1477
|
+
isa_kind_ = simd_kind;
|
|
1478
|
+
return true;
|
|
1479
|
+
}
|
|
1480
|
+
bool configure_with_simsimd() noexcept {
|
|
1481
|
+
static simsimd_capability_t static_capabilities = simsimd_capabilities();
|
|
1482
|
+
return configure_with_simsimd(static_capabilities);
|
|
1483
|
+
}
|
|
1484
|
+
result_t invoke_simsimd(uptr_t a, uptr_t b) const noexcept {
|
|
1485
|
+
simsimd_distance_t result;
|
|
1486
|
+
// Here `reinterpret_cast` raises warning... we know what we are doing!
|
|
1487
|
+
auto function_pointer = (simsimd_metric_punned_t)(metric_ptr_);
|
|
1488
|
+
function_pointer(reinterpret_cast<void const*>(a), reinterpret_cast<void const*>(b), metric_size_arg_, &result);
|
|
1489
|
+
return (result_t)result;
|
|
1490
|
+
}
|
|
1491
|
+
result_t invoke_simsimd_reverse(uptr_t a, uptr_t b) const noexcept { return 1 - invoke_simsimd(a, b); }
|
|
1492
|
+
#else
|
|
1493
|
+
bool configure_with_simsimd() noexcept { return false; }
|
|
1494
|
+
#endif
|
|
1495
|
+
result_t invoke_array_array_size(uptr_t a, uptr_t b) const noexcept {
|
|
1496
|
+
auto function_pointer = (metric_array_array_size_t)(metric_ptr_);
|
|
1497
|
+
result_t result = function_pointer(a, b, metric_size_arg_);
|
|
1498
|
+
return result;
|
|
1499
|
+
}
|
|
1500
|
+
result_t invoke_array_array(uptr_t a, uptr_t b) const noexcept {
|
|
1501
|
+
auto function_pointer = (metric_array_array_t)(metric_ptr_);
|
|
1502
|
+
result_t result = function_pointer(a, b);
|
|
1503
|
+
return result;
|
|
1504
|
+
}
|
|
1505
|
+
void configure_with_autovec() noexcept {
|
|
1506
|
+
switch (metric_kind_) {
|
|
1507
|
+
case metric_kind_t::ip_k: {
|
|
1508
|
+
switch (scalar_kind_) {
|
|
1509
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f32_t>>; break;
|
|
1510
|
+
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f16_t, f32_t>>; break;
|
|
1511
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<i8_t, f32_t>>; break;
|
|
1512
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_ip_gt<f64_t>>; break;
|
|
1513
|
+
default: metric_ptr_ = 0; break;
|
|
1514
|
+
}
|
|
1515
|
+
break;
|
|
1516
|
+
}
|
|
1517
|
+
case metric_kind_t::cos_k: {
|
|
1518
|
+
switch (scalar_kind_) {
|
|
1519
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f32_t>>; break;
|
|
1520
|
+
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f16_t, f32_t>>; break;
|
|
1521
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<i8_t, f32_t>>; break;
|
|
1522
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_cos_gt<f64_t>>; break;
|
|
1523
|
+
default: metric_ptr_ = 0; break;
|
|
1524
|
+
}
|
|
1525
|
+
break;
|
|
1526
|
+
}
|
|
1527
|
+
case metric_kind_t::l2sq_k: {
|
|
1528
|
+
switch (scalar_kind_) {
|
|
1529
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f32_t>>; break;
|
|
1530
|
+
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f16_t, f32_t>>; break;
|
|
1531
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<i8_t, f32_t>>; break;
|
|
1532
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_l2sq_gt<f64_t>>; break;
|
|
1533
|
+
default: metric_ptr_ = 0; break;
|
|
1534
|
+
}
|
|
1535
|
+
break;
|
|
1536
|
+
}
|
|
1537
|
+
case metric_kind_t::pearson_k: {
|
|
1538
|
+
switch (scalar_kind_) {
|
|
1539
|
+
case scalar_kind_t::i8_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<i8_t, f32_t>>; break;
|
|
1540
|
+
case scalar_kind_t::f16_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f16_t, f32_t>>; break;
|
|
1541
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f32_t>>; break;
|
|
1542
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_pearson_gt<f64_t>>; break;
|
|
1543
|
+
default: metric_ptr_ = 0; break;
|
|
1544
|
+
}
|
|
1545
|
+
break;
|
|
1546
|
+
}
|
|
1547
|
+
case metric_kind_t::haversine_k: {
|
|
1548
|
+
switch (scalar_kind_) {
|
|
1549
|
+
case scalar_kind_t::f16_k:
|
|
1550
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f16_t, f32_t>>;
|
|
1551
|
+
break;
|
|
1552
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f32_t>>; break;
|
|
1553
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_haversine_gt<f64_t>>; break;
|
|
1554
|
+
default: metric_ptr_ = 0; break;
|
|
1555
|
+
}
|
|
1556
|
+
break;
|
|
1557
|
+
}
|
|
1558
|
+
case metric_kind_t::divergence_k: {
|
|
1559
|
+
switch (scalar_kind_) {
|
|
1560
|
+
case scalar_kind_t::f16_k:
|
|
1561
|
+
metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f16_t, f32_t>>;
|
|
1562
|
+
break;
|
|
1563
|
+
case scalar_kind_t::f32_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f32_t>>; break;
|
|
1564
|
+
case scalar_kind_t::f64_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_divergence_gt<f64_t>>; break;
|
|
1565
|
+
default: metric_ptr_ = 0; break;
|
|
1566
|
+
}
|
|
1567
|
+
break;
|
|
1568
|
+
}
|
|
1569
|
+
case metric_kind_t::jaccard_k: // Equivalent to Tanimoto
|
|
1570
|
+
case metric_kind_t::tanimoto_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_tanimoto_gt<b1x8_t>>; break;
|
|
1571
|
+
case metric_kind_t::hamming_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_hamming_gt<b1x8_t>>; break;
|
|
1572
|
+
case metric_kind_t::sorensen_k: metric_ptr_ = (uptr_t)&equidimensional_<metric_sorensen_gt<b1x8_t>>; break;
|
|
1573
|
+
default: return;
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
template <typename typed_at>
|
|
1578
|
+
inline static result_t equidimensional_(uptr_t a, uptr_t b, uptr_t a_dimensions) noexcept {
|
|
1579
|
+
using scalar_t = typename typed_at::scalar_t;
|
|
1580
|
+
return static_cast<result_t>(typed_at{}((scalar_t const*)a, (scalar_t const*)b, a_dimensions));
|
|
1581
|
+
}
|
|
1582
|
+
};
|
|
1583
|
+
|
|
1584
|
+
/**
|
|
1585
|
+
* @brief View over a potentially-strided memory buffer, containing a row-major matrix.
|
|
1586
|
+
*/
|
|
1587
|
+
template <typename scalar_at> //
|
|
1588
|
+
class vectors_view_gt {
|
|
1589
|
+
using scalar_t = scalar_at;
|
|
1590
|
+
|
|
1591
|
+
scalar_t const* begin_{};
|
|
1592
|
+
std::size_t dimensions_{};
|
|
1593
|
+
std::size_t count_{};
|
|
1594
|
+
std::size_t stride_bytes_{};
|
|
1595
|
+
|
|
1596
|
+
public:
|
|
1597
|
+
vectors_view_gt() noexcept = default;
|
|
1598
|
+
vectors_view_gt(vectors_view_gt const&) noexcept = default;
|
|
1599
|
+
vectors_view_gt& operator=(vectors_view_gt const&) noexcept = default;
|
|
1600
|
+
|
|
1601
|
+
vectors_view_gt(scalar_t const* begin, std::size_t dimensions, std::size_t count = 1) noexcept
|
|
1602
|
+
: vectors_view_gt(begin, dimensions, count, dimensions * sizeof(scalar_at)) {}
|
|
1603
|
+
|
|
1604
|
+
vectors_view_gt(scalar_t const* begin, std::size_t dimensions, std::size_t count, std::size_t stride_bytes) noexcept
|
|
1605
|
+
: begin_(begin), dimensions_(dimensions), count_(count), stride_bytes_(stride_bytes) {}
|
|
1606
|
+
|
|
1607
|
+
explicit operator bool() const noexcept { return begin_; }
|
|
1608
|
+
std::size_t size() const noexcept { return count_; }
|
|
1609
|
+
std::size_t dimensions() const noexcept { return dimensions_; }
|
|
1610
|
+
std::size_t stride() const noexcept { return stride_bytes_; }
|
|
1611
|
+
scalar_t const* data() const noexcept { return begin_; }
|
|
1612
|
+
scalar_t const* at(std::size_t i) const noexcept {
|
|
1613
|
+
return reinterpret_cast<scalar_t const*>(reinterpret_cast<byte_t const*>(begin_) + i * stride_bytes_);
|
|
1614
|
+
}
|
|
1615
|
+
};
|
|
1616
|
+
|
|
1617
|
+
struct exact_offset_and_distance_t {
|
|
1618
|
+
u32_t offset;
|
|
1619
|
+
f32_t distance;
|
|
1620
|
+
};
|
|
1621
|
+
|
|
1622
|
+
using exact_search_results_t = vectors_view_gt<exact_offset_and_distance_t>;
|
|
1623
|
+
|
|
1624
|
+
/**
|
|
1625
|
+
* @brief Helper-structure for exact search operations.
|
|
1626
|
+
* Perfect if you have @b <1M vectors and @b <100 queries per call.
|
|
1627
|
+
*
|
|
1628
|
+
* Uses a 3-step procedure to minimize:
|
|
1629
|
+
* - cache-misses on vector lookups,
|
|
1630
|
+
* - multi-threaded contention on concurrent writes.
|
|
1631
|
+
*/
|
|
1632
|
+
class exact_search_t {
|
|
1633
|
+
|
|
1634
|
+
inline static bool smaller_distance(exact_offset_and_distance_t a, exact_offset_and_distance_t b) noexcept {
|
|
1635
|
+
return a.distance < b.distance;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
using keys_and_distances_t = buffer_gt<exact_offset_and_distance_t>;
|
|
1639
|
+
keys_and_distances_t keys_and_distances;
|
|
1640
|
+
|
|
1641
|
+
public:
|
|
1642
|
+
template <typename scalar_at, typename executor_at = dummy_executor_t, typename progress_at = dummy_progress_t>
|
|
1643
|
+
exact_search_results_t operator()( //
|
|
1644
|
+
vectors_view_gt<scalar_at> dataset, vectors_view_gt<scalar_at> queries, //
|
|
1645
|
+
std::size_t wanted, metric_punned_t const& metric, //
|
|
1646
|
+
executor_at&& executor = executor_at{}, progress_at&& progress = progress_at{}) {
|
|
1647
|
+
return operator()( //
|
|
1648
|
+
metric, //
|
|
1649
|
+
reinterpret_cast<byte_t const*>(dataset.data()), dataset.size(), dataset.stride(), //
|
|
1650
|
+
reinterpret_cast<byte_t const*>(queries.data()), queries.size(), queries.stride(), //
|
|
1651
|
+
wanted, executor, progress);
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
template <typename executor_at = dummy_executor_t, typename progress_at = dummy_progress_t>
|
|
1655
|
+
exact_search_results_t operator()( //
|
|
1656
|
+
byte_t const* dataset_data, std::size_t dataset_count, std::size_t dataset_stride, //
|
|
1657
|
+
byte_t const* queries_data, std::size_t queries_count, std::size_t queries_stride, //
|
|
1658
|
+
std::size_t wanted, metric_punned_t const& metric, executor_at&& executor = executor_at{},
|
|
1659
|
+
progress_at&& progress = progress_at{}) {
|
|
1660
|
+
|
|
1661
|
+
// Allocate temporary memory to store the distance matrix
|
|
1662
|
+
// Previous version didn't need temporary memory, but the performance was much lower.
|
|
1663
|
+
// In the new design we keep two buffers - original and transposed, as in-place transpositions
|
|
1664
|
+
// of non-rectangular matrixes is expensive.
|
|
1665
|
+
std::size_t tasks_count = dataset_count * queries_count;
|
|
1666
|
+
if (keys_and_distances.size() < tasks_count * 2)
|
|
1667
|
+
keys_and_distances = keys_and_distances_t(tasks_count * 2);
|
|
1668
|
+
if (keys_and_distances.size() < tasks_count * 2)
|
|
1669
|
+
return {};
|
|
1670
|
+
|
|
1671
|
+
exact_offset_and_distance_t* keys_and_distances_per_dataset = keys_and_distances.data();
|
|
1672
|
+
exact_offset_and_distance_t* keys_and_distances_per_query = keys_and_distances_per_dataset + tasks_count;
|
|
1673
|
+
|
|
1674
|
+
// §1. Compute distances in a data-parallel fashion
|
|
1675
|
+
std::atomic<std::size_t> processed{0};
|
|
1676
|
+
executor.dynamic(dataset_count, [&](std::size_t thread_idx, std::size_t dataset_idx) {
|
|
1677
|
+
byte_t const* dataset = dataset_data + dataset_idx * dataset_stride;
|
|
1678
|
+
for (std::size_t query_idx = 0; query_idx != queries_count; ++query_idx) {
|
|
1679
|
+
byte_t const* query = queries_data + query_idx * queries_stride;
|
|
1680
|
+
auto distance = metric(dataset, query);
|
|
1681
|
+
std::size_t task_idx = queries_count * dataset_idx + query_idx;
|
|
1682
|
+
keys_and_distances_per_dataset[task_idx].offset = static_cast<u32_t>(dataset_idx);
|
|
1683
|
+
keys_and_distances_per_dataset[task_idx].distance = static_cast<f32_t>(distance);
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
// It's more efficient in this case to report progress from a single thread
|
|
1687
|
+
processed += queries_count;
|
|
1688
|
+
if (thread_idx == 0)
|
|
1689
|
+
if (!progress(processed.load(), tasks_count))
|
|
1690
|
+
return false;
|
|
1691
|
+
return true;
|
|
1692
|
+
});
|
|
1693
|
+
if (processed.load() != tasks_count)
|
|
1694
|
+
return {};
|
|
1695
|
+
|
|
1696
|
+
// §2. Transpose in a single thread to avoid contention writing into the same memory buffers
|
|
1697
|
+
for (std::size_t query_idx = 0; query_idx != queries_count; ++query_idx) {
|
|
1698
|
+
for (std::size_t dataset_idx = 0; dataset_idx != dataset_count; ++dataset_idx) {
|
|
1699
|
+
std::size_t from_idx = queries_count * dataset_idx + query_idx;
|
|
1700
|
+
std::size_t to_idx = dataset_count * query_idx + dataset_idx;
|
|
1701
|
+
keys_and_distances_per_query[to_idx] = keys_and_distances_per_dataset[from_idx];
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
// §3. Partial-sort every query result
|
|
1706
|
+
executor.fixed(queries_count, [&](std::size_t, std::size_t query_idx) {
|
|
1707
|
+
auto start = keys_and_distances_per_query + dataset_count * query_idx;
|
|
1708
|
+
if (wanted > 1) {
|
|
1709
|
+
// TODO: Consider alternative sorting approaches
|
|
1710
|
+
// radix_sort(start, start + dataset_count, wanted);
|
|
1711
|
+
// std::sort(start, start + dataset_count, &smaller_distance);
|
|
1712
|
+
std::partial_sort(start, start + wanted, start + dataset_count, &smaller_distance);
|
|
1713
|
+
} else {
|
|
1714
|
+
auto min_it = std::min_element(start, start + dataset_count, &smaller_distance);
|
|
1715
|
+
if (min_it != start)
|
|
1716
|
+
std::swap(*min_it, *start);
|
|
1717
|
+
}
|
|
1718
|
+
});
|
|
1719
|
+
|
|
1720
|
+
// At the end report the latest numbers, because the reporter thread may be finished earlier
|
|
1721
|
+
progress(tasks_count, tasks_count);
|
|
1722
|
+
return {keys_and_distances_per_query, wanted, queries_count,
|
|
1723
|
+
dataset_count * sizeof(exact_offset_and_distance_t)};
|
|
1724
|
+
}
|
|
1725
|
+
};
|
|
1726
|
+
|
|
1727
|
+
/**
|
|
1728
|
+
* @brief C++11 Multi-Hash-Set with Linear Probing.
|
|
1729
|
+
*
|
|
1730
|
+
* - Allows multiple equivalent values,
|
|
1731
|
+
* - Supports transparent hashing and equality operator.
|
|
1732
|
+
* - Doesn't throw exceptions, if forbidden.
|
|
1733
|
+
* - Doesn't need reserving a value for deletions.
|
|
1734
|
+
*
|
|
1735
|
+
* @section Layout
|
|
1736
|
+
*
|
|
1737
|
+
* For every slot we store 2 extra bits for 3 possible states: empty, populated, or deleted.
|
|
1738
|
+
* With linear probing the hashes at the end of the populated region will spill into its first half.
|
|
1739
|
+
*/
|
|
1740
|
+
template <typename element_at, typename hash_at, typename equals_at, typename allocator_at = std::allocator<char>>
|
|
1741
|
+
class flat_hash_multi_set_gt {
|
|
1742
|
+
public:
|
|
1743
|
+
using element_t = element_at;
|
|
1744
|
+
using hash_t = hash_at;
|
|
1745
|
+
using equals_t = equals_at;
|
|
1746
|
+
using allocator_t = allocator_at;
|
|
1747
|
+
|
|
1748
|
+
static constexpr std::size_t slots_per_bucket() { return 64; }
|
|
1749
|
+
static constexpr std::size_t bytes_per_bucket() {
|
|
1750
|
+
return slots_per_bucket() * sizeof(element_t) + sizeof(bucket_header_t);
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
private:
|
|
1754
|
+
struct bucket_header_t {
|
|
1755
|
+
std::uint64_t populated{};
|
|
1756
|
+
std::uint64_t deleted{};
|
|
1757
|
+
};
|
|
1758
|
+
char* data_ = nullptr;
|
|
1759
|
+
std::size_t buckets_ = 0;
|
|
1760
|
+
std::size_t populated_slots_ = 0;
|
|
1761
|
+
/// @brief Number of slots
|
|
1762
|
+
std::size_t capacity_slots_ = 0;
|
|
1763
|
+
|
|
1764
|
+
struct slot_ref_t {
|
|
1765
|
+
bucket_header_t& header;
|
|
1766
|
+
std::uint64_t mask;
|
|
1767
|
+
element_t& element;
|
|
1768
|
+
};
|
|
1769
|
+
|
|
1770
|
+
slot_ref_t slot_ref(char* data, std::size_t slot_index) const noexcept {
|
|
1771
|
+
std::size_t bucket_index = slot_index / slots_per_bucket();
|
|
1772
|
+
std::size_t in_bucket_index = slot_index % slots_per_bucket();
|
|
1773
|
+
auto bucket_pointer = data + bytes_per_bucket() * bucket_index;
|
|
1774
|
+
auto slot_pointer = bucket_pointer + sizeof(bucket_header_t) + sizeof(element_t) * in_bucket_index;
|
|
1775
|
+
return {
|
|
1776
|
+
*reinterpret_cast<bucket_header_t*>(bucket_pointer),
|
|
1777
|
+
static_cast<std::uint64_t>(1ull) << in_bucket_index,
|
|
1778
|
+
*reinterpret_cast<element_t*>(slot_pointer),
|
|
1779
|
+
};
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
slot_ref_t slot_ref(std::size_t slot_index) const noexcept { return slot_ref(data_, slot_index); }
|
|
1783
|
+
|
|
1784
|
+
bool populate_slot(slot_ref_t slot, element_t const& new_element) {
|
|
1785
|
+
if (slot.header.populated & slot.mask) {
|
|
1786
|
+
slot.element = new_element;
|
|
1787
|
+
slot.header.deleted &= ~slot.mask;
|
|
1788
|
+
return false;
|
|
1789
|
+
} else {
|
|
1790
|
+
new (&slot.element) element_t(new_element);
|
|
1791
|
+
slot.header.populated |= slot.mask;
|
|
1792
|
+
return true;
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
public:
|
|
1797
|
+
std::size_t size() const noexcept { return populated_slots_; }
|
|
1798
|
+
std::size_t capacity() const noexcept { return capacity_slots_; }
|
|
1799
|
+
|
|
1800
|
+
flat_hash_multi_set_gt() noexcept {}
|
|
1801
|
+
~flat_hash_multi_set_gt() noexcept { reset(); }
|
|
1802
|
+
|
|
1803
|
+
flat_hash_multi_set_gt(flat_hash_multi_set_gt const& other) {
|
|
1804
|
+
|
|
1805
|
+
// On Windows allocating a zero-size array would fail
|
|
1806
|
+
if (!other.buckets_) {
|
|
1807
|
+
reset();
|
|
1808
|
+
return;
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
// Allocate new memory
|
|
1812
|
+
data_ = (char*)allocator_t{}.allocate(other.buckets_ * bytes_per_bucket());
|
|
1813
|
+
if (!data_)
|
|
1814
|
+
throw std::bad_alloc();
|
|
1815
|
+
|
|
1816
|
+
// Copy metadata
|
|
1817
|
+
buckets_ = other.buckets_;
|
|
1818
|
+
populated_slots_ = other.populated_slots_;
|
|
1819
|
+
capacity_slots_ = other.capacity_slots_;
|
|
1820
|
+
|
|
1821
|
+
// Initialize new buckets to empty
|
|
1822
|
+
std::memset(data_, 0, buckets_ * bytes_per_bucket());
|
|
1823
|
+
|
|
1824
|
+
// Copy elements and bucket headers
|
|
1825
|
+
for (std::size_t i = 0; i < capacity_slots_; ++i) {
|
|
1826
|
+
slot_ref_t old_slot = other.slot_ref(i);
|
|
1827
|
+
if ((old_slot.header.populated & old_slot.mask) && !(old_slot.header.deleted & old_slot.mask)) {
|
|
1828
|
+
slot_ref_t new_slot = slot_ref(i);
|
|
1829
|
+
populate_slot(new_slot, old_slot.element);
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
flat_hash_multi_set_gt& operator=(flat_hash_multi_set_gt const& other) {
|
|
1835
|
+
|
|
1836
|
+
// On Windows allocating a zero-size array would fail
|
|
1837
|
+
if (!other.buckets_) {
|
|
1838
|
+
reset();
|
|
1839
|
+
return *this;
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// Handle self-assignment
|
|
1843
|
+
if (this == &other)
|
|
1844
|
+
return *this;
|
|
1845
|
+
|
|
1846
|
+
// Clear existing data
|
|
1847
|
+
clear();
|
|
1848
|
+
if (data_)
|
|
1849
|
+
allocator_t{}.deallocate(data_, buckets_ * bytes_per_bucket());
|
|
1850
|
+
|
|
1851
|
+
// Allocate new memory
|
|
1852
|
+
data_ = (char*)allocator_t{}.allocate(other.buckets_ * bytes_per_bucket());
|
|
1853
|
+
if (!data_)
|
|
1854
|
+
throw std::bad_alloc();
|
|
1855
|
+
|
|
1856
|
+
// Copy metadata
|
|
1857
|
+
buckets_ = other.buckets_;
|
|
1858
|
+
populated_slots_ = other.populated_slots_;
|
|
1859
|
+
capacity_slots_ = other.capacity_slots_;
|
|
1860
|
+
|
|
1861
|
+
// Initialize new buckets to empty
|
|
1862
|
+
std::memset(data_, 0, buckets_ * bytes_per_bucket());
|
|
1863
|
+
|
|
1864
|
+
// Copy elements and bucket headers
|
|
1865
|
+
for (std::size_t i = 0; i < capacity_slots_; ++i) {
|
|
1866
|
+
slot_ref_t old_slot = other.slot_ref(i);
|
|
1867
|
+
if ((old_slot.header.populated & old_slot.mask) && !(old_slot.header.deleted & old_slot.mask)) {
|
|
1868
|
+
slot_ref_t new_slot = slot_ref(i);
|
|
1869
|
+
populate_slot(new_slot, old_slot.element);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
return *this;
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
void clear() noexcept {
|
|
1877
|
+
// Call the destructors
|
|
1878
|
+
for (std::size_t i = 0; i < capacity_slots_; ++i) {
|
|
1879
|
+
slot_ref_t slot = slot_ref(i);
|
|
1880
|
+
if ((slot.header.populated & slot.mask) & (~slot.header.deleted & slot.mask))
|
|
1881
|
+
slot.element.~element_t();
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
// Reset populated slots count
|
|
1885
|
+
if (data_)
|
|
1886
|
+
std::memset(data_, 0, buckets_ * bytes_per_bucket());
|
|
1887
|
+
populated_slots_ = 0;
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
void reset() noexcept {
|
|
1891
|
+
clear(); // Clear all elements
|
|
1892
|
+
if (data_)
|
|
1893
|
+
allocator_t{}.deallocate(data_, buckets_ * bytes_per_bucket());
|
|
1894
|
+
buckets_ = 0;
|
|
1895
|
+
populated_slots_ = 0;
|
|
1896
|
+
capacity_slots_ = 0;
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
bool try_reserve(std::size_t capacity) noexcept {
|
|
1900
|
+
if (capacity * 3u <= capacity_slots_ * 2u)
|
|
1901
|
+
return true;
|
|
1902
|
+
|
|
1903
|
+
// Calculate new sizes
|
|
1904
|
+
std::size_t new_slots = ceil2((capacity * 3ul) / 2ul);
|
|
1905
|
+
std::size_t new_buckets = divide_round_up<slots_per_bucket()>(new_slots);
|
|
1906
|
+
new_slots = new_buckets * slots_per_bucket(); // This must be a power of two!
|
|
1907
|
+
std::size_t new_bytes = new_buckets * bytes_per_bucket();
|
|
1908
|
+
|
|
1909
|
+
// Allocate new memory
|
|
1910
|
+
char* new_data = (char*)allocator_t{}.allocate(new_bytes);
|
|
1911
|
+
if (!new_data)
|
|
1912
|
+
return false;
|
|
1913
|
+
|
|
1914
|
+
// Initialize new buckets to empty
|
|
1915
|
+
std::memset(new_data, 0, new_bytes);
|
|
1916
|
+
|
|
1917
|
+
// Rehash and copy existing elements to new_data
|
|
1918
|
+
hash_t hasher;
|
|
1919
|
+
for (std::size_t i = 0; i < capacity_slots_; ++i) {
|
|
1920
|
+
slot_ref_t old_slot = slot_ref(i);
|
|
1921
|
+
if ((~old_slot.header.populated & old_slot.mask) | (old_slot.header.deleted & old_slot.mask))
|
|
1922
|
+
continue;
|
|
1923
|
+
|
|
1924
|
+
// Rehash
|
|
1925
|
+
std::size_t hash_value = hasher(old_slot.element);
|
|
1926
|
+
std::size_t new_slot_index = hash_value & (new_slots - 1);
|
|
1927
|
+
|
|
1928
|
+
// Linear probing to find an empty slot in new_data
|
|
1929
|
+
while (true) {
|
|
1930
|
+
slot_ref_t new_slot = slot_ref(new_data, new_slot_index);
|
|
1931
|
+
if (!(new_slot.header.populated & new_slot.mask) || (new_slot.header.deleted & new_slot.mask)) {
|
|
1932
|
+
populate_slot(new_slot, std::move(old_slot.element));
|
|
1933
|
+
new_slot.header.populated |= new_slot.mask;
|
|
1934
|
+
break;
|
|
1935
|
+
}
|
|
1936
|
+
new_slot_index = (new_slot_index + 1) & (new_slots - 1);
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
// Deallocate old data and update pointers and sizes
|
|
1941
|
+
if (data_)
|
|
1942
|
+
allocator_t{}.deallocate(data_, buckets_ * bytes_per_bucket());
|
|
1943
|
+
data_ = new_data;
|
|
1944
|
+
buckets_ = new_buckets;
|
|
1945
|
+
capacity_slots_ = new_slots;
|
|
1946
|
+
|
|
1947
|
+
return true;
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
template <typename query_at> class equal_iterator_gt {
|
|
1951
|
+
public:
|
|
1952
|
+
using iterator_category = std::forward_iterator_tag;
|
|
1953
|
+
using value_type = element_t;
|
|
1954
|
+
using difference_type = std::ptrdiff_t;
|
|
1955
|
+
using pointer = element_t*;
|
|
1956
|
+
using reference = element_t&;
|
|
1957
|
+
|
|
1958
|
+
equal_iterator_gt(std::size_t index, flat_hash_multi_set_gt* parent, query_at const& query,
|
|
1959
|
+
equals_t const& equals)
|
|
1960
|
+
: index_(index), parent_(parent), query_(query), equals_(equals) {}
|
|
1961
|
+
|
|
1962
|
+
// Pre-increment
|
|
1963
|
+
equal_iterator_gt& operator++() {
|
|
1964
|
+
do {
|
|
1965
|
+
index_ = (index_ + 1) & (parent_->capacity_slots_ - 1);
|
|
1966
|
+
} while (!equals_(parent_->slot_ref(index_).element, query_) &&
|
|
1967
|
+
(parent_->slot_ref(index_).header.populated & parent_->slot_ref(index_).mask));
|
|
1968
|
+
return *this;
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
equal_iterator_gt operator++(int) {
|
|
1972
|
+
equal_iterator_gt temp = *this;
|
|
1973
|
+
++(*this);
|
|
1974
|
+
return temp;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
reference operator*() { return parent_->slot_ref(index_).element; }
|
|
1978
|
+
pointer operator->() { return &parent_->slot_ref(index_).element; }
|
|
1979
|
+
bool operator!=(equal_iterator_gt const& other) const { return !(*this == other); }
|
|
1980
|
+
bool operator==(equal_iterator_gt const& other) const {
|
|
1981
|
+
return index_ == other.index_ && parent_ == other.parent_;
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
private:
|
|
1985
|
+
std::size_t index_;
|
|
1986
|
+
flat_hash_multi_set_gt* parent_;
|
|
1987
|
+
query_at query_; // Store the query object
|
|
1988
|
+
equals_t equals_; // Store the equals functor
|
|
1989
|
+
};
|
|
1990
|
+
|
|
1991
|
+
/**
|
|
1992
|
+
* @brief Returns an iterator range of all elements matching the given query.
|
|
1993
|
+
*
|
|
1994
|
+
* Technically, the second iterator points to the first empty slot after a
|
|
1995
|
+
* range of equal values and non-equal values with similar hashes.
|
|
1996
|
+
*/
|
|
1997
|
+
template <typename query_at>
|
|
1998
|
+
std::pair<equal_iterator_gt<query_at>, equal_iterator_gt<query_at>>
|
|
1999
|
+
equal_range(query_at const& query) const noexcept {
|
|
2000
|
+
|
|
2001
|
+
equals_t equals;
|
|
2002
|
+
auto this_ptr = const_cast<flat_hash_multi_set_gt*>(this);
|
|
2003
|
+
auto end = equal_iterator_gt<query_at>(capacity_slots_, this_ptr, query, equals);
|
|
2004
|
+
if (!capacity_slots_)
|
|
2005
|
+
return {end, end};
|
|
2006
|
+
|
|
2007
|
+
hash_t hasher;
|
|
2008
|
+
std::size_t hash_value = hasher(query);
|
|
2009
|
+
std::size_t first_equal_index = hash_value & (capacity_slots_ - 1);
|
|
2010
|
+
std::size_t const start_index = first_equal_index;
|
|
2011
|
+
|
|
2012
|
+
// Linear probing to find the first equal element
|
|
2013
|
+
do {
|
|
2014
|
+
slot_ref_t slot = slot_ref(first_equal_index);
|
|
2015
|
+
if (slot.header.populated & ~slot.header.deleted & slot.mask) {
|
|
2016
|
+
if (equals(slot.element, query))
|
|
2017
|
+
break;
|
|
2018
|
+
}
|
|
2019
|
+
// Stop if we find an empty slot
|
|
2020
|
+
else if (~slot.header.populated & slot.mask)
|
|
2021
|
+
return {end, end};
|
|
2022
|
+
|
|
2023
|
+
// Move to the next slot
|
|
2024
|
+
first_equal_index = (first_equal_index + 1) & (capacity_slots_ - 1);
|
|
2025
|
+
} while (first_equal_index != start_index);
|
|
2026
|
+
|
|
2027
|
+
// If no matching element was found, return end iterators
|
|
2028
|
+
if (first_equal_index == capacity_slots_)
|
|
2029
|
+
return {end, end};
|
|
2030
|
+
|
|
2031
|
+
// Start from the first matching element and find the end of the populated range
|
|
2032
|
+
std::size_t first_empty_index = first_equal_index;
|
|
2033
|
+
do {
|
|
2034
|
+
first_empty_index = (first_empty_index + 1) & (capacity_slots_ - 1);
|
|
2035
|
+
slot_ref_t slot = slot_ref(first_empty_index);
|
|
2036
|
+
|
|
2037
|
+
// If we find an empty slot, this is our end
|
|
2038
|
+
if (~slot.header.populated & slot.mask)
|
|
2039
|
+
break;
|
|
2040
|
+
} while (first_empty_index != start_index);
|
|
2041
|
+
|
|
2042
|
+
return {equal_iterator_gt<query_at>(first_equal_index, this_ptr, query, equals),
|
|
2043
|
+
equal_iterator_gt<query_at>(first_empty_index, this_ptr, query, equals)};
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
template <typename similar_at> bool pop_first(similar_at&& query, element_t& popped_value) noexcept {
|
|
2047
|
+
|
|
2048
|
+
if (!capacity_slots_)
|
|
2049
|
+
return false;
|
|
2050
|
+
|
|
2051
|
+
hash_t hasher;
|
|
2052
|
+
equals_t equals;
|
|
2053
|
+
std::size_t hash_value = hasher(query);
|
|
2054
|
+
std::size_t slot_index = hash_value & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
|
|
2055
|
+
std::size_t start_index = slot_index; // To detect loop in probing
|
|
2056
|
+
|
|
2057
|
+
// Linear probing to find the first match
|
|
2058
|
+
do {
|
|
2059
|
+
slot_ref_t slot = slot_ref(slot_index);
|
|
2060
|
+
if (slot.header.populated & slot.mask) {
|
|
2061
|
+
if ((~slot.header.deleted & slot.mask) && equals(slot.element, query)) {
|
|
2062
|
+
// Found a match, mark as deleted
|
|
2063
|
+
slot.header.deleted |= slot.mask;
|
|
2064
|
+
--populated_slots_;
|
|
2065
|
+
popped_value = slot.element;
|
|
2066
|
+
return true; // Successfully removed
|
|
2067
|
+
}
|
|
2068
|
+
} else {
|
|
2069
|
+
// Stop if we find an empty slot
|
|
2070
|
+
break;
|
|
2071
|
+
}
|
|
2072
|
+
|
|
2073
|
+
// Move to the next slot
|
|
2074
|
+
slot_index = (slot_index + 1) & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
|
|
2075
|
+
} while (slot_index != start_index);
|
|
2076
|
+
|
|
2077
|
+
return false; // No match found
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
template <typename similar_at> std::size_t erase(similar_at&& query) noexcept {
|
|
2081
|
+
|
|
2082
|
+
if (!capacity_slots_)
|
|
2083
|
+
return 0;
|
|
2084
|
+
|
|
2085
|
+
hash_t hasher;
|
|
2086
|
+
equals_t equals;
|
|
2087
|
+
std::size_t hash_value = hasher(query);
|
|
2088
|
+
std::size_t slot_index = hash_value & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
|
|
2089
|
+
std::size_t const start_index = slot_index; // To detect loop in probing
|
|
2090
|
+
std::size_t count = 0; // Count of elements removed
|
|
2091
|
+
|
|
2092
|
+
// Linear probing to find all matches
|
|
2093
|
+
do {
|
|
2094
|
+
slot_ref_t slot = slot_ref(slot_index);
|
|
2095
|
+
if (slot.header.populated & slot.mask) {
|
|
2096
|
+
if ((~slot.header.deleted & slot.mask) && equals(slot.element, query)) {
|
|
2097
|
+
// Found a match, mark as deleted
|
|
2098
|
+
slot.header.deleted |= slot.mask;
|
|
2099
|
+
--populated_slots_;
|
|
2100
|
+
++count; // Increment count of elements removed
|
|
2101
|
+
}
|
|
2102
|
+
} else {
|
|
2103
|
+
// Stop if we find an empty slot
|
|
2104
|
+
break;
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
// Move to the next slot
|
|
2108
|
+
slot_index = (slot_index + 1) & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
|
|
2109
|
+
} while (slot_index != start_index);
|
|
2110
|
+
|
|
2111
|
+
return count; // Return the number of elements removed
|
|
2112
|
+
}
|
|
2113
|
+
|
|
2114
|
+
template <typename similar_at> element_t const* find(similar_at&& query) const noexcept {
|
|
2115
|
+
|
|
2116
|
+
if (!capacity_slots_)
|
|
2117
|
+
return nullptr;
|
|
2118
|
+
|
|
2119
|
+
hash_t hasher;
|
|
2120
|
+
equals_t equals;
|
|
2121
|
+
std::size_t hash_value = hasher(query);
|
|
2122
|
+
std::size_t slot_index = hash_value & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
|
|
2123
|
+
std::size_t start_index = slot_index; // To detect loop in probing
|
|
2124
|
+
|
|
2125
|
+
// Linear probing to find the first match
|
|
2126
|
+
do {
|
|
2127
|
+
slot_ref_t slot = slot_ref(slot_index);
|
|
2128
|
+
if (slot.header.populated & slot.mask) {
|
|
2129
|
+
if ((~slot.header.deleted & slot.mask) && equals(slot.element, query))
|
|
2130
|
+
return &slot.element; // Found a match, return pointer to the element
|
|
2131
|
+
} else {
|
|
2132
|
+
// Stop if we find an empty slot
|
|
2133
|
+
break;
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
// Move to the next slot
|
|
2137
|
+
slot_index = (slot_index + 1) & (capacity_slots_ - 1); // Assuming capacity_slots_ is a power of 2
|
|
2138
|
+
} while (slot_index != start_index);
|
|
2139
|
+
|
|
2140
|
+
return nullptr; // No match found
|
|
2141
|
+
}
|
|
2142
|
+
|
|
2143
|
+
element_t const* end() const noexcept { return nullptr; }
|
|
2144
|
+
|
|
2145
|
+
template <typename func_at> void for_each(func_at&& func) const {
|
|
2146
|
+
for (std::size_t bucket_index = 0; bucket_index < buckets_; ++bucket_index) {
|
|
2147
|
+
auto bucket_pointer = data_ + bytes_per_bucket() * bucket_index;
|
|
2148
|
+
bucket_header_t& header = *reinterpret_cast<bucket_header_t*>(bucket_pointer);
|
|
2149
|
+
std::uint64_t populated = header.populated;
|
|
2150
|
+
std::uint64_t deleted = header.deleted;
|
|
2151
|
+
|
|
2152
|
+
// Iterate through slots in the bucket
|
|
2153
|
+
for (std::size_t in_bucket_index = 0; in_bucket_index < slots_per_bucket(); ++in_bucket_index) {
|
|
2154
|
+
std::uint64_t mask = std::uint64_t(1ull) << in_bucket_index;
|
|
2155
|
+
|
|
2156
|
+
// Check if the slot is populated and not deleted
|
|
2157
|
+
if ((populated & ~deleted) & mask) {
|
|
2158
|
+
auto slot_pointer = bucket_pointer + sizeof(bucket_header_t) + sizeof(element_t) * in_bucket_index;
|
|
2159
|
+
element_t const& element = *reinterpret_cast<element_t const*>(slot_pointer);
|
|
2160
|
+
func(element);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
template <typename similar_at> std::size_t count(similar_at&& query) const noexcept {
|
|
2167
|
+
|
|
2168
|
+
if (!capacity_slots_)
|
|
2169
|
+
return 0;
|
|
2170
|
+
|
|
2171
|
+
hash_t hasher;
|
|
2172
|
+
equals_t equals;
|
|
2173
|
+
std::size_t hash_value = hasher(query);
|
|
2174
|
+
std::size_t slot_index = hash_value & (capacity_slots_ - 1);
|
|
2175
|
+
std::size_t start_index = slot_index; // To detect loop in probing
|
|
2176
|
+
std::size_t count = 0;
|
|
2177
|
+
|
|
2178
|
+
// Linear probing to find the range
|
|
2179
|
+
do {
|
|
2180
|
+
slot_ref_t slot = slot_ref(slot_index);
|
|
2181
|
+
if ((slot.header.populated & slot.mask) && (~slot.header.deleted & slot.mask)) {
|
|
2182
|
+
if (equals(slot.element, query))
|
|
2183
|
+
++count;
|
|
2184
|
+
} else if (~slot.header.populated & slot.mask) {
|
|
2185
|
+
// Stop if we find an empty slot
|
|
2186
|
+
break;
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
// Move to the next slot
|
|
2190
|
+
slot_index = (slot_index + 1) & (capacity_slots_ - 1);
|
|
2191
|
+
} while (slot_index != start_index);
|
|
2192
|
+
|
|
2193
|
+
return count;
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
template <typename similar_at> bool contains(similar_at&& query) const noexcept {
|
|
2197
|
+
|
|
2198
|
+
if (!capacity_slots_)
|
|
2199
|
+
return false;
|
|
2200
|
+
|
|
2201
|
+
hash_t hasher;
|
|
2202
|
+
equals_t equals;
|
|
2203
|
+
std::size_t hash_value = hasher(query);
|
|
2204
|
+
std::size_t slot_index = hash_value & (capacity_slots_ - 1);
|
|
2205
|
+
std::size_t start_index = slot_index; // To detect loop in probing
|
|
2206
|
+
|
|
2207
|
+
// Linear probing to find the first match
|
|
2208
|
+
do {
|
|
2209
|
+
slot_ref_t slot = slot_ref(slot_index);
|
|
2210
|
+
if (slot.header.populated & slot.mask) {
|
|
2211
|
+
if ((~slot.header.deleted & slot.mask) && equals(slot.element, query))
|
|
2212
|
+
return true; // Found a match, exit early
|
|
2213
|
+
} else
|
|
2214
|
+
// Stop if we find an empty slot
|
|
2215
|
+
break;
|
|
2216
|
+
|
|
2217
|
+
// Move to the next slot
|
|
2218
|
+
slot_index = (slot_index + 1) & (capacity_slots_ - 1);
|
|
2219
|
+
} while (slot_index != start_index);
|
|
2220
|
+
|
|
2221
|
+
return false; // No match found
|
|
2222
|
+
}
|
|
2223
|
+
|
|
2224
|
+
void reserve(std::size_t capacity) {
|
|
2225
|
+
if (!try_reserve(capacity))
|
|
2226
|
+
throw std::bad_alloc();
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
bool try_emplace(element_t const& element) noexcept {
|
|
2230
|
+
// Check if we need to resize
|
|
2231
|
+
if (populated_slots_ * 3u >= capacity_slots_ * 2u)
|
|
2232
|
+
if (!try_reserve(populated_slots_ + 1))
|
|
2233
|
+
return false;
|
|
2234
|
+
|
|
2235
|
+
hash_t hasher;
|
|
2236
|
+
std::size_t hash_value = hasher(element);
|
|
2237
|
+
std::size_t slot_index = hash_value & (capacity_slots_ - 1);
|
|
2238
|
+
|
|
2239
|
+
// Linear probing
|
|
2240
|
+
while (true) {
|
|
2241
|
+
slot_ref_t slot = slot_ref(slot_index);
|
|
2242
|
+
if ((~slot.header.populated & slot.mask) | (slot.header.deleted & slot.mask)) {
|
|
2243
|
+
// Found an empty or deleted slot
|
|
2244
|
+
populate_slot(slot, element);
|
|
2245
|
+
++populated_slots_;
|
|
2246
|
+
return true;
|
|
2247
|
+
}
|
|
2248
|
+
// Move to the next slot
|
|
2249
|
+
slot_index = (slot_index + 1) & (capacity_slots_ - 1);
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
};
|
|
2253
|
+
|
|
2254
|
+
} // namespace usearch
|
|
2255
|
+
} // namespace unum
|