usearch 0.22.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +3 -3
- package/.releaserc +10 -2
- package/.vscode/launch.json +10 -10
- package/.vscode/settings.json +9 -1
- package/.vscode/tasks.json +4 -4
- package/CMakeLists.txt +6 -1
- package/Cargo.toml +2 -2
- package/README.md +35 -41
- package/build.rs +2 -2
- package/c/CMakeLists.txt +48 -0
- package/c/README.md +34 -0
- package/c/lib.cpp +83 -72
- package/c/test.c +20 -20
- package/c/usearch.h +132 -32
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +3 -1
- package/cpp/README.md +16 -16
- package/cpp/bench.cpp +50 -126
- package/cpp/test.cpp +164 -80
- package/docs/benchmarks.md +26 -5
- package/docs/c/reference.rst +3 -3
- package/docs/compilation.md +31 -49
- package/docs/cpp/index.rst +1 -1
- package/docs/cpp/reference.rst +4 -4
- package/golang/lib.go +275 -141
- package/golang/lib_test.go +105 -0
- package/include/usearch/index.hpp +1681 -1656
- package/include/usearch/index_dense.hpp +1400 -0
- package/include/usearch/index_plugins.hpp +1263 -0
- package/java/README.md +1 -1
- package/java/cloud/unum/usearch/Index.java +10 -10
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +34 -37
- package/java/test/IndexTest.java +1 -1
- package/javascript/README.md +6 -6
- package/javascript/lib.cpp +94 -55
- package/javascript/test.js +25 -11
- package/javascript/usearch.d.ts +87 -0
- package/javascript/usearch.js +2 -118
- package/objc/USearchObjective.mm +57 -31
- package/objc/include/USearchObjective.h +32 -22
- package/package.json +5 -3
- package/python/README.md +23 -14
- package/python/lib.cpp +468 -444
- package/python/scripts/bench.ipynb +3 -2
- package/python/scripts/bench.py +4 -4
- package/python/scripts/index_faiss.py +5 -5
- package/python/scripts/join.py +4 -4
- package/python/scripts/test.py +152 -61
- package/python/scripts/test_jit.py +10 -12
- package/python/scripts/test_sparse.py +32 -0
- package/python/usearch/client.py +26 -28
- package/python/usearch/eval.py +73 -23
- package/python/usearch/index.py +520 -228
- package/python/usearch/io.py +57 -25
- package/python/usearch/numba.py +4 -4
- package/python/usearch/server.py +10 -10
- package/rust/lib.cpp +22 -27
- package/rust/lib.hpp +11 -12
- package/rust/lib.rs +3 -3
- package/setup.py +5 -2
- package/simsimd/.github/workflows/prerelease.yml +73 -0
- package/simsimd/.github/workflows/release.yml +110 -0
- package/simsimd/.github/workflows/update_version.sh +3 -0
- package/simsimd/.releaserc +77 -0
- package/simsimd/.vscode/launch.json +27 -1
- package/simsimd/.vscode/settings.json +22 -1
- package/simsimd/.vscode/tasks.json +6 -0
- package/simsimd/CMakeLists.txt +3 -2
- package/simsimd/README.md +82 -46
- package/simsimd/bench.cxx +63 -24
- package/simsimd/include/simsimd/simsimd.h +88 -66
- package/simsimd/include/simsimd/simsimd_chem.h +126 -0
- package/simsimd/package.json +8 -0
- package/simsimd/pyproject.toml +42 -0
- package/simsimd/python/python.c +81 -0
- package/simsimd/python/run.sh +1 -0
- package/simsimd/python/test.c +109 -0
- package/simsimd/python/test.py +23 -0
- package/simsimd/setup.py +44 -0
- package/swift/Index+Sugar.swift +75 -23
- package/swift/README.md +26 -2
- package/swift/Test.swift +19 -7
- package/swift/USearch.swift +2 -2
- package/tsconfig.json +8 -0
- package/wasm/CMakeLists.txt +1 -0
- package/wasmer.toml +1 -1
- package/wolfram/CMakeLists.txt +1 -0
- package/wolfram/lib.cpp +11 -13
- package/golang/usearch.h +0 -9
- package/golang/usearch_par_test.go +0 -40
- package/golang/usearch_test.go +0 -82
- package/include/usearch/index_punned_dense.hpp +0 -1001
- package/include/usearch/index_punned_helpers.hpp +0 -676
|
@@ -0,0 +1,1263 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
#include <stdlib.h> // `aligned_alloc`
|
|
3
|
+
|
|
4
|
+
#include <cstring> // `std::strncmp`
|
|
5
|
+
#include <numeric> // `std::iota`
|
|
6
|
+
#include <thread> // `std::thread`
|
|
7
|
+
#include <vector> // `std::vector`
|
|
8
|
+
|
|
9
|
+
#include <usearch/index.hpp> // `expected_gt` and macros
|
|
10
|
+
|
|
11
|
+
#if USEARCH_USE_OPENMP
|
|
12
|
+
#include <omp.h> // `omp_get_num_threads()`
|
|
13
|
+
#endif
|
|
14
|
+
|
|
15
|
+
#if defined(USEARCH_DEFINED_LINUX)
|
|
16
|
+
#include <sys/auxv.h> // `getauxval()`
|
|
17
|
+
#endif
|
|
18
|
+
|
|
19
|
+
#if defined(USEARCH_DEFINED_ARM)
|
|
20
|
+
#include <arm_fp16.h> // `__fp16`
|
|
21
|
+
#endif
|
|
22
|
+
|
|
23
|
+
#if !defined(USEARCH_USE_NATIVE_F16)
|
|
24
|
+
#if defined(__AVX512F__)
|
|
25
|
+
#define USEARCH_USE_NATIVE_F16 1
|
|
26
|
+
#elif defined(USEARCH_DEFINED_ARM)
|
|
27
|
+
#define USEARCH_USE_NATIVE_F16 1
|
|
28
|
+
#else
|
|
29
|
+
#define USEARCH_USE_NATIVE_F16 0
|
|
30
|
+
#include <fp16/fp16.h>
|
|
31
|
+
#endif
|
|
32
|
+
#else
|
|
33
|
+
#define USEARCH_USE_NATIVE_F16 0
|
|
34
|
+
#include <fp16/fp16.h>
|
|
35
|
+
#endif
|
|
36
|
+
|
|
37
|
+
#if !defined(USEARCH_USE_SIMSIMD)
|
|
38
|
+
#define USEARCH_USE_SIMSIMD 0
|
|
39
|
+
#endif
|
|
40
|
+
|
|
41
|
+
#if USEARCH_USE_SIMSIMD
|
|
42
|
+
#include <simsimd/simsimd.h>
|
|
43
|
+
#endif
|
|
44
|
+
|
|
45
|
+
namespace unum {
|
|
46
|
+
namespace usearch {
|
|
47
|
+
|
|
48
|
+
using u40_t = uint40_t;
|
|
49
|
+
enum b1x8_t : unsigned char {};
|
|
50
|
+
|
|
51
|
+
struct uuid_t {
|
|
52
|
+
std::uint8_t octets[16];
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
class i8_bits_t;
|
|
56
|
+
class f16_bits_t;
|
|
57
|
+
|
|
58
|
+
#if USEARCH_USE_NATIVE_F16
|
|
59
|
+
#if defined(USEARCH_DEFINED_ARM)
|
|
60
|
+
using f16_native_t = __fp16;
|
|
61
|
+
#else
|
|
62
|
+
using f16_native_t = _Float16;
|
|
63
|
+
#endif
|
|
64
|
+
using f16_t = f16_native_t;
|
|
65
|
+
#else
|
|
66
|
+
using f16_native_t = void;
|
|
67
|
+
using f16_t = f16_bits_t;
|
|
68
|
+
#endif
|
|
69
|
+
|
|
70
|
+
using f64_t = double;
|
|
71
|
+
using f32_t = float;
|
|
72
|
+
|
|
73
|
+
using u64_t = std::uint64_t;
|
|
74
|
+
using u32_t = std::uint32_t;
|
|
75
|
+
using u16_t = std::uint16_t;
|
|
76
|
+
using u8_t = std::uint8_t;
|
|
77
|
+
|
|
78
|
+
using i64_t = std::int64_t;
|
|
79
|
+
using i32_t = std::int32_t;
|
|
80
|
+
using i16_t = std::int16_t;
|
|
81
|
+
using i8_t = std::int8_t;
|
|
82
|
+
|
|
83
|
+
enum class metric_kind_t : std::uint8_t {
|
|
84
|
+
unknown_k = 0,
|
|
85
|
+
// Classics:
|
|
86
|
+
ip_k = 'i',
|
|
87
|
+
cos_k = 'c',
|
|
88
|
+
l2sq_k = 'e',
|
|
89
|
+
|
|
90
|
+
// Custom:
|
|
91
|
+
pearson_k = 'p',
|
|
92
|
+
haversine_k = 'h',
|
|
93
|
+
|
|
94
|
+
// Sets:
|
|
95
|
+
jaccard_k = 'j',
|
|
96
|
+
hamming_k = 'b',
|
|
97
|
+
tanimoto_k = 't',
|
|
98
|
+
sorensen_k = 's',
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
enum class scalar_kind_t : std::uint8_t {
|
|
102
|
+
unknown_k = 0,
|
|
103
|
+
// Custom:
|
|
104
|
+
b1x8_k,
|
|
105
|
+
u40_k,
|
|
106
|
+
uuid_k,
|
|
107
|
+
// Common:
|
|
108
|
+
f64_k,
|
|
109
|
+
f32_k,
|
|
110
|
+
f16_k,
|
|
111
|
+
f8_k,
|
|
112
|
+
// Common Integral:
|
|
113
|
+
u64_k,
|
|
114
|
+
u32_k,
|
|
115
|
+
u16_k,
|
|
116
|
+
u8_k,
|
|
117
|
+
i64_k,
|
|
118
|
+
i32_k,
|
|
119
|
+
i16_k,
|
|
120
|
+
i8_k,
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
enum class isa_kind_t {
|
|
124
|
+
auto_k,
|
|
125
|
+
neon_k,
|
|
126
|
+
sve_k,
|
|
127
|
+
avx2_k,
|
|
128
|
+
avx512_k,
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
enum class prefetching_kind_t {
|
|
132
|
+
none_k,
|
|
133
|
+
cpu_k,
|
|
134
|
+
io_uring_k,
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
template <typename scalar_at> scalar_kind_t scalar_kind() noexcept {
|
|
138
|
+
if (std::is_same<scalar_at, b1x8_t>())
|
|
139
|
+
return scalar_kind_t::b1x8_k;
|
|
140
|
+
if (std::is_same<scalar_at, uint40_t>())
|
|
141
|
+
return scalar_kind_t::u40_k;
|
|
142
|
+
if (std::is_same<scalar_at, uuid_t>())
|
|
143
|
+
return scalar_kind_t::uuid_k;
|
|
144
|
+
if (std::is_same<scalar_at, f64_t>())
|
|
145
|
+
return scalar_kind_t::f64_k;
|
|
146
|
+
if (std::is_same<scalar_at, f32_t>())
|
|
147
|
+
return scalar_kind_t::f32_k;
|
|
148
|
+
if (std::is_same<scalar_at, f16_t>())
|
|
149
|
+
return scalar_kind_t::f16_k;
|
|
150
|
+
if (std::is_same<scalar_at, i8_bits_t>())
|
|
151
|
+
return scalar_kind_t::i8_k;
|
|
152
|
+
if (std::is_same<scalar_at, u64_t>())
|
|
153
|
+
return scalar_kind_t::u64_k;
|
|
154
|
+
if (std::is_same<scalar_at, u32_t>())
|
|
155
|
+
return scalar_kind_t::u32_k;
|
|
156
|
+
if (std::is_same<scalar_at, u16_t>())
|
|
157
|
+
return scalar_kind_t::u16_k;
|
|
158
|
+
if (std::is_same<scalar_at, u8_t>())
|
|
159
|
+
return scalar_kind_t::u8_k;
|
|
160
|
+
if (std::is_same<scalar_at, i64_t>())
|
|
161
|
+
return scalar_kind_t::i64_k;
|
|
162
|
+
if (std::is_same<scalar_at, i32_t>())
|
|
163
|
+
return scalar_kind_t::i32_k;
|
|
164
|
+
if (std::is_same<scalar_at, i16_t>())
|
|
165
|
+
return scalar_kind_t::i16_k;
|
|
166
|
+
if (std::is_same<scalar_at, i8_t>())
|
|
167
|
+
return scalar_kind_t::i8_k;
|
|
168
|
+
return scalar_kind_t::unknown_k;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
template <typename at> at angle_to_radians(at angle) noexcept { return angle * at(3.14159265358979323846) / at(180); }
|
|
172
|
+
|
|
173
|
+
template <typename at> at square(at value) noexcept { return value * value; }
|
|
174
|
+
|
|
175
|
+
template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp) noexcept {
|
|
176
|
+
return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
|
|
177
|
+
}
|
|
178
|
+
template <typename at> inline at clamp(at v, at lo, at hi) noexcept {
|
|
179
|
+
return usearch::clamp(v, lo, hi, std::less<at>{});
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
inline bool str_equals(char const* begin, std::size_t len, char const* other_begin) noexcept {
|
|
183
|
+
std::size_t other_len = std::strlen(other_begin);
|
|
184
|
+
return len == other_len && std::strncmp(begin, other_begin, len) == 0;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
inline char const* isa_name(isa_kind_t isa_kind) noexcept {
|
|
188
|
+
switch (isa_kind) {
|
|
189
|
+
case isa_kind_t::auto_k: return "auto";
|
|
190
|
+
case isa_kind_t::neon_k: return "neon";
|
|
191
|
+
case isa_kind_t::sve_k: return "sve";
|
|
192
|
+
case isa_kind_t::avx2_k: return "avx2";
|
|
193
|
+
case isa_kind_t::avx512_k: return "avx512";
|
|
194
|
+
default: return "";
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
inline bool hardware_supports(isa_kind_t isa_kind) noexcept {
|
|
199
|
+
#if defined(USEARCH_DEFINED_ARM) && defined(USEARCH_DEFINED_LINUX)
|
|
200
|
+
unsigned long capabilities = getauxval(AT_HWCAP);
|
|
201
|
+
switch (isa_kind) {
|
|
202
|
+
case isa_kind_t::neon_k: return true; // Must be supported on 64-bit Arm
|
|
203
|
+
case isa_kind_t::sve_k: return capabilities & HWCAP_SVE;
|
|
204
|
+
default: return false;
|
|
205
|
+
}
|
|
206
|
+
#endif
|
|
207
|
+
|
|
208
|
+
#if defined(USEARCH_DEFINED_X86) && defined(USEARCH_DEFINED_GCC)
|
|
209
|
+
__builtin_cpu_init();
|
|
210
|
+
switch (isa_kind) {
|
|
211
|
+
case isa_kind_t::avx2_k: return __builtin_cpu_supports("avx2");
|
|
212
|
+
case isa_kind_t::avx512_k: return __builtin_cpu_supports("avx512f");
|
|
213
|
+
default: return false;
|
|
214
|
+
}
|
|
215
|
+
#endif
|
|
216
|
+
|
|
217
|
+
(void)isa_kind;
|
|
218
|
+
return false;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
inline std::size_t bits_per_scalar(scalar_kind_t scalar_kind) noexcept {
|
|
222
|
+
switch (scalar_kind) {
|
|
223
|
+
case scalar_kind_t::f64_k: return 64;
|
|
224
|
+
case scalar_kind_t::f32_k: return 32;
|
|
225
|
+
case scalar_kind_t::f16_k: return 16;
|
|
226
|
+
case scalar_kind_t::i8_k: return 8;
|
|
227
|
+
case scalar_kind_t::b1x8_k: return 1;
|
|
228
|
+
default: return 0;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
inline std::size_t bits_per_scalar_word(scalar_kind_t scalar_kind) noexcept {
|
|
233
|
+
switch (scalar_kind) {
|
|
234
|
+
case scalar_kind_t::f64_k: return 64;
|
|
235
|
+
case scalar_kind_t::f32_k: return 32;
|
|
236
|
+
case scalar_kind_t::f16_k: return 16;
|
|
237
|
+
case scalar_kind_t::i8_k: return 8;
|
|
238
|
+
case scalar_kind_t::b1x8_k: return 8;
|
|
239
|
+
default: return 0;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
inline char const* scalar_kind_name(scalar_kind_t scalar_kind) noexcept {
|
|
244
|
+
switch (scalar_kind) {
|
|
245
|
+
case scalar_kind_t::f32_k: return "f32";
|
|
246
|
+
case scalar_kind_t::f16_k: return "f16";
|
|
247
|
+
case scalar_kind_t::f64_k: return "f64";
|
|
248
|
+
case scalar_kind_t::i8_k: return "i8";
|
|
249
|
+
case scalar_kind_t::b1x8_k: return "b1x8";
|
|
250
|
+
default: return "";
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
inline char const* metric_kind_name(metric_kind_t metric) noexcept {
|
|
255
|
+
switch (metric) {
|
|
256
|
+
case metric_kind_t::unknown_k: return "unknown";
|
|
257
|
+
case metric_kind_t::ip_k: return "ip";
|
|
258
|
+
case metric_kind_t::cos_k: return "cos";
|
|
259
|
+
case metric_kind_t::l2sq_k: return "l2sq";
|
|
260
|
+
case metric_kind_t::pearson_k: return "pearson";
|
|
261
|
+
case metric_kind_t::haversine_k: return "haversine";
|
|
262
|
+
case metric_kind_t::jaccard_k: return "jaccard";
|
|
263
|
+
case metric_kind_t::hamming_k: return "hamming";
|
|
264
|
+
case metric_kind_t::tanimoto_k: return "tanimoto";
|
|
265
|
+
case metric_kind_t::sorensen_k: return "sorensen";
|
|
266
|
+
}
|
|
267
|
+
return "";
|
|
268
|
+
}
|
|
269
|
+
inline expected_gt<scalar_kind_t> scalar_kind_from_name(char const* name, std::size_t len) {
|
|
270
|
+
expected_gt<scalar_kind_t> parsed;
|
|
271
|
+
if (str_equals(name, len, "f32"))
|
|
272
|
+
parsed.result = scalar_kind_t::f32_k;
|
|
273
|
+
else if (str_equals(name, len, "f64"))
|
|
274
|
+
parsed.result = scalar_kind_t::f64_k;
|
|
275
|
+
else if (str_equals(name, len, "f16"))
|
|
276
|
+
parsed.result = scalar_kind_t::f16_k;
|
|
277
|
+
else if (str_equals(name, len, "i8"))
|
|
278
|
+
parsed.result = scalar_kind_t::i8_k;
|
|
279
|
+
else
|
|
280
|
+
parsed.failed("Unknown type, choose: f32, f16, f64, i8");
|
|
281
|
+
return parsed;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
inline expected_gt<metric_kind_t> metric_from_name(char const* name, std::size_t len) {
|
|
285
|
+
expected_gt<metric_kind_t> parsed;
|
|
286
|
+
if (str_equals(name, len, "l2sq") || str_equals(name, len, "euclidean_sq")) {
|
|
287
|
+
parsed.result = metric_kind_t::l2sq_k;
|
|
288
|
+
} else if (str_equals(name, len, "ip") || str_equals(name, len, "inner") || str_equals(name, len, "dot")) {
|
|
289
|
+
parsed.result = metric_kind_t::ip_k;
|
|
290
|
+
} else if (str_equals(name, len, "cos") || str_equals(name, len, "angular")) {
|
|
291
|
+
parsed.result = metric_kind_t::cos_k;
|
|
292
|
+
} else if (str_equals(name, len, "haversine")) {
|
|
293
|
+
parsed.result = metric_kind_t::haversine_k;
|
|
294
|
+
} else if (str_equals(name, len, "pearson")) {
|
|
295
|
+
parsed.result = metric_kind_t::pearson_k;
|
|
296
|
+
} else if (str_equals(name, len, "hamming")) {
|
|
297
|
+
parsed.result = metric_kind_t::hamming_k;
|
|
298
|
+
} else if (str_equals(name, len, "tanimoto")) {
|
|
299
|
+
parsed.result = metric_kind_t::tanimoto_k;
|
|
300
|
+
} else if (str_equals(name, len, "sorensen")) {
|
|
301
|
+
parsed.result = metric_kind_t::sorensen_k;
|
|
302
|
+
} else
|
|
303
|
+
parsed.failed(
|
|
304
|
+
"Unknown distance, choose: l2sq, ip, cos, haversine, jaccard, pearson, hamming, tanimoto, sorensen");
|
|
305
|
+
return parsed;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
inline float f16_to_f32(std::uint16_t u16) noexcept {
|
|
309
|
+
#if USEARCH_USE_NATIVE_F16
|
|
310
|
+
f16_native_t f16;
|
|
311
|
+
std::memcpy(&f16, &u16, sizeof(std::uint16_t));
|
|
312
|
+
return float(f16);
|
|
313
|
+
#else
|
|
314
|
+
return fp16_ieee_to_fp32_value(u16);
|
|
315
|
+
#endif
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
inline std::uint16_t f32_to_f16(float f32) noexcept {
|
|
319
|
+
#if USEARCH_USE_NATIVE_F16
|
|
320
|
+
f16_native_t f16 = f16_native_t(f32);
|
|
321
|
+
std::uint16_t u16;
|
|
322
|
+
std::memcpy(&u16, &f16, sizeof(std::uint16_t));
|
|
323
|
+
return u16;
|
|
324
|
+
#else
|
|
325
|
+
return fp16_ieee_from_fp32_value(f32);
|
|
326
|
+
#endif
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* @brief Numeric type for the IEEE 754 half-precision floating point.
|
|
331
|
+
* If hardware support isn't available, falls back to a hardware
|
|
332
|
+
* agnostic in-software implementation.
|
|
333
|
+
*/
|
|
334
|
+
class f16_bits_t {
|
|
335
|
+
std::uint16_t uint16_{};
|
|
336
|
+
|
|
337
|
+
public:
|
|
338
|
+
inline f16_bits_t() noexcept : uint16_(0) {}
|
|
339
|
+
inline f16_bits_t(f16_bits_t&&) = default;
|
|
340
|
+
inline f16_bits_t& operator=(f16_bits_t&&) = default;
|
|
341
|
+
inline f16_bits_t(f16_bits_t const&) = default;
|
|
342
|
+
inline f16_bits_t& operator=(f16_bits_t const&) = default;
|
|
343
|
+
|
|
344
|
+
inline operator float() const noexcept { return f16_to_f32(uint16_); }
|
|
345
|
+
inline explicit operator bool() const noexcept { return f16_to_f32(uint16_) > 0.5f; }
|
|
346
|
+
|
|
347
|
+
inline f16_bits_t(i8_bits_t) noexcept;
|
|
348
|
+
inline f16_bits_t(bool v) noexcept : uint16_(f32_to_f16(v)) {}
|
|
349
|
+
inline f16_bits_t(float v) noexcept : uint16_(f32_to_f16(v)) {}
|
|
350
|
+
inline f16_bits_t(double v) noexcept : uint16_(f32_to_f16(v)) {}
|
|
351
|
+
|
|
352
|
+
inline f16_bits_t operator+(f16_bits_t other) const noexcept { return {float(*this) + float(other)}; }
|
|
353
|
+
inline f16_bits_t operator-(f16_bits_t other) const noexcept { return {float(*this) - float(other)}; }
|
|
354
|
+
inline f16_bits_t operator*(f16_bits_t other) const noexcept { return {float(*this) * float(other)}; }
|
|
355
|
+
inline f16_bits_t operator/(f16_bits_t other) const noexcept { return {float(*this) / float(other)}; }
|
|
356
|
+
inline f16_bits_t operator+(float other) const noexcept { return {float(*this) + other}; }
|
|
357
|
+
inline f16_bits_t operator-(float other) const noexcept { return {float(*this) - other}; }
|
|
358
|
+
inline f16_bits_t operator*(float other) const noexcept { return {float(*this) * other}; }
|
|
359
|
+
inline f16_bits_t operator/(float other) const noexcept { return {float(*this) / other}; }
|
|
360
|
+
inline f16_bits_t operator+(double other) const noexcept { return {float(*this) + other}; }
|
|
361
|
+
inline f16_bits_t operator-(double other) const noexcept { return {float(*this) - other}; }
|
|
362
|
+
inline f16_bits_t operator*(double other) const noexcept { return {float(*this) * other}; }
|
|
363
|
+
inline f16_bits_t operator/(double other) const noexcept { return {float(*this) / other}; }
|
|
364
|
+
|
|
365
|
+
inline f16_bits_t& operator+=(float v) noexcept {
|
|
366
|
+
uint16_ = f32_to_f16(v + f16_to_f32(uint16_));
|
|
367
|
+
return *this;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
inline f16_bits_t& operator-=(float v) noexcept {
|
|
371
|
+
uint16_ = f32_to_f16(v - f16_to_f32(uint16_));
|
|
372
|
+
return *this;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
inline f16_bits_t& operator*=(float v) noexcept {
|
|
376
|
+
uint16_ = f32_to_f16(v * f16_to_f32(uint16_));
|
|
377
|
+
return *this;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
inline f16_bits_t& operator/=(float v) noexcept {
|
|
381
|
+
uint16_ = f32_to_f16(v / f16_to_f32(uint16_));
|
|
382
|
+
return *this;
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* @brief Numeric type for uniformly-distributed floating point
|
|
388
|
+
* values within [-1,1] range, quantized to integers [-100,100].
|
|
389
|
+
*/
|
|
390
|
+
class i8_bits_t {
|
|
391
|
+
std::int8_t int8_{};
|
|
392
|
+
|
|
393
|
+
public:
|
|
394
|
+
constexpr static float divisor_k = 100.f;
|
|
395
|
+
constexpr static std::int8_t min_k = -100;
|
|
396
|
+
constexpr static std::int8_t max_k = 100;
|
|
397
|
+
|
|
398
|
+
inline i8_bits_t() noexcept : int8_(0) {}
|
|
399
|
+
inline i8_bits_t(bool v) noexcept : int8_(v ? max_k : 0) {}
|
|
400
|
+
|
|
401
|
+
inline i8_bits_t(i8_bits_t&&) = default;
|
|
402
|
+
inline i8_bits_t& operator=(i8_bits_t&&) = default;
|
|
403
|
+
inline i8_bits_t(i8_bits_t const&) = default;
|
|
404
|
+
inline i8_bits_t& operator=(i8_bits_t const&) = default;
|
|
405
|
+
|
|
406
|
+
inline operator float() const noexcept { return float(int8_) / divisor_k; }
|
|
407
|
+
inline operator f16_t() const noexcept { return float(int8_) / divisor_k; }
|
|
408
|
+
inline operator double() const noexcept { return double(int8_) / divisor_k; }
|
|
409
|
+
inline explicit operator bool() const noexcept { return int8_ > (max_k / 2); }
|
|
410
|
+
inline explicit operator std::int8_t() const noexcept { return int8_; }
|
|
411
|
+
inline explicit operator std::int16_t() const noexcept { return int8_; }
|
|
412
|
+
inline explicit operator std::int32_t() const noexcept { return int8_; }
|
|
413
|
+
inline explicit operator std::int64_t() const noexcept { return int8_; }
|
|
414
|
+
|
|
415
|
+
inline i8_bits_t(f16_t v)
|
|
416
|
+
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
417
|
+
inline i8_bits_t(float v)
|
|
418
|
+
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
419
|
+
inline i8_bits_t(double v)
|
|
420
|
+
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
421
|
+
};
|
|
422
|
+
|
|
423
|
+
inline f16_bits_t::f16_bits_t(i8_bits_t v) noexcept : f16_bits_t(float(v)) {}
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* @brief An STL-based executor or a "thread-pool" for parallel execution.
|
|
427
|
+
* Isn't efficient for small batches, as it recreates the threads on every call.
|
|
428
|
+
*/
|
|
429
|
+
class executor_stl_t {
|
|
430
|
+
std::size_t threads_count_{};
|
|
431
|
+
|
|
432
|
+
public:
|
|
433
|
+
/**
|
|
434
|
+
* @param threads_count The number of threads to be used for parallel execution.
|
|
435
|
+
*/
|
|
436
|
+
executor_stl_t(std::size_t threads_count = 0) noexcept
|
|
437
|
+
: threads_count_(threads_count ? threads_count : std::thread::hardware_concurrency()) {}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* @return Maximum number of threads available to the executor.
|
|
441
|
+
*/
|
|
442
|
+
std::size_t size() const noexcept { return threads_count_; }
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* @brief Executes tasks in bulk using the specified thread-aware function.
|
|
446
|
+
* @param tasks The total number of tasks to be executed.
|
|
447
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
448
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
449
|
+
*/
|
|
450
|
+
template <typename thread_aware_function_at>
|
|
451
|
+
void execute_bulk(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
452
|
+
std::vector<std::thread> threads_pool;
|
|
453
|
+
std::size_t tasks_per_thread = tasks;
|
|
454
|
+
if (threads_count_ > 1) {
|
|
455
|
+
tasks_per_thread = (tasks / threads_count_) + ((tasks % threads_count_) != 0);
|
|
456
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count_; ++thread_idx) {
|
|
457
|
+
threads_pool.emplace_back([=]() {
|
|
458
|
+
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
459
|
+
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread); ++task_idx)
|
|
460
|
+
thread_aware_function(thread_idx, task_idx);
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
for (std::size_t task_idx = 0; task_idx < (std::min)(tasks, tasks_per_thread); ++task_idx)
|
|
465
|
+
thread_aware_function(0, task_idx);
|
|
466
|
+
for (std::thread& thread : threads_pool)
|
|
467
|
+
thread.join();
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* @brief Saturates every available thread with the given workload, until they finish.
|
|
472
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index.
|
|
473
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
474
|
+
*/
|
|
475
|
+
template <typename thread_aware_function_at>
|
|
476
|
+
void execute_bulk(thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
477
|
+
if (threads_count_ == 1)
|
|
478
|
+
return thread_aware_function(0);
|
|
479
|
+
std::vector<std::thread> threads_pool;
|
|
480
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count_; ++thread_idx)
|
|
481
|
+
threads_pool.emplace_back([=]() { thread_aware_function(thread_idx); });
|
|
482
|
+
thread_aware_function(0);
|
|
483
|
+
for (std::thread& thread : threads_pool)
|
|
484
|
+
thread.join();
|
|
485
|
+
}
|
|
486
|
+
};
|
|
487
|
+
|
|
488
|
+
#if USEARCH_USE_OPENMP
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* @brief An OpenMP-based executor or a "thread-pool" for parallel execution.
|
|
492
|
+
* Is the preferred implementation, when available, and maximum performance is needed.
|
|
493
|
+
*/
|
|
494
|
+
class executor_openmp_t {
|
|
495
|
+
public:
|
|
496
|
+
/**
|
|
497
|
+
* @param threads_count The number of threads to be used for parallel execution.
|
|
498
|
+
*/
|
|
499
|
+
executor_openmp_t(std::size_t threads_count = 0) noexcept {
|
|
500
|
+
omp_set_num_threads(threads_count ? threads_count : std::thread::hardware_concurrency());
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* @return Maximum number of threads available to the executor.
|
|
505
|
+
*/
|
|
506
|
+
std::size_t size() const noexcept { return omp_get_num_threads(); }
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* @brief Executes tasks in bulk using the specified thread-aware function.
|
|
510
|
+
* @param tasks The total number of tasks to be executed.
|
|
511
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
512
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
513
|
+
*/
|
|
514
|
+
template <typename thread_aware_function_at>
|
|
515
|
+
void execute_bulk(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
516
|
+
#pragma omp parallel for schedule(dynamic)
|
|
517
|
+
for (std::size_t i = 0; i != tasks; ++i) {
|
|
518
|
+
thread_aware_function(omp_get_thread_num(), i);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* @brief Saturates every available thread with the given workload, until they finish.
|
|
524
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index.
|
|
525
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
526
|
+
*/
|
|
527
|
+
template <typename thread_aware_function_at>
|
|
528
|
+
void execute_bulk(thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
529
|
+
#pragma omp parallel
|
|
530
|
+
{ thread_aware_function(omp_get_thread_num()); }
|
|
531
|
+
}
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
using executor_default_t = executor_openmp_t;
|
|
535
|
+
|
|
536
|
+
#else
|
|
537
|
+
|
|
538
|
+
using executor_default_t = executor_stl_t;
|
|
539
|
+
|
|
540
|
+
#endif
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* @brief Uses OS-specific APIs for aligned memory allocations.
|
|
544
|
+
*/
|
|
545
|
+
template <typename element_at = char, std::size_t alignment_ak = 64> //
|
|
546
|
+
class aligned_allocator_gt {
|
|
547
|
+
public:
|
|
548
|
+
using value_type = element_at;
|
|
549
|
+
using size_type = std::size_t;
|
|
550
|
+
using pointer = element_at*;
|
|
551
|
+
using const_pointer = element_at const*;
|
|
552
|
+
template <typename other_element_at> struct rebind {
|
|
553
|
+
using other = aligned_allocator_gt<other_element_at>;
|
|
554
|
+
};
|
|
555
|
+
|
|
556
|
+
constexpr std::size_t alignment() const { return alignment_ak; }
|
|
557
|
+
|
|
558
|
+
pointer allocate(size_type length) const {
|
|
559
|
+
std::size_t length_bytes = alignment_ak * divide_round_up<alignment_ak>(length * sizeof(value_type));
|
|
560
|
+
std::size_t alignment = alignment_ak;
|
|
561
|
+
// void* result = nullptr;
|
|
562
|
+
// int status = posix_memalign(&result, alignment, length_bytes);
|
|
563
|
+
// return status == 0 ? (pointer)result : nullptr;
|
|
564
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
565
|
+
return (pointer)_aligned_malloc(length_bytes, alignment);
|
|
566
|
+
#else
|
|
567
|
+
return (pointer)aligned_alloc(alignment, length_bytes);
|
|
568
|
+
#endif
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
void deallocate(pointer begin, size_type) const {
|
|
572
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
573
|
+
_aligned_free(begin);
|
|
574
|
+
#else
|
|
575
|
+
free(begin);
|
|
576
|
+
#endif
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
|
|
580
|
+
using aligned_allocator_t = aligned_allocator_gt<>;
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* @brief Memory-mapping allocator designed for "alloc many, free at once" usage patterns.
|
|
584
|
+
* @b Thread-safe, @b except constructors and destructors.
|
|
585
|
+
*
|
|
586
|
+
* Using this memory allocator won't affect your overall speed much, as that is not the bottleneck.
|
|
587
|
+
* However, it can drastically improve memory usage especcially for huge indexes of small vectors.
|
|
588
|
+
*/
|
|
589
|
+
template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
590
|
+
|
|
591
|
+
static constexpr std::size_t min_capacity() { return 1024 * 1024 * 4; }
|
|
592
|
+
static constexpr std::size_t capacity_multiplier() { return 2; }
|
|
593
|
+
static constexpr std::size_t head_size() {
|
|
594
|
+
/// Pointer to the the previous arena and the size of the current one.
|
|
595
|
+
return divide_round_up<alignment_ak>(sizeof(byte_t*) + sizeof(std::size_t)) * alignment_ak;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
std::mutex mutex_;
|
|
599
|
+
byte_t* last_arena_ = nullptr;
|
|
600
|
+
std::size_t last_usage_ = head_size();
|
|
601
|
+
std::size_t last_capacity_ = min_capacity();
|
|
602
|
+
std::size_t wasted_space_ = 0;
|
|
603
|
+
|
|
604
|
+
public:
|
|
605
|
+
using value_type = byte_t;
|
|
606
|
+
using size_type = std::size_t;
|
|
607
|
+
using pointer = byte_t*;
|
|
608
|
+
using const_pointer = byte_t const*;
|
|
609
|
+
|
|
610
|
+
memory_mapping_allocator_gt() = default;
|
|
611
|
+
memory_mapping_allocator_gt(memory_mapping_allocator_gt&& other) noexcept
|
|
612
|
+
: last_arena_(exchange(other.last_arena_, nullptr)), last_usage_(exchange(other.last_usage_, 0)),
|
|
613
|
+
last_capacity_(exchange(other.last_capacity_, 0)), wasted_space_(exchange(other.wasted_space_, 0)) {}
|
|
614
|
+
|
|
615
|
+
memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt&& other) noexcept {
|
|
616
|
+
std::swap(last_arena_, other.last_arena_);
|
|
617
|
+
std::swap(last_usage_, other.last_usage_);
|
|
618
|
+
std::swap(last_capacity_, other.last_capacity_);
|
|
619
|
+
std::swap(wasted_space_, other.wasted_space_);
|
|
620
|
+
return *this;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
~memory_mapping_allocator_gt() noexcept { reset(); }
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* @brief Discards all previously allocated memory buffers.
|
|
627
|
+
*/
|
|
628
|
+
void reset() noexcept {
|
|
629
|
+
byte_t* last_arena = last_arena_;
|
|
630
|
+
while (last_arena) {
|
|
631
|
+
byte_t* previous_arena;
|
|
632
|
+
std::memcpy(&previous_arena, last_arena, sizeof(byte_t*));
|
|
633
|
+
std::size_t last_cap;
|
|
634
|
+
std::memcpy(&last_cap, last_arena + sizeof(byte_t*), sizeof(std::size_t));
|
|
635
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
636
|
+
::VirtualFree(last_arena, 0, MEM_RELEASE);
|
|
637
|
+
#else
|
|
638
|
+
munmap(last_arena, last_cap);
|
|
639
|
+
#endif
|
|
640
|
+
last_arena = previous_arena;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// Clear the references:
|
|
644
|
+
last_arena_ = nullptr;
|
|
645
|
+
last_usage_ = head_size();
|
|
646
|
+
last_capacity_ = min_capacity();
|
|
647
|
+
wasted_space_ = 0;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* @brief Copy constructor.
|
|
652
|
+
* @note This is a no-op copy constructor since the allocator is not copyable.
|
|
653
|
+
*/
|
|
654
|
+
memory_mapping_allocator_gt(memory_mapping_allocator_gt const&) noexcept {}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* @brief Copy assignment operator.
|
|
658
|
+
* @note This is a no-op copy assignment operator since the allocator is not copyable.
|
|
659
|
+
* @return Reference to the allocator after the assignment.
|
|
660
|
+
*/
|
|
661
|
+
memory_mapping_allocator_gt& operator=(memory_mapping_allocator_gt const&) noexcept {
|
|
662
|
+
reset();
|
|
663
|
+
return *this;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/**
|
|
667
|
+
* @brief Allocates an @b uninitialized block of memory of the specified size.
|
|
668
|
+
* @param count_bytes The number of bytes to allocate.
|
|
669
|
+
* @return A pointer to the allocated memory block, or `nullptr` if allocation fails.
|
|
670
|
+
*/
|
|
671
|
+
inline byte_t* allocate(std::size_t count_bytes) noexcept {
|
|
672
|
+
std::size_t extended_bytes = divide_round_up<alignment_ak>(count_bytes) * alignment_ak;
|
|
673
|
+
// Check if the requested allocation size is absurd
|
|
674
|
+
if (extended_bytes > min_capacity())
|
|
675
|
+
return nullptr;
|
|
676
|
+
|
|
677
|
+
std::unique_lock<std::mutex> lock(mutex_);
|
|
678
|
+
if (!last_arena_ || (last_usage_ + extended_bytes > last_capacity_)) {
|
|
679
|
+
std::size_t new_cap = last_capacity_ * capacity_multiplier();
|
|
680
|
+
#if defined(USEARCH_DEFINED_WINDOWS)
|
|
681
|
+
byte_t* new_arena = (byte_t*)(::VirtualAlloc(NULL, new_cap, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE));
|
|
682
|
+
if (new_arena == nullptr)
|
|
683
|
+
return nullptr;
|
|
684
|
+
#else
|
|
685
|
+
byte_t* new_arena = (byte_t*)mmap(NULL, new_cap, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
|
|
686
|
+
if (!new_arena)
|
|
687
|
+
return nullptr;
|
|
688
|
+
#endif
|
|
689
|
+
std::memcpy(new_arena, &last_arena_, sizeof(byte_t*));
|
|
690
|
+
std::memcpy(new_arena + sizeof(byte_t*), &new_cap, sizeof(std::size_t));
|
|
691
|
+
|
|
692
|
+
wasted_space_ += total_reserved();
|
|
693
|
+
last_arena_ = new_arena;
|
|
694
|
+
last_capacity_ = new_cap;
|
|
695
|
+
last_usage_ = head_size();
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
wasted_space_ += extended_bytes - count_bytes;
|
|
699
|
+
return last_arena_ + exchange(last_usage_, last_usage_ + extended_bytes);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* @brief Returns the amount of memory used by the allocator across all arenas.
|
|
704
|
+
* @return The amount of space in bytes.
|
|
705
|
+
*/
|
|
706
|
+
std::size_t total_allocated() const noexcept {
|
|
707
|
+
if (!last_arena_)
|
|
708
|
+
return 0;
|
|
709
|
+
std::size_t total_used = 0;
|
|
710
|
+
std::size_t last_capacity = last_capacity_;
|
|
711
|
+
do {
|
|
712
|
+
total_used += last_capacity;
|
|
713
|
+
last_capacity /= capacity_multiplier();
|
|
714
|
+
} while (last_capacity >= min_capacity());
|
|
715
|
+
return total_used;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* @brief Returns the amount of wasted space due to alignment.
|
|
720
|
+
* @return The amount of wasted space in bytes.
|
|
721
|
+
*/
|
|
722
|
+
std::size_t total_wasted() const noexcept { return wasted_space_; }
|
|
723
|
+
|
|
724
|
+
/**
|
|
725
|
+
* @brief Returns the amount of remaining memory already reserved but not yet used.
|
|
726
|
+
* @return The amount of reserved memory in bytes.
|
|
727
|
+
*/
|
|
728
|
+
std::size_t total_reserved() const noexcept { return last_arena_ ? last_capacity_ - last_usage_ : 0; }
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* @warning The very first memory de-allocation discards all the arenas!
|
|
732
|
+
*/
|
|
733
|
+
void deallocate(byte_t* = nullptr, std::size_t = 0) noexcept { reset(); }
|
|
734
|
+
};
|
|
735
|
+
|
|
736
|
+
using memory_mapping_allocator_t = memory_mapping_allocator_gt<>;
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* @brief Utility class used to cast arrays of one scalar type to another,
|
|
740
|
+
* avoiding unnecessary conversions.
|
|
741
|
+
*/
|
|
742
|
+
template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
|
|
743
|
+
inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
|
|
744
|
+
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
745
|
+
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
746
|
+
auto converter = [](from_scalar_at from) { return to_scalar_at(from); };
|
|
747
|
+
std::transform(typed_input, typed_input + dim, typed_output, converter);
|
|
748
|
+
return true;
|
|
749
|
+
}
|
|
750
|
+
};
|
|
751
|
+
|
|
752
|
+
template <> struct cast_gt<f32_t, f32_t> {
|
|
753
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
754
|
+
};
|
|
755
|
+
|
|
756
|
+
template <> struct cast_gt<f64_t, f64_t> {
|
|
757
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
758
|
+
};
|
|
759
|
+
|
|
760
|
+
template <> struct cast_gt<f16_bits_t, f16_bits_t> {
|
|
761
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
762
|
+
};
|
|
763
|
+
|
|
764
|
+
template <> struct cast_gt<i8_bits_t, i8_bits_t> {
|
|
765
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
766
|
+
};
|
|
767
|
+
|
|
768
|
+
template <> struct cast_gt<b1x8_t, b1x8_t> {
|
|
769
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
770
|
+
};
|
|
771
|
+
|
|
772
|
+
template <typename from_scalar_at> struct cast_gt<from_scalar_at, b1x8_t> {
|
|
773
|
+
inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
|
|
774
|
+
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
775
|
+
unsigned char* typed_output = reinterpret_cast<unsigned char*>(output);
|
|
776
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
777
|
+
typed_output[i / CHAR_BIT] |= bool(typed_input[i]) ? (128 >> (i & (CHAR_BIT - 1))) : 0;
|
|
778
|
+
return true;
|
|
779
|
+
}
|
|
780
|
+
};
|
|
781
|
+
|
|
782
|
+
template <typename to_scalar_at> struct cast_gt<b1x8_t, to_scalar_at> {
|
|
783
|
+
inline bool operator()(byte_t const* input, std::size_t dim, byte_t* output) const {
|
|
784
|
+
unsigned char const* typed_input = reinterpret_cast<unsigned char const*>(input);
|
|
785
|
+
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
786
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
787
|
+
typed_output[i] = bool(typed_input[i / CHAR_BIT] & (128 >> (i & (CHAR_BIT - 1))));
|
|
788
|
+
return true;
|
|
789
|
+
}
|
|
790
|
+
};
|
|
791
|
+
|
|
792
|
+
/**
|
|
793
|
+
* @brief Inner (Dot) Product distance.
|
|
794
|
+
*/
|
|
795
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_ip_gt {
|
|
796
|
+
using scalar_t = scalar_at;
|
|
797
|
+
using result_t = result_at;
|
|
798
|
+
|
|
799
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
800
|
+
result_t ab{};
|
|
801
|
+
#if USEARCH_USE_OPENMP
|
|
802
|
+
#pragma omp simd reduction(+ : ab)
|
|
803
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
804
|
+
#pragma clang loop vectorize(enable)
|
|
805
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
806
|
+
#pragma GCC ivdep
|
|
807
|
+
#endif
|
|
808
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
809
|
+
ab += result_t(a[i]) * result_t(b[i]);
|
|
810
|
+
return 1 - ab;
|
|
811
|
+
}
|
|
812
|
+
};
|
|
813
|
+
|
|
814
|
+
/**
|
|
815
|
+
* @brief Cosine (Angular) distance.
|
|
816
|
+
* Identical to the Inner Product of normalized vectors.
|
|
817
|
+
* Unless you are running on an tiny embedded platform, this metric
|
|
818
|
+
* is recommended over `::metric_ip_gt` for low-precision scalars.
|
|
819
|
+
*/
|
|
820
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_cos_gt {
|
|
821
|
+
using scalar_t = scalar_at;
|
|
822
|
+
using result_t = result_at;
|
|
823
|
+
|
|
824
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
825
|
+
result_t ab{}, a2{}, b2{};
|
|
826
|
+
#if USEARCH_USE_OPENMP
|
|
827
|
+
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
828
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
829
|
+
#pragma clang loop vectorize(enable)
|
|
830
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
831
|
+
#pragma GCC ivdep
|
|
832
|
+
#endif
|
|
833
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
834
|
+
ab += result_t(a[i]) * result_t(b[i]), //
|
|
835
|
+
a2 += square<result_t>(a[i]), //
|
|
836
|
+
b2 += square<result_t>(b[i]);
|
|
837
|
+
|
|
838
|
+
result_t result_if_zero[2][2];
|
|
839
|
+
result_if_zero[0][0] = 1 - ab / (std::sqrt(a2) * std::sqrt(b2));
|
|
840
|
+
result_if_zero[0][1] = result_if_zero[1][0] = 1;
|
|
841
|
+
result_if_zero[1][1] = 0;
|
|
842
|
+
return result_if_zero[a2 == 0][b2 == 0];
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
|
|
846
|
+
/**
|
|
847
|
+
* @brief Squared Euclidean (L2) distance.
|
|
848
|
+
* Square root is avoided at the end, as it won't affect the ordering.
|
|
849
|
+
*/
|
|
850
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_l2sq_gt {
|
|
851
|
+
using scalar_t = scalar_at;
|
|
852
|
+
using result_t = result_at;
|
|
853
|
+
|
|
854
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
855
|
+
result_t ab_deltas_sq{};
|
|
856
|
+
#if USEARCH_USE_OPENMP
|
|
857
|
+
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
858
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
859
|
+
#pragma clang loop vectorize(enable)
|
|
860
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
861
|
+
#pragma GCC ivdep
|
|
862
|
+
#endif
|
|
863
|
+
for (std::size_t i = 0; i != dim; ++i)
|
|
864
|
+
ab_deltas_sq += square(result_t(a[i]) - result_t(b[i]));
|
|
865
|
+
return ab_deltas_sq;
|
|
866
|
+
}
|
|
867
|
+
};
|
|
868
|
+
|
|
869
|
+
/**
|
|
870
|
+
* @brief Hamming distance computes the number of differing bits in
|
|
871
|
+
* two arrays of integers. An example would be a textual document,
|
|
872
|
+
* tokenized and hashed into a fixed-capacity bitset.
|
|
873
|
+
*/
|
|
874
|
+
template <typename scalar_at = std::uint64_t, typename result_at = std::size_t> struct metric_hamming_gt {
|
|
875
|
+
using scalar_t = scalar_at;
|
|
876
|
+
using result_t = result_at;
|
|
877
|
+
static_assert( //
|
|
878
|
+
std::is_unsigned<scalar_t>::value ||
|
|
879
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
880
|
+
"Hamming distance requires unsigned integral words");
|
|
881
|
+
|
|
882
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
883
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
884
|
+
result_t matches{};
|
|
885
|
+
#if USEARCH_USE_OPENMP
|
|
886
|
+
#pragma omp simd reduction(+ : matches)
|
|
887
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
888
|
+
#pragma clang loop vectorize(enable)
|
|
889
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
890
|
+
#pragma GCC ivdep
|
|
891
|
+
#endif
|
|
892
|
+
for (std::size_t i = 0; i != words; ++i)
|
|
893
|
+
matches += std::bitset<bits_per_word_k>(a[i] ^ b[i]).count();
|
|
894
|
+
return matches;
|
|
895
|
+
}
|
|
896
|
+
};
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* @brief Tanimoto distance is the intersection over bitwise union.
|
|
900
|
+
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
901
|
+
*/
|
|
902
|
+
template <typename scalar_at = std::uint64_t, typename result_at = float> struct metric_tanimoto_gt {
|
|
903
|
+
using scalar_t = scalar_at;
|
|
904
|
+
using result_t = result_at;
|
|
905
|
+
static_assert( //
|
|
906
|
+
std::is_unsigned<scalar_t>::value ||
|
|
907
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
908
|
+
"Tanimoto distance requires unsigned integral words");
|
|
909
|
+
static_assert(std::is_floating_point<result_t>::value, "Tanimoto distance will be a fraction");
|
|
910
|
+
|
|
911
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
912
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
913
|
+
result_t and_count{};
|
|
914
|
+
result_t or_count{};
|
|
915
|
+
#if USEARCH_USE_OPENMP
|
|
916
|
+
#pragma omp simd reduction(+ : and_count, or_count)
|
|
917
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
918
|
+
#pragma clang loop vectorize(enable)
|
|
919
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
920
|
+
#pragma GCC ivdep
|
|
921
|
+
#endif
|
|
922
|
+
for (std::size_t i = 0; i != words; ++i)
|
|
923
|
+
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
924
|
+
or_count += std::bitset<bits_per_word_k>(a[i] | b[i]).count();
|
|
925
|
+
return 1 - result_t(and_count) / or_count;
|
|
926
|
+
}
|
|
927
|
+
};
|
|
928
|
+
|
|
929
|
+
/**
|
|
930
|
+
* @brief Sorensen-Dice or F1 distance is the intersection over bitwise union.
|
|
931
|
+
* Often used in chemistry and biology to compare molecular fingerprints.
|
|
932
|
+
*/
|
|
933
|
+
template <typename scalar_at = std::uint64_t, typename result_at = float> struct metric_sorensen_gt {
|
|
934
|
+
using scalar_t = scalar_at;
|
|
935
|
+
using result_t = result_at;
|
|
936
|
+
static_assert( //
|
|
937
|
+
std::is_unsigned<scalar_t>::value ||
|
|
938
|
+
(std::is_enum<scalar_t>::value && std::is_unsigned<typename std::underlying_type<scalar_t>::type>::value),
|
|
939
|
+
"Sorensen-Dice distance requires unsigned integral words");
|
|
940
|
+
static_assert(std::is_floating_point<result_t>::value, "Sorensen-Dice distance will be a fraction");
|
|
941
|
+
|
|
942
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t words) const noexcept {
|
|
943
|
+
constexpr std::size_t bits_per_word_k = sizeof(scalar_t) * CHAR_BIT;
|
|
944
|
+
result_t and_count{};
|
|
945
|
+
result_t any_count{};
|
|
946
|
+
#if USEARCH_USE_OPENMP
|
|
947
|
+
#pragma omp simd reduction(+ : and_count, any_count)
|
|
948
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
949
|
+
#pragma clang loop vectorize(enable)
|
|
950
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
951
|
+
#pragma GCC ivdep
|
|
952
|
+
#endif
|
|
953
|
+
for (std::size_t i = 0; i != words; ++i)
|
|
954
|
+
and_count += std::bitset<bits_per_word_k>(a[i] & b[i]).count(),
|
|
955
|
+
any_count += std::bitset<bits_per_word_k>(a[i]).count() + std::bitset<bits_per_word_k>(b[i]).count();
|
|
956
|
+
return 1 - 2 * result_t(and_count) / any_count;
|
|
957
|
+
}
|
|
958
|
+
};
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* @brief Counts the number of matching elements in two unique sorted sets.
|
|
962
|
+
* Can be used to compute the similarity between two textual documents
|
|
963
|
+
* using the IDs of tokens present in them.
|
|
964
|
+
* Similar to `metric_tanimoto_gt` for dense representations.
|
|
965
|
+
*/
|
|
966
|
+
template <typename scalar_at = std::int32_t, typename result_at = float> struct metric_jaccard_gt {
|
|
967
|
+
using scalar_t = scalar_at;
|
|
968
|
+
using result_t = result_at;
|
|
969
|
+
static_assert(!std::is_floating_point<scalar_t>::value, "Jaccard distance requires integral scalars");
|
|
970
|
+
|
|
971
|
+
inline result_t operator()( //
|
|
972
|
+
scalar_t const* a, scalar_t const* b, std::size_t a_length, std::size_t b_length) const noexcept {
|
|
973
|
+
result_t intersection{};
|
|
974
|
+
std::size_t i{};
|
|
975
|
+
std::size_t j{};
|
|
976
|
+
while (i != a_length && j != b_length) {
|
|
977
|
+
intersection += a[i] == b[j];
|
|
978
|
+
i += a[i] < b[j];
|
|
979
|
+
j += a[i] >= b[j];
|
|
980
|
+
}
|
|
981
|
+
return 1 - intersection / (a_length + b_length - intersection);
|
|
982
|
+
}
|
|
983
|
+
};
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* @brief Measures Pearson Correlation between two sequences.
|
|
987
|
+
*/
|
|
988
|
+
template <typename scalar_at = float, typename result_at = float> struct metric_pearson_gt {
|
|
989
|
+
using scalar_t = scalar_at;
|
|
990
|
+
using result_t = result_at;
|
|
991
|
+
|
|
992
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t dim) const noexcept {
|
|
993
|
+
result_t a_sum{}, b_sum{}, ab_sum{};
|
|
994
|
+
result_t a_sq_sum{}, b_sq_sum{};
|
|
995
|
+
#if USEARCH_USE_OPENMP
|
|
996
|
+
#pragma omp simd reduction(+ : a_sum, b_sum, ab_sum, a_sq_sum, b_sq_sum)
|
|
997
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
998
|
+
#pragma clang loop vectorize(enable)
|
|
999
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1000
|
+
#pragma GCC ivdep
|
|
1001
|
+
#endif
|
|
1002
|
+
for (std::size_t i = 0; i != dim; ++i) {
|
|
1003
|
+
a_sum += result_t(a[i]);
|
|
1004
|
+
b_sum += result_t(b[i]);
|
|
1005
|
+
ab_sum += result_t(a[i]) * result_t(b[i]);
|
|
1006
|
+
a_sq_sum += result_t(a[i]) * result_t(a[i]);
|
|
1007
|
+
b_sq_sum += result_t(b[i]) * result_t(b[i]);
|
|
1008
|
+
}
|
|
1009
|
+
result_t denom = std::sqrt((dim * a_sq_sum - a_sum * a_sum) * (dim * b_sq_sum - b_sum * b_sum));
|
|
1010
|
+
result_t corr = (dim * ab_sum - a_sum * b_sum) / denom;
|
|
1011
|
+
return -corr;
|
|
1012
|
+
}
|
|
1013
|
+
};
|
|
1014
|
+
|
|
1015
|
+
struct cos_i8_t {
|
|
1016
|
+
using scalar_t = i8_bits_t;
|
|
1017
|
+
using result_t = f32_t;
|
|
1018
|
+
|
|
1019
|
+
inline result_t operator()(i8_bits_t const* a, i8_bits_t const* b, std::size_t dim) const noexcept {
|
|
1020
|
+
std::int32_t ab{}, a2{}, b2{};
|
|
1021
|
+
#if USEARCH_USE_OPENMP
|
|
1022
|
+
#pragma omp simd reduction(+ : ab, a2, b2)
|
|
1023
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1024
|
+
#pragma clang loop vectorize(enable)
|
|
1025
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1026
|
+
#pragma GCC ivdep
|
|
1027
|
+
#endif
|
|
1028
|
+
for (std::size_t i = 0; i != dim; i++) {
|
|
1029
|
+
std::int16_t ai{a[i]};
|
|
1030
|
+
std::int16_t bi{b[i]};
|
|
1031
|
+
ab += ai * bi;
|
|
1032
|
+
a2 += square(ai);
|
|
1033
|
+
b2 += square(bi);
|
|
1034
|
+
}
|
|
1035
|
+
return (ab != 0) ? (1.f - ab / (std::sqrt(a2) * std::sqrt(b2))) : 0;
|
|
1036
|
+
}
|
|
1037
|
+
};
|
|
1038
|
+
|
|
1039
|
+
struct l2sq_i8_t {
|
|
1040
|
+
using scalar_t = i8_bits_t;
|
|
1041
|
+
using result_t = f32_t;
|
|
1042
|
+
|
|
1043
|
+
inline result_t operator()(i8_bits_t const* a, i8_bits_t const* b, std::size_t dim) const noexcept {
|
|
1044
|
+
std::int32_t ab_deltas_sq{};
|
|
1045
|
+
#if USEARCH_USE_OPENMP
|
|
1046
|
+
#pragma omp simd reduction(+ : ab_deltas_sq)
|
|
1047
|
+
#elif defined(USEARCH_DEFINED_CLANG)
|
|
1048
|
+
#pragma clang loop vectorize(enable)
|
|
1049
|
+
#elif defined(USEARCH_DEFINED_GCC)
|
|
1050
|
+
#pragma GCC ivdep
|
|
1051
|
+
#endif
|
|
1052
|
+
for (std::size_t i = 0; i != dim; i++)
|
|
1053
|
+
ab_deltas_sq += square(std::int16_t(a[i]) - std::int16_t(b[i]));
|
|
1054
|
+
return ab_deltas_sq;
|
|
1055
|
+
}
|
|
1056
|
+
};
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* @brief Haversine distance for the shortest distance between two nodes on
|
|
1060
|
+
* the surface of a 3D sphere, defined with latitude and longitude.
|
|
1061
|
+
*/
|
|
1062
|
+
template <typename scalar_at = float, typename result_at = scalar_at> struct metric_haversine_gt {
|
|
1063
|
+
using scalar_t = scalar_at;
|
|
1064
|
+
using result_t = result_at;
|
|
1065
|
+
static_assert(!std::is_integral<scalar_t>::value, "Latitude and longitude must be floating-node");
|
|
1066
|
+
|
|
1067
|
+
inline result_t operator()(scalar_t const* a, scalar_t const* b, std::size_t = 2) const noexcept {
|
|
1068
|
+
result_t lat_a = a[0], lon_a = a[1];
|
|
1069
|
+
result_t lat_b = b[0], lon_b = b[1];
|
|
1070
|
+
|
|
1071
|
+
result_t lat_delta = angle_to_radians<result_t>(lat_b - lat_a);
|
|
1072
|
+
result_t lon_delta = angle_to_radians<result_t>(lon_b - lon_a);
|
|
1073
|
+
|
|
1074
|
+
result_t converted_lat_a = angle_to_radians<result_t>(lat_a);
|
|
1075
|
+
result_t converted_lat_b = angle_to_radians<result_t>(lat_b);
|
|
1076
|
+
|
|
1077
|
+
result_t x = //
|
|
1078
|
+
square(std::sin(lat_delta / 2.f)) +
|
|
1079
|
+
std::cos(converted_lat_a) * std::cos(converted_lat_b) * square(std::sin(lon_delta / 2.f));
|
|
1080
|
+
|
|
1081
|
+
return std::atan2(std::sqrt(x), std::sqrt(1.f - x));
|
|
1082
|
+
}
|
|
1083
|
+
};
|
|
1084
|
+
|
|
1085
|
+
using distance_punned_t = float;
|
|
1086
|
+
using span_punned_t = span_gt<byte_t const>;
|
|
1087
|
+
|
|
1088
|
+
/**
|
|
1089
|
+
* @brief Type-punned metric class, that can wrap an existing STL `std::function`.
|
|
1090
|
+
* Additional annotation is useful for
|
|
1091
|
+
*/
|
|
1092
|
+
class metric_punned_t {
|
|
1093
|
+
public:
|
|
1094
|
+
using scalar_t = byte_t;
|
|
1095
|
+
using result_t = distance_punned_t;
|
|
1096
|
+
using stl_function_t = std::function<float(byte_t const*, byte_t const*)>;
|
|
1097
|
+
|
|
1098
|
+
private:
|
|
1099
|
+
stl_function_t stl_function_;
|
|
1100
|
+
std::size_t dimensions_ = 0;
|
|
1101
|
+
metric_kind_t metric_kind_ = metric_kind_t::unknown_k;
|
|
1102
|
+
scalar_kind_t scalar_kind_ = scalar_kind_t::unknown_k;
|
|
1103
|
+
isa_kind_t isa_kind_ = isa_kind_t::auto_k;
|
|
1104
|
+
|
|
1105
|
+
public:
|
|
1106
|
+
/**
|
|
1107
|
+
* @brief Computes the distance between two vectors of fixed length.
|
|
1108
|
+
* ! The only relevant function in the object. Everything else is just dynamic dispatch.
|
|
1109
|
+
*/
|
|
1110
|
+
inline result_t operator()(byte_t const* a, byte_t const* b) const noexcept { return stl_function_(a, b); }
|
|
1111
|
+
|
|
1112
|
+
inline metric_punned_t() = default;
|
|
1113
|
+
inline metric_punned_t(metric_punned_t const&) = default;
|
|
1114
|
+
inline metric_punned_t& operator=(metric_punned_t const&) = default;
|
|
1115
|
+
|
|
1116
|
+
inline metric_punned_t( //
|
|
1117
|
+
stl_function_t stl_function, std::size_t dimensions = 0, metric_kind_t metric_kind = metric_kind_t::unknown_k,
|
|
1118
|
+
scalar_kind_t scalar_kind = scalar_kind_t::unknown_k, isa_kind_t isa_kind = isa_kind_t::auto_k)
|
|
1119
|
+
: stl_function_(stl_function), dimensions_(dimensions), metric_kind_(metric_kind), scalar_kind_(scalar_kind),
|
|
1120
|
+
isa_kind_(isa_kind) {}
|
|
1121
|
+
|
|
1122
|
+
inline metric_punned_t( //
|
|
1123
|
+
std::size_t dimensions, metric_kind_t metric_kind = metric_kind_t::cos_k,
|
|
1124
|
+
scalar_kind_t scalar_kind = scalar_kind_t::f32_k) {
|
|
1125
|
+
std::size_t bytes_per_vector = divide_round_up<CHAR_BIT>(dimensions * bits_per_scalar(scalar_kind));
|
|
1126
|
+
*this = make_(bytes_per_vector, metric_kind, scalar_kind);
|
|
1127
|
+
dimensions_ = dimensions;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
std::size_t dimensions() const noexcept { return dimensions_; }
|
|
1131
|
+
metric_kind_t metric_kind() const noexcept { return metric_kind_; }
|
|
1132
|
+
scalar_kind_t scalar_kind() const noexcept { return scalar_kind_; }
|
|
1133
|
+
isa_kind_t isa_kind() const noexcept { return isa_kind_; }
|
|
1134
|
+
|
|
1135
|
+
inline std::size_t bytes_per_vector() const noexcept {
|
|
1136
|
+
return divide_round_up<CHAR_BIT>(dimensions_ * bits_per_scalar(scalar_kind_));
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
inline std::size_t scalar_words() const noexcept {
|
|
1140
|
+
return divide_round_up(dimensions_ * bits_per_scalar(scalar_kind_), bits_per_scalar_word(scalar_kind_));
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
private:
|
|
1144
|
+
static metric_punned_t make_(std::size_t bytes_per_vector, metric_kind_t metric_kind, scalar_kind_t scalar_kind) {
|
|
1145
|
+
|
|
1146
|
+
switch (metric_kind) {
|
|
1147
|
+
case metric_kind_t::ip_k: return ip_metric_(bytes_per_vector, scalar_kind);
|
|
1148
|
+
case metric_kind_t::cos_k: return cos_metric_(bytes_per_vector, scalar_kind);
|
|
1149
|
+
case metric_kind_t::l2sq_k: return l2sq_metric_(bytes_per_vector, scalar_kind);
|
|
1150
|
+
case metric_kind_t::pearson_k: return pearson_metric_(bytes_per_vector, scalar_kind);
|
|
1151
|
+
case metric_kind_t::haversine_k: return haversine_metric_(scalar_kind);
|
|
1152
|
+
|
|
1153
|
+
case metric_kind_t::hamming_k:
|
|
1154
|
+
return {to_stl_<metric_hamming_gt<b1x8_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::hamming_k,
|
|
1155
|
+
scalar_kind_t::b1x8_k, isa_kind_t::auto_k};
|
|
1156
|
+
|
|
1157
|
+
case metric_kind_t::jaccard_k: // Equivalent to Tanimoto
|
|
1158
|
+
case metric_kind_t::tanimoto_k:
|
|
1159
|
+
return {to_stl_<metric_tanimoto_gt<b1x8_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::tanimoto_k,
|
|
1160
|
+
scalar_kind_t::b1x8_k, isa_kind_t::auto_k};
|
|
1161
|
+
|
|
1162
|
+
case metric_kind_t::sorensen_k:
|
|
1163
|
+
return {to_stl_<metric_sorensen_gt<b1x8_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::sorensen_k,
|
|
1164
|
+
scalar_kind_t::b1x8_k, isa_kind_t::auto_k};
|
|
1165
|
+
|
|
1166
|
+
default: return {};
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
template <typename typed_at> static stl_function_t to_stl_(std::size_t bytes) {
|
|
1171
|
+
using scalar_t = typename typed_at::scalar_t;
|
|
1172
|
+
return [=](byte_t const* a, byte_t const* b) -> result_t {
|
|
1173
|
+
return typed_at{}((scalar_t const*)a, (scalar_t const*)b, bytes / sizeof(scalar_t));
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
template <typename scalar_at>
|
|
1178
|
+
static stl_function_t pun_stl_(std::function<result_t(scalar_at const*, scalar_at const*)> typed) {
|
|
1179
|
+
return [=](byte_t const* a, byte_t const* b) -> result_t {
|
|
1180
|
+
return typed((scalar_at const*)a, (scalar_at const*)b);
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// clang-format off
|
|
1185
|
+
static metric_punned_t ip_metric_f32_(std::size_t bytes_per_vector) {
|
|
1186
|
+
#if USEARCH_USE_SIMSIMD
|
|
1187
|
+
if (hardware_supports(isa_kind_t::sve_k)) return {pun_stl_<f32_t>([=](f32_t const* a, f32_t const* b) { return simsimd_dot_f32_sve(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::sve_k};
|
|
1188
|
+
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 16 == 0) return {pun_stl_<f32_t>([=](f32_t const* a, f32_t const* b) { return simsimd_dot_f32x4_neon(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::neon_k};
|
|
1189
|
+
if (hardware_supports(isa_kind_t::avx2_k) && bytes_per_vector % 16 == 0) return {pun_stl_<f32_t>([=](f32_t const* a, f32_t const* b) { return simsimd_dot_f32x4_avx2(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::avx2_k};
|
|
1190
|
+
#endif
|
|
1191
|
+
return {to_stl_<metric_ip_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
static metric_punned_t cos_metric_f16_(std::size_t bytes_per_vector) {
|
|
1195
|
+
#if USEARCH_USE_SIMSIMD
|
|
1196
|
+
if (hardware_supports(isa_kind_t::avx512_k) && bytes_per_vector % 32 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_cos_f16x16_avx512(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::avx512_k};
|
|
1197
|
+
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 8 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_cos_f16x4_neon(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::neon_k};
|
|
1198
|
+
#endif
|
|
1199
|
+
return {to_stl_<metric_cos_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
static metric_punned_t cos_metric_i8_(std::size_t bytes_per_vector) {
|
|
1203
|
+
#if USEARCH_USE_SIMSIMD
|
|
1204
|
+
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 16 == 0) return {pun_stl_<int8_t>([=](int8_t const* a, int8_t const* b) { return simsimd_cos_i8x16_neon(a, b, bytes_per_vector); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::neon_k};
|
|
1205
|
+
#endif
|
|
1206
|
+
return {to_stl_<cos_i8_t>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
static metric_punned_t ip_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
|
|
1210
|
+
switch (scalar_kind) { // The two most common numeric types for the most common metric have optimized versions
|
|
1211
|
+
case scalar_kind_t::f32_k: return ip_metric_f32_(bytes_per_vector);
|
|
1212
|
+
case scalar_kind_t::f16_k: return cos_metric_f16_(bytes_per_vector); // Dot-product accumulates error, Cosine-distance normalizes it
|
|
1213
|
+
case scalar_kind_t::i8_k: return cos_metric_i8_(bytes_per_vector);
|
|
1214
|
+
case scalar_kind_t::f64_k: return {to_stl_<metric_ip_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1215
|
+
default: return {};
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
static metric_punned_t l2sq_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
|
|
1220
|
+
switch (scalar_kind) {
|
|
1221
|
+
case scalar_kind_t::i8_k: return {to_stl_<l2sq_i8_t>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1222
|
+
case scalar_kind_t::f16_k: return {to_stl_<metric_l2sq_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1223
|
+
case scalar_kind_t::f32_k: return {to_stl_<metric_l2sq_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1224
|
+
case scalar_kind_t::f64_k: return {to_stl_<metric_l2sq_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1225
|
+
default: return {};
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
static metric_punned_t cos_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
|
|
1230
|
+
switch (scalar_kind) {
|
|
1231
|
+
case scalar_kind_t::i8_k: return cos_metric_i8_(bytes_per_vector);
|
|
1232
|
+
case scalar_kind_t::f16_k: return cos_metric_f16_(bytes_per_vector);
|
|
1233
|
+
case scalar_kind_t::f32_k: return {to_stl_<metric_cos_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1234
|
+
case scalar_kind_t::f64_k: return {to_stl_<metric_cos_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1235
|
+
default: return {};
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
static metric_punned_t haversine_metric_(scalar_kind_t scalar_kind) {
|
|
1240
|
+
std::size_t bytes_per_vector = 2u * bits_per_scalar(scalar_kind) / CHAR_BIT;
|
|
1241
|
+
switch (scalar_kind) {
|
|
1242
|
+
case scalar_kind_t::i8_k: return {to_stl_<metric_haversine_gt<i8_bits_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1243
|
+
case scalar_kind_t::f16_k: return {to_stl_<metric_haversine_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1244
|
+
case scalar_kind_t::f32_k: return {to_stl_<metric_haversine_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1245
|
+
case scalar_kind_t::f64_k: return {to_stl_<metric_haversine_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::haversine_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1246
|
+
default: return {};
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
static metric_punned_t pearson_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
|
|
1251
|
+
switch (scalar_kind) {
|
|
1252
|
+
case scalar_kind_t::i8_k: return {to_stl_<metric_pearson_gt<i8_bits_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1253
|
+
case scalar_kind_t::f16_k: return {to_stl_<metric_pearson_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1254
|
+
case scalar_kind_t::f32_k: return {to_stl_<metric_pearson_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1255
|
+
case scalar_kind_t::f64_k: return {to_stl_<metric_pearson_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::pearson_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1256
|
+
default: return {};
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
// clang-format on
|
|
1260
|
+
};
|
|
1261
|
+
|
|
1262
|
+
} // namespace usearch
|
|
1263
|
+
} // namespace unum
|