usearch 0.4.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.clang-tidy +60 -0
- package/.github/workflows/release.yml +42 -16
- package/.github/workflows/update_version.sh +1 -0
- package/.vscode/launch.json +22 -0
- package/.vscode/settings.json +5 -0
- package/.vscode/tasks.json +4 -4
- package/CITATION.cff +11 -0
- package/CMakeLists.txt +108 -95
- package/Cargo.toml +1 -1
- package/Dockerfile +14 -0
- package/README.md +8 -7
- package/VERSION +1 -1
- package/binding.gyp +2 -2
- package/build.rs +2 -2
- package/cmake/config.cmake.in +15 -0
- package/cmake/pkg-config.pc.in +4 -0
- package/cmake/usearchConfigVersion.cmake.in +20 -0
- package/conanfile.py +1 -1
- package/docs/benchmarks.md +11 -2
- package/docs/{Compilation.md → compilation.md} +54 -27
- package/include/usearch/usearch.hpp +664 -390
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +17 -16
- package/javascript/lib.cpp +13 -13
- package/javascript/test.js +1 -1
- package/objc/USearchObjective.mm +46 -29
- package/objc/include/USearchObjective.h +18 -26
- package/package.json +1 -1
- package/pyproject.toml +3 -2
- package/python/lib.cpp +179 -120
- package/python/{bench.py → scripts/bench.py} +6 -1
- package/python/scripts/bench_params.py +112 -0
- package/python/{test.py → scripts/test.py} +32 -13
- package/python/usearch/client.py +100 -14
- package/python/usearch/server.py +104 -23
- package/rust/lib.cpp +15 -14
- package/rust/lib.hpp +5 -5
- package/rust/lib.rs +3 -3
- package/setup.py +7 -0
- package/src/CMakeLists.txt +107 -0
- package/src/bench.cpp +45 -15
- package/src/{advanced.hpp → punned.hpp} +227 -199
- package/src/test.cpp +62 -0
- package/swift/Test.swift +1 -1
- package/wolfram/lib.cpp +14 -14
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
#pragma once
|
|
2
|
-
|
|
3
|
-
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
|
|
4
|
-
#define _USE_MATH_DEFINES
|
|
5
|
-
|
|
6
|
-
#define WINDOWS
|
|
7
|
-
#endif
|
|
8
|
-
|
|
9
2
|
#include <stdlib.h> // `aligned_alloc`
|
|
10
3
|
|
|
11
4
|
#include <functional> // `std::function`
|
|
@@ -14,25 +7,26 @@
|
|
|
14
7
|
#include <thread> // `std::thread`
|
|
15
8
|
#include <vector> // `std::vector`
|
|
16
9
|
|
|
17
|
-
#
|
|
10
|
+
#include <fp16/fp16.h>
|
|
11
|
+
|
|
12
|
+
#include <usearch/usearch.hpp>
|
|
13
|
+
|
|
14
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
18
15
|
#include <omp.h> // `omp_get_num_threads()`
|
|
19
16
|
#endif
|
|
20
17
|
|
|
21
|
-
#if
|
|
18
|
+
#if defined(USEARCH_IS_LINUX)
|
|
22
19
|
#include <sys/auxv.h>
|
|
23
20
|
#endif
|
|
24
21
|
|
|
25
|
-
#if defined(
|
|
22
|
+
#if defined(USEARCH_IS_ARM)
|
|
26
23
|
#include <arm_fp16.h>
|
|
27
24
|
#endif
|
|
28
25
|
|
|
29
|
-
#
|
|
30
|
-
#if defined(USEARCH_USE_SIMD)
|
|
26
|
+
#if defined(USEARCH_USE_SIMSIMD)
|
|
31
27
|
#include <simsimd/simsimd.h>
|
|
32
28
|
#endif
|
|
33
29
|
|
|
34
|
-
#include <usearch/usearch.hpp>
|
|
35
|
-
|
|
36
30
|
namespace unum {
|
|
37
31
|
namespace usearch {
|
|
38
32
|
|
|
@@ -41,8 +35,7 @@ using punned_distance_t = float;
|
|
|
41
35
|
using punned_metric_t = punned_distance_t (*)(byte_t const*, byte_t const*, std::size_t, std::size_t);
|
|
42
36
|
|
|
43
37
|
template <typename metric_at>
|
|
44
|
-
|
|
45
|
-
byte_t const* a, byte_t const* b, std::size_t a_bytes, std::size_t b_bytes) noexcept {
|
|
38
|
+
punned_distance_t punned_metric(byte_t const* a, byte_t const* b, std::size_t a_bytes, std::size_t b_bytes) {
|
|
46
39
|
using scalar_t = typename metric_at::scalar_t;
|
|
47
40
|
return metric_at{}((scalar_t const*)a, (scalar_t const*)b, a_bytes / sizeof(scalar_t), b_bytes / sizeof(scalar_t));
|
|
48
41
|
}
|
|
@@ -50,22 +43,20 @@ static punned_distance_t punned_metric( //
|
|
|
50
43
|
using punned_stateful_metric_t =
|
|
51
44
|
std::function<punned_distance_t(byte_t const*, byte_t const*, std::size_t, std::size_t)>;
|
|
52
45
|
|
|
53
|
-
template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp)
|
|
46
|
+
template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp) {
|
|
54
47
|
return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
|
|
55
48
|
}
|
|
56
|
-
template <typename at> inline at clamp(at v, at lo, at hi)
|
|
57
|
-
return usearch::clamp(v, lo, hi, std::less<at>{});
|
|
58
|
-
}
|
|
49
|
+
template <typename at> inline at clamp(at v, at lo, at hi) { return usearch::clamp(v, lo, hi, std::less<at>{}); }
|
|
59
50
|
|
|
60
51
|
class f8_bits_t;
|
|
61
52
|
class f16_bits_t;
|
|
62
53
|
|
|
63
|
-
inline float f16_to_f32(std::uint16_t u16)
|
|
54
|
+
inline float f16_to_f32(std::uint16_t u16) {
|
|
64
55
|
#if defined(__AVX512F__)
|
|
65
56
|
_Float16 f16;
|
|
66
57
|
std::memcpy(&f16, &u16, sizeof(std::uint16_t));
|
|
67
58
|
return float(f16);
|
|
68
|
-
#elif defined(
|
|
59
|
+
#elif defined(USEARCH_IS_ARM)
|
|
69
60
|
__fp16 f16;
|
|
70
61
|
std::memcpy(&f16, &u16, sizeof(std::uint16_t));
|
|
71
62
|
return float(f16);
|
|
@@ -74,13 +65,13 @@ inline float f16_to_f32(std::uint16_t u16) noexcept {
|
|
|
74
65
|
#endif
|
|
75
66
|
}
|
|
76
67
|
|
|
77
|
-
inline std::uint16_t f32_to_f16(float f32)
|
|
68
|
+
inline std::uint16_t f32_to_f16(float f32) {
|
|
78
69
|
#if defined(__AVX512F__)
|
|
79
70
|
_Float16 f16 = _Float16(f32);
|
|
80
71
|
std::uint16_t u16;
|
|
81
72
|
std::memcpy(&u16, &f16, sizeof(std::uint16_t));
|
|
82
73
|
return u16;
|
|
83
|
-
#elif defined(
|
|
74
|
+
#elif defined(USEARCH_IS_ARM)
|
|
84
75
|
__fp16 f16 = __fp16(f32);
|
|
85
76
|
std::uint16_t u16;
|
|
86
77
|
std::memcpy(&u16, &f16, sizeof(std::uint16_t));
|
|
@@ -94,47 +85,47 @@ class f16_bits_t {
|
|
|
94
85
|
std::uint16_t uint16_{};
|
|
95
86
|
|
|
96
87
|
public:
|
|
97
|
-
inline f16_bits_t()
|
|
88
|
+
inline f16_bits_t() : uint16_(0) {}
|
|
98
89
|
inline f16_bits_t(f16_bits_t&&) = default;
|
|
99
90
|
inline f16_bits_t& operator=(f16_bits_t&&) = default;
|
|
100
91
|
inline f16_bits_t(f16_bits_t const&) = default;
|
|
101
92
|
inline f16_bits_t& operator=(f16_bits_t const&) = default;
|
|
102
93
|
|
|
103
|
-
inline operator float() const
|
|
104
|
-
|
|
105
|
-
inline f16_bits_t(f8_bits_t)
|
|
106
|
-
inline f16_bits_t(float v)
|
|
107
|
-
inline f16_bits_t(double v)
|
|
108
|
-
|
|
109
|
-
inline f16_bits_t operator+(f16_bits_t other) const
|
|
110
|
-
inline f16_bits_t operator-(f16_bits_t other) const
|
|
111
|
-
inline f16_bits_t operator*(f16_bits_t other) const
|
|
112
|
-
inline f16_bits_t operator/(f16_bits_t other) const
|
|
113
|
-
inline f16_bits_t operator+(float other) const
|
|
114
|
-
inline f16_bits_t operator-(float other) const
|
|
115
|
-
inline f16_bits_t operator*(float other) const
|
|
116
|
-
inline f16_bits_t operator/(float other) const
|
|
117
|
-
inline f16_bits_t operator+(double other) const
|
|
118
|
-
inline f16_bits_t operator-(double other) const
|
|
119
|
-
inline f16_bits_t operator*(double other) const
|
|
120
|
-
inline f16_bits_t operator/(double other) const
|
|
121
|
-
|
|
122
|
-
inline f16_bits_t& operator+=(float v)
|
|
94
|
+
inline operator float() const { return f16_to_f32(uint16_); }
|
|
95
|
+
|
|
96
|
+
inline f16_bits_t(f8_bits_t);
|
|
97
|
+
inline f16_bits_t(float v) : uint16_(f32_to_f16(v)) {}
|
|
98
|
+
inline f16_bits_t(double v) : uint16_(f32_to_f16(v)) {}
|
|
99
|
+
|
|
100
|
+
inline f16_bits_t operator+(f16_bits_t other) const { return {float(*this) + float(other)}; }
|
|
101
|
+
inline f16_bits_t operator-(f16_bits_t other) const { return {float(*this) - float(other)}; }
|
|
102
|
+
inline f16_bits_t operator*(f16_bits_t other) const { return {float(*this) * float(other)}; }
|
|
103
|
+
inline f16_bits_t operator/(f16_bits_t other) const { return {float(*this) / float(other)}; }
|
|
104
|
+
inline f16_bits_t operator+(float other) const { return {float(*this) + other}; }
|
|
105
|
+
inline f16_bits_t operator-(float other) const { return {float(*this) - other}; }
|
|
106
|
+
inline f16_bits_t operator*(float other) const { return {float(*this) * other}; }
|
|
107
|
+
inline f16_bits_t operator/(float other) const { return {float(*this) / other}; }
|
|
108
|
+
inline f16_bits_t operator+(double other) const { return {float(*this) + other}; }
|
|
109
|
+
inline f16_bits_t operator-(double other) const { return {float(*this) - other}; }
|
|
110
|
+
inline f16_bits_t operator*(double other) const { return {float(*this) * other}; }
|
|
111
|
+
inline f16_bits_t operator/(double other) const { return {float(*this) / other}; }
|
|
112
|
+
|
|
113
|
+
inline f16_bits_t& operator+=(float v) {
|
|
123
114
|
uint16_ = f32_to_f16(v + f16_to_f32(uint16_));
|
|
124
115
|
return *this;
|
|
125
116
|
}
|
|
126
117
|
|
|
127
|
-
inline f16_bits_t& operator-=(float v)
|
|
118
|
+
inline f16_bits_t& operator-=(float v) {
|
|
128
119
|
uint16_ = f32_to_f16(v - f16_to_f32(uint16_));
|
|
129
120
|
return *this;
|
|
130
121
|
}
|
|
131
122
|
|
|
132
|
-
inline f16_bits_t& operator*=(float v)
|
|
123
|
+
inline f16_bits_t& operator*=(float v) {
|
|
133
124
|
uint16_ = f32_to_f16(v * f16_to_f32(uint16_));
|
|
134
125
|
return *this;
|
|
135
126
|
}
|
|
136
127
|
|
|
137
|
-
inline f16_bits_t& operator/=(float v)
|
|
128
|
+
inline f16_bits_t& operator/=(float v) {
|
|
138
129
|
uint16_ = f32_to_f16(v / f16_to_f32(uint16_));
|
|
139
130
|
return *this;
|
|
140
131
|
}
|
|
@@ -152,30 +143,30 @@ class f8_bits_t {
|
|
|
152
143
|
constexpr static std::int8_t min_k = -100;
|
|
153
144
|
constexpr static std::int8_t max_k = 100;
|
|
154
145
|
|
|
155
|
-
inline f8_bits_t()
|
|
146
|
+
inline f8_bits_t() : int8_(0) {}
|
|
156
147
|
inline f8_bits_t(f8_bits_t&&) = default;
|
|
157
148
|
inline f8_bits_t& operator=(f8_bits_t&&) = default;
|
|
158
149
|
inline f8_bits_t(f8_bits_t const&) = default;
|
|
159
150
|
inline f8_bits_t& operator=(f8_bits_t const&) = default;
|
|
160
151
|
|
|
161
|
-
inline operator float() const
|
|
162
|
-
inline operator f16_bits_t() const
|
|
163
|
-
inline operator double() const
|
|
164
|
-
inline explicit operator std::int8_t() const
|
|
165
|
-
inline explicit operator std::int16_t() const
|
|
166
|
-
inline explicit operator std::int32_t() const
|
|
167
|
-
inline explicit operator std::int64_t() const
|
|
152
|
+
inline operator float() const { return float(int8_) / divisor_k; }
|
|
153
|
+
inline operator f16_bits_t() const { return float(int8_) / divisor_k; }
|
|
154
|
+
inline operator double() const { return double(int8_) / divisor_k; }
|
|
155
|
+
inline explicit operator std::int8_t() const { return int8_; }
|
|
156
|
+
inline explicit operator std::int16_t() const { return int8_; }
|
|
157
|
+
inline explicit operator std::int32_t() const { return int8_; }
|
|
158
|
+
inline explicit operator std::int64_t() const { return int8_; }
|
|
168
159
|
|
|
169
|
-
inline f8_bits_t(float v)
|
|
160
|
+
inline f8_bits_t(float v)
|
|
170
161
|
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
171
|
-
inline f8_bits_t(double v)
|
|
162
|
+
inline f8_bits_t(double v)
|
|
172
163
|
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
173
164
|
};
|
|
174
165
|
|
|
175
|
-
inline f16_bits_t::f16_bits_t(f8_bits_t v)
|
|
166
|
+
inline f16_bits_t::f16_bits_t(f8_bits_t v) : f16_bits_t(float(v)) {}
|
|
176
167
|
|
|
177
168
|
struct cos_f8_t {
|
|
178
|
-
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const
|
|
169
|
+
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const {
|
|
179
170
|
std::int32_t ab{}, a2{}, b2{};
|
|
180
171
|
for (std::size_t i = 0; i != n; i++) {
|
|
181
172
|
std::int16_t ai{a[i]};
|
|
@@ -189,7 +180,7 @@ struct cos_f8_t {
|
|
|
189
180
|
};
|
|
190
181
|
|
|
191
182
|
struct l2sq_f8_t {
|
|
192
|
-
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const
|
|
183
|
+
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const {
|
|
193
184
|
std::int32_t ab_deltas_sq{};
|
|
194
185
|
for (std::size_t i = 0; i != n; i++)
|
|
195
186
|
ab_deltas_sq += square(std::int16_t(a[i]) - std::int16_t(b[i]));
|
|
@@ -212,7 +203,7 @@ void multithreaded(std::size_t threads, std::size_t tasks, callback_at&& callbac
|
|
|
212
203
|
return;
|
|
213
204
|
}
|
|
214
205
|
|
|
215
|
-
#if defined(
|
|
206
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
216
207
|
omp_set_num_threads(threads);
|
|
217
208
|
#pragma omp parallel for schedule(dynamic)
|
|
218
209
|
for (std::size_t i = 0; i < tasks; ++i)
|
|
@@ -246,13 +237,13 @@ class aligned_allocator_gt {
|
|
|
246
237
|
using other = aligned_allocator_gt<other_element_at>;
|
|
247
238
|
};
|
|
248
239
|
|
|
249
|
-
pointer allocate(size_type length) const
|
|
240
|
+
pointer allocate(size_type length) const {
|
|
250
241
|
void* result = nullptr;
|
|
251
242
|
int status = posix_memalign(&result, alignment_ak, ceil2(length * sizeof(value_type)));
|
|
252
243
|
return status == 0 ? (pointer)result : nullptr;
|
|
253
244
|
}
|
|
254
245
|
|
|
255
|
-
void deallocate(pointer begin, size_type) const
|
|
246
|
+
void deallocate(pointer begin, size_type) const { std::free(begin); }
|
|
256
247
|
};
|
|
257
248
|
|
|
258
249
|
using aligned_allocator_t = aligned_allocator_gt<>;
|
|
@@ -264,7 +255,7 @@ enum class accuracy_t {
|
|
|
264
255
|
f8_k,
|
|
265
256
|
};
|
|
266
257
|
|
|
267
|
-
inline char const* accuracy_name(accuracy_t accuracy)
|
|
258
|
+
inline char const* accuracy_name(accuracy_t accuracy) {
|
|
268
259
|
switch (accuracy) {
|
|
269
260
|
case accuracy_t::f32_k: return "f32";
|
|
270
261
|
case accuracy_t::f16_k: return "f16";
|
|
@@ -274,7 +265,7 @@ inline char const* accuracy_name(accuracy_t accuracy) noexcept {
|
|
|
274
265
|
}
|
|
275
266
|
}
|
|
276
267
|
|
|
277
|
-
inline bool str_equals(char const* begin, std::size_t len, char const* other_begin)
|
|
268
|
+
inline bool str_equals(char const* begin, std::size_t len, char const* other_begin) {
|
|
278
269
|
std::size_t other_len = strlen(other_begin);
|
|
279
270
|
return len == other_len && strncmp(begin, other_begin, len) == 0;
|
|
280
271
|
}
|
|
@@ -298,10 +289,10 @@ template <typename index_at>
|
|
|
298
289
|
inline index_at index_from_name( //
|
|
299
290
|
char const* name, std::size_t len, std::size_t dimensions, accuracy_t accuracy, config_t const& config) {
|
|
300
291
|
|
|
301
|
-
if (str_equals(name, len, "
|
|
292
|
+
if (str_equals(name, len, "l2sq") || str_equals(name, len, "euclidean_sq")) {
|
|
302
293
|
if (dimensions == 0)
|
|
303
294
|
throw std::invalid_argument("The number of dimensions must be positive");
|
|
304
|
-
return index_at::
|
|
295
|
+
return index_at::l2sq(dimensions, accuracy, config);
|
|
305
296
|
} else if (str_equals(name, len, "ip") || str_equals(name, len, "inner") || str_equals(name, len, "dot")) {
|
|
306
297
|
if (dimensions == 0)
|
|
307
298
|
throw std::invalid_argument("The number of dimensions must be positive");
|
|
@@ -315,7 +306,7 @@ inline index_at index_from_name( //
|
|
|
315
306
|
throw std::invalid_argument("The number of dimensions must be equal to two");
|
|
316
307
|
return index_at::haversine(accuracy, config);
|
|
317
308
|
} else
|
|
318
|
-
throw std::invalid_argument("Unknown distance, choose:
|
|
309
|
+
throw std::invalid_argument("Unknown distance, choose: l2sq, ip, cos, hamming, jaccard");
|
|
319
310
|
return {};
|
|
320
311
|
}
|
|
321
312
|
|
|
@@ -327,7 +318,7 @@ enum class isa_t {
|
|
|
327
318
|
avx512_k,
|
|
328
319
|
};
|
|
329
320
|
|
|
330
|
-
inline char const* isa_name(isa_t isa)
|
|
321
|
+
inline char const* isa_name(isa_t isa) {
|
|
331
322
|
switch (isa) {
|
|
332
323
|
case isa_t::auto_k: return "auto";
|
|
333
324
|
case isa_t::neon_k: return "neon";
|
|
@@ -338,7 +329,7 @@ inline char const* isa_name(isa_t isa) noexcept {
|
|
|
338
329
|
}
|
|
339
330
|
}
|
|
340
331
|
|
|
341
|
-
inline std::size_t bytes_per_scalar(accuracy_t accuracy)
|
|
332
|
+
inline std::size_t bytes_per_scalar(accuracy_t accuracy) {
|
|
342
333
|
switch (accuracy) {
|
|
343
334
|
case accuracy_t::f32_k: return 4;
|
|
344
335
|
case accuracy_t::f16_k: return 2;
|
|
@@ -349,10 +340,10 @@ inline std::size_t bytes_per_scalar(accuracy_t accuracy) noexcept {
|
|
|
349
340
|
}
|
|
350
341
|
|
|
351
342
|
inline bool supports_arm_sve() {
|
|
352
|
-
#if defined(
|
|
353
|
-
#if
|
|
354
|
-
unsigned long
|
|
355
|
-
if (
|
|
343
|
+
#if defined(USEARCH_IS_ARM)
|
|
344
|
+
#if defined(USEARCH_IS_LINUX)
|
|
345
|
+
unsigned long capabilities = getauxval(AT_HWCAP);
|
|
346
|
+
if (capabilities & HWCAP_SVE)
|
|
356
347
|
return true;
|
|
357
348
|
#endif
|
|
358
349
|
#endif
|
|
@@ -360,7 +351,7 @@ inline bool supports_arm_sve() {
|
|
|
360
351
|
}
|
|
361
352
|
|
|
362
353
|
template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
|
|
363
|
-
bool operator()(byte_t const* input, std::size_t bytes_in_input, byte_t* output)
|
|
354
|
+
bool operator()(byte_t const* input, std::size_t bytes_in_input, byte_t* output) const {
|
|
364
355
|
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
365
356
|
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
366
357
|
std::transform( //
|
|
@@ -371,15 +362,19 @@ template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
|
|
|
371
362
|
};
|
|
372
363
|
|
|
373
364
|
template <> struct cast_gt<f32_t, f32_t> {
|
|
374
|
-
bool operator()(byte_t const*, std::size_t, byte_t*)
|
|
365
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
375
366
|
};
|
|
376
367
|
|
|
377
368
|
template <> struct cast_gt<f64_t, f64_t> {
|
|
378
|
-
bool operator()(byte_t const*, std::size_t, byte_t*)
|
|
369
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
379
370
|
};
|
|
380
371
|
|
|
381
372
|
template <> struct cast_gt<f16_bits_t, f16_bits_t> {
|
|
382
|
-
bool operator()(byte_t const*, std::size_t, byte_t*)
|
|
373
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
template <> struct cast_gt<f8_bits_t, f8_bits_t> {
|
|
377
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
383
378
|
};
|
|
384
379
|
|
|
385
380
|
/**
|
|
@@ -387,15 +382,12 @@ template <> struct cast_gt<f16_bits_t, f16_bits_t> {
|
|
|
387
382
|
* vectors with automatic down-casting and hardware acceleration.
|
|
388
383
|
*/
|
|
389
384
|
template <typename label_at = std::int64_t, typename id_at = std::uint32_t> //
|
|
390
|
-
class
|
|
385
|
+
class punned_gt {
|
|
391
386
|
public:
|
|
392
387
|
using label_t = label_at;
|
|
393
388
|
using id_t = id_at;
|
|
394
389
|
using distance_t = punned_distance_t;
|
|
395
390
|
|
|
396
|
-
using f32_t = float;
|
|
397
|
-
using f64_t = double;
|
|
398
|
-
|
|
399
391
|
private:
|
|
400
392
|
/// @brief Schema: input buffer, bytes in input buffer, output buffer.
|
|
401
393
|
using cast_t = std::function<bool(byte_t const*, std::size_t, byte_t*)>;
|
|
@@ -422,22 +414,24 @@ class auto_index_gt {
|
|
|
422
414
|
} casts_;
|
|
423
415
|
|
|
424
416
|
punned_stateful_metric_t root_metric_;
|
|
425
|
-
config_t root_config_;
|
|
426
417
|
|
|
427
418
|
mutable std::vector<std::size_t> available_threads_;
|
|
428
419
|
mutable std::mutex available_threads_mutex_;
|
|
429
420
|
|
|
430
421
|
public:
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
422
|
+
using search_results_t = typename index_t::search_results_t;
|
|
423
|
+
using add_result_t = typename index_t::add_result_t;
|
|
424
|
+
|
|
425
|
+
punned_gt() = default;
|
|
426
|
+
punned_gt(punned_gt&& other) { swap(other); }
|
|
427
|
+
punned_gt& operator=(punned_gt&& other) {
|
|
434
428
|
swap(other);
|
|
435
429
|
return *this;
|
|
436
430
|
}
|
|
437
431
|
|
|
438
|
-
~
|
|
432
|
+
~punned_gt() { aligned_index_free_(index_); }
|
|
439
433
|
|
|
440
|
-
void swap(
|
|
434
|
+
void swap(punned_gt& other) {
|
|
441
435
|
std::swap(dimensions_, other.dimensions_);
|
|
442
436
|
std::swap(casted_vector_bytes_, other.casted_vector_bytes_);
|
|
443
437
|
std::swap(accuracy_, other.accuracy_);
|
|
@@ -446,22 +440,20 @@ class auto_index_gt {
|
|
|
446
440
|
std::swap(cast_buffer_, other.cast_buffer_);
|
|
447
441
|
std::swap(casts_, other.casts_);
|
|
448
442
|
std::swap(root_metric_, other.root_metric_);
|
|
449
|
-
std::swap(root_config_, other.root_config_);
|
|
450
443
|
std::swap(available_threads_, other.available_threads_);
|
|
451
444
|
}
|
|
452
445
|
|
|
453
|
-
|
|
454
|
-
std::size_t connectivity() const noexcept { return index_->connectivity(); }
|
|
455
|
-
std::size_t size() const noexcept { return index_->size(); }
|
|
456
|
-
std::size_t capacity() const noexcept { return index_->capacity(); }
|
|
457
|
-
config_t const& config() const noexcept { return root_config_; }
|
|
458
|
-
void clear() noexcept { return index_->clear(); }
|
|
446
|
+
static config_t optimize(config_t config) noexcept { return index_t::optimize(config); }
|
|
459
447
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
std::size_t
|
|
463
|
-
|
|
464
|
-
}
|
|
448
|
+
std::size_t dimensions() const { return dimensions_; }
|
|
449
|
+
std::size_t connectivity() const { return index_->connectivity(); }
|
|
450
|
+
std::size_t size() const { return index_->size(); }
|
|
451
|
+
std::size_t capacity() const { return index_->capacity(); }
|
|
452
|
+
config_t const& config() const { return index_->config(); }
|
|
453
|
+
void clear() { return index_->clear(); }
|
|
454
|
+
|
|
455
|
+
accuracy_t accuracy() const { return accuracy_; }
|
|
456
|
+
isa_t acceleration() const { return acceleration_; }
|
|
465
457
|
|
|
466
458
|
void save(char const* path) const { index_->save(path); }
|
|
467
459
|
void load(char const* path) { index_->load(path); }
|
|
@@ -469,36 +461,50 @@ class auto_index_gt {
|
|
|
469
461
|
void reserve(std::size_t capacity) { index_->reserve(capacity); }
|
|
470
462
|
|
|
471
463
|
// clang-format off
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
464
|
+
add_result_t add(label_t label, f8_bits_t const* vector) { return add_(label, vector, casts_.from_f8); }
|
|
465
|
+
add_result_t add(label_t label, f16_bits_t const* vector) { return add_(label, vector, casts_.from_f16); }
|
|
466
|
+
add_result_t add(label_t label, f32_t const* vector) { return add_(label, vector, casts_.from_f32); }
|
|
467
|
+
add_result_t add(label_t label, f64_t const* vector) { return add_(label, vector, casts_.from_f64); }
|
|
468
|
+
|
|
469
|
+
add_result_t add(label_t label, f8_bits_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f8); }
|
|
470
|
+
add_result_t add(label_t label, f16_bits_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f16); }
|
|
471
|
+
add_result_t add(label_t label, f32_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f32); }
|
|
472
|
+
add_result_t add(label_t label, f64_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f64); }
|
|
473
|
+
|
|
474
|
+
search_results_t search(f8_bits_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f8); }
|
|
475
|
+
search_results_t search(f16_bits_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f16); }
|
|
476
|
+
search_results_t search(f32_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f32); }
|
|
477
|
+
search_results_t search(f64_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f64); }
|
|
478
|
+
|
|
479
|
+
search_results_t search(f8_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f8); }
|
|
480
|
+
search_results_t search(f16_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f16); }
|
|
481
|
+
search_results_t search(f32_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f32); }
|
|
482
|
+
search_results_t search(f64_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f64); }
|
|
483
|
+
|
|
484
|
+
search_results_t search_around(label_t hint, f8_bits_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f8); }
|
|
485
|
+
search_results_t search_around(label_t hint, f16_bits_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f16); }
|
|
486
|
+
search_results_t search_around(label_t hint, f32_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f32); }
|
|
487
|
+
search_results_t search_around(label_t hint, f64_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f64); }
|
|
488
|
+
|
|
489
|
+
search_results_t search_around(label_t hint, f8_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f8); }
|
|
490
|
+
search_results_t search_around(label_t hint, f16_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f16); }
|
|
491
|
+
search_results_t search_around(label_t hint, f32_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f32); }
|
|
492
|
+
search_results_t search_around(label_t hint, f64_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f64); }
|
|
493
|
+
|
|
494
|
+
static punned_gt ip(std::size_t dimensions, accuracy_t accuracy = accuracy_t::f16_k, config_t config = {}) { return make_(dimensions, accuracy, ip_metric_(dimensions, accuracy), make_casts_(accuracy), config); }
|
|
495
|
+
static punned_gt l2sq(std::size_t dimensions, accuracy_t accuracy = accuracy_t::f16_k, config_t config = {}) { return make_(dimensions, accuracy, l2_metric_(dimensions, accuracy), make_casts_(accuracy), config); }
|
|
496
|
+
static punned_gt cos(std::size_t dimensions, accuracy_t accuracy = accuracy_t::f32_k, config_t config = {}) { return make_(dimensions, accuracy, cos_metric_(dimensions, accuracy), make_casts_(accuracy), config); }
|
|
497
|
+
static punned_gt haversine(accuracy_t accuracy = accuracy_t::f32_k, config_t config = {}) { return make_(2, accuracy, haversine_metric_(accuracy), make_casts_(accuracy), config); }
|
|
492
498
|
// clang-format on
|
|
493
499
|
|
|
494
|
-
static
|
|
500
|
+
static punned_gt udf( //
|
|
495
501
|
std::size_t dimensions, punned_stateful_metric_t metric, //
|
|
496
502
|
accuracy_t accuracy = accuracy_t::f32_k, config_t config = {}) {
|
|
497
|
-
return
|
|
503
|
+
return make_(dimensions, accuracy, {metric, isa_t::auto_k}, make_casts_(accuracy), config);
|
|
498
504
|
}
|
|
499
505
|
|
|
500
|
-
|
|
501
|
-
|
|
506
|
+
punned_gt fork() const {
|
|
507
|
+
punned_gt result;
|
|
502
508
|
|
|
503
509
|
result.dimensions_ = dimensions_;
|
|
504
510
|
result.accuracy_ = accuracy_;
|
|
@@ -508,33 +514,32 @@ class auto_index_gt {
|
|
|
508
514
|
result.casts_ = casts_;
|
|
509
515
|
|
|
510
516
|
result.root_metric_ = root_metric_;
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
new (raw) index_t(root_config_, root_metric_);
|
|
517
|
+
index_t* raw = aligned_index_alloc_();
|
|
518
|
+
new (raw) index_t(config(), root_metric_);
|
|
514
519
|
result.index_ = raw;
|
|
515
520
|
|
|
516
521
|
return result;
|
|
517
522
|
}
|
|
518
523
|
|
|
519
524
|
private:
|
|
520
|
-
static index_t*
|
|
521
|
-
#if defined(
|
|
525
|
+
static index_t* aligned_index_alloc_() {
|
|
526
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
522
527
|
return (index_t*)_aligned_malloc(64, 64 * divide_round_up<64>(sizeof(index_t)));
|
|
523
528
|
#else
|
|
524
529
|
return (index_t*)aligned_alloc(64, 64 * divide_round_up<64>(sizeof(index_t)));
|
|
525
530
|
#endif
|
|
526
531
|
}
|
|
527
532
|
|
|
528
|
-
static void
|
|
533
|
+
static void aligned_index_free_(index_t* raw) { free(raw); }
|
|
529
534
|
|
|
530
535
|
struct thread_lock_t {
|
|
531
|
-
|
|
536
|
+
punned_gt const& parent;
|
|
532
537
|
std::size_t thread_id;
|
|
533
538
|
|
|
534
|
-
~thread_lock_t() { parent.
|
|
539
|
+
~thread_lock_t() { parent.thread_unlock_(thread_id); }
|
|
535
540
|
};
|
|
536
541
|
|
|
537
|
-
thread_lock_t
|
|
542
|
+
thread_lock_t thread_lock_() const {
|
|
538
543
|
available_threads_mutex_.lock();
|
|
539
544
|
std::size_t thread_id = available_threads_.back();
|
|
540
545
|
available_threads_.pop_back();
|
|
@@ -542,61 +547,85 @@ class auto_index_gt {
|
|
|
542
547
|
return {*this, thread_id};
|
|
543
548
|
}
|
|
544
549
|
|
|
545
|
-
void
|
|
550
|
+
void thread_unlock_(std::size_t thread_id) const {
|
|
546
551
|
available_threads_mutex_.lock();
|
|
547
552
|
available_threads_.push_back(thread_id);
|
|
548
553
|
available_threads_mutex_.unlock();
|
|
549
554
|
}
|
|
550
555
|
|
|
551
556
|
template <typename scalar_at>
|
|
552
|
-
|
|
557
|
+
add_result_t add_(label_t label, scalar_at const* vector, add_config_t config, cast_t const& cast) {
|
|
553
558
|
byte_t const* vector_data = reinterpret_cast<byte_t const*>(vector);
|
|
554
559
|
std::size_t vector_bytes = dimensions_ * sizeof(scalar_at);
|
|
555
560
|
|
|
556
561
|
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * config.thread;
|
|
557
|
-
bool casted = cast(vector_data,
|
|
562
|
+
bool casted = cast(vector_data, vector_bytes, casted_data);
|
|
558
563
|
if (casted)
|
|
559
564
|
vector_data = casted_data, vector_bytes = casted_vector_bytes_, config.store_vector = true;
|
|
560
565
|
|
|
561
|
-
index_->add(label, {vector_data, vector_bytes}, config);
|
|
566
|
+
return index_->add(label, {vector_data, vector_bytes}, config);
|
|
562
567
|
}
|
|
563
568
|
|
|
564
569
|
template <typename scalar_at>
|
|
565
|
-
|
|
570
|
+
search_results_t search_( //
|
|
566
571
|
scalar_at const* vector, std::size_t wanted, //
|
|
567
|
-
label_t* matches, distance_t* distances, //
|
|
568
572
|
search_config_t config, cast_t const& cast) const {
|
|
569
573
|
|
|
570
574
|
byte_t const* vector_data = reinterpret_cast<byte_t const*>(vector);
|
|
571
575
|
std::size_t vector_bytes = dimensions_ * sizeof(scalar_at);
|
|
572
576
|
|
|
573
577
|
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * config.thread;
|
|
574
|
-
bool casted = cast(vector_data,
|
|
578
|
+
bool casted = cast(vector_data, vector_bytes, casted_data);
|
|
575
579
|
if (casted)
|
|
576
580
|
vector_data = casted_data, vector_bytes = casted_vector_bytes_;
|
|
577
581
|
|
|
578
|
-
return index_->search({vector_data, vector_bytes}, wanted,
|
|
582
|
+
return index_->search({vector_data, vector_bytes}, wanted, config);
|
|
579
583
|
}
|
|
580
584
|
|
|
581
|
-
template <typename scalar_at>
|
|
582
|
-
|
|
585
|
+
template <typename scalar_at>
|
|
586
|
+
search_results_t search_around_( //
|
|
587
|
+
label_t hint, scalar_at const* vector, std::size_t wanted, //
|
|
588
|
+
search_config_t config, cast_t const& cast) const {
|
|
589
|
+
|
|
590
|
+
byte_t const* vector_data = reinterpret_cast<byte_t const*>(vector);
|
|
591
|
+
std::size_t vector_bytes = dimensions_ * sizeof(scalar_at);
|
|
592
|
+
|
|
593
|
+
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * config.thread;
|
|
594
|
+
bool casted = cast(vector_data, vector_bytes, casted_data);
|
|
595
|
+
if (casted)
|
|
596
|
+
vector_data = casted_data, vector_bytes = casted_vector_bytes_;
|
|
597
|
+
|
|
598
|
+
return index_->search_around(hint, {vector_data, vector_bytes}, wanted, config);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
template <typename scalar_at> add_result_t add_(label_t label, scalar_at const* vector, cast_t const& cast) {
|
|
602
|
+
thread_lock_t lock = thread_lock_();
|
|
583
603
|
add_config_t config;
|
|
584
604
|
config.thread = lock.thread_id;
|
|
585
|
-
return
|
|
605
|
+
return add_(label, vector, config, cast);
|
|
586
606
|
}
|
|
587
607
|
|
|
588
608
|
template <typename scalar_at>
|
|
589
|
-
|
|
609
|
+
search_results_t search_( //
|
|
590
610
|
scalar_at const* vector, std::size_t wanted, //
|
|
591
|
-
label_t* matches, distance_t* distances, //
|
|
592
611
|
cast_t const& cast) const {
|
|
593
|
-
thread_lock_t lock =
|
|
612
|
+
thread_lock_t lock = thread_lock_();
|
|
613
|
+
search_config_t config;
|
|
614
|
+
config.thread = lock.thread_id;
|
|
615
|
+
return search_(vector, wanted, config, cast);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
template <typename scalar_at>
|
|
619
|
+
search_results_t search_around_( //
|
|
620
|
+
label_t hint, scalar_at const* vector, std::size_t wanted, //
|
|
621
|
+
cast_t const& cast) const {
|
|
622
|
+
thread_lock_t lock = thread_lock_();
|
|
594
623
|
search_config_t config;
|
|
595
624
|
config.thread = lock.thread_id;
|
|
596
|
-
return
|
|
625
|
+
return search_around_(hint, vector, wanted, config, cast);
|
|
597
626
|
}
|
|
598
627
|
|
|
599
|
-
static
|
|
628
|
+
static punned_gt make_( //
|
|
600
629
|
std::size_t dimensions, accuracy_t accuracy, //
|
|
601
630
|
metric_and_meta_t metric_and_meta, casts_t casts, //
|
|
602
631
|
config_t config) {
|
|
@@ -605,7 +634,7 @@ class auto_index_gt {
|
|
|
605
634
|
config.max_threads_add = config.max_threads_add ? config.max_threads_add : hardware_threads;
|
|
606
635
|
config.max_threads_search = config.max_threads_search ? config.max_threads_search : hardware_threads;
|
|
607
636
|
std::size_t max_threads = (std::max)(config.max_threads_add, config.max_threads_search);
|
|
608
|
-
|
|
637
|
+
punned_gt result;
|
|
609
638
|
result.dimensions_ = dimensions;
|
|
610
639
|
result.accuracy_ = accuracy;
|
|
611
640
|
result.casted_vector_bytes_ = bytes_per_scalar(accuracy) * dimensions;
|
|
@@ -613,20 +642,19 @@ class auto_index_gt {
|
|
|
613
642
|
result.casts_ = casts;
|
|
614
643
|
result.acceleration_ = metric_and_meta.acceleration;
|
|
615
644
|
result.root_metric_ = metric_and_meta.metric;
|
|
616
|
-
result.root_config_ = config;
|
|
617
645
|
|
|
618
646
|
// Fill the thread IDs.
|
|
619
647
|
result.available_threads_.resize(max_threads);
|
|
620
648
|
std::iota(result.available_threads_.begin(), result.available_threads_.end(), 0ul);
|
|
621
649
|
|
|
622
|
-
//
|
|
623
|
-
index_t* raw =
|
|
650
|
+
// Available since C11, but only C++17, so we use the C version.
|
|
651
|
+
index_t* raw = aligned_index_alloc_();
|
|
624
652
|
new (raw) index_t(config, metric_and_meta.metric);
|
|
625
653
|
result.index_ = raw;
|
|
626
654
|
return result;
|
|
627
655
|
}
|
|
628
656
|
|
|
629
|
-
template <typename to_scalar_at> static casts_t
|
|
657
|
+
template <typename to_scalar_at> static casts_t make_casts_() {
|
|
630
658
|
casts_t result;
|
|
631
659
|
result.from_f8 = cast_gt<f8_bits_t, to_scalar_at>{};
|
|
632
660
|
result.from_f16 = cast_gt<f16_bits_t, to_scalar_at>{};
|
|
@@ -635,126 +663,126 @@ class auto_index_gt {
|
|
|
635
663
|
return result;
|
|
636
664
|
}
|
|
637
665
|
|
|
638
|
-
static casts_t
|
|
666
|
+
static casts_t make_casts_(accuracy_t accuracy) {
|
|
639
667
|
switch (accuracy) {
|
|
640
|
-
case accuracy_t::f64_k: return
|
|
641
|
-
case accuracy_t::f32_k: return
|
|
642
|
-
case accuracy_t::f16_k: return
|
|
643
|
-
case accuracy_t::f8_k: return
|
|
668
|
+
case accuracy_t::f64_k: return make_casts_<f64_t>();
|
|
669
|
+
case accuracy_t::f32_k: return make_casts_<f32_t>();
|
|
670
|
+
case accuracy_t::f16_k: return make_casts_<f16_bits_t>();
|
|
671
|
+
case accuracy_t::f8_k: return make_casts_<f8_bits_t>();
|
|
644
672
|
default: return {};
|
|
645
673
|
}
|
|
646
674
|
}
|
|
647
675
|
|
|
648
676
|
template <typename scalar_at, typename typed_at> //
|
|
649
|
-
static punned_stateful_metric_t
|
|
650
|
-
return [=](byte_t const* a, byte_t const* b, std::size_t bytes, std::size_t)
|
|
677
|
+
static punned_stateful_metric_t pun_metric_(typed_at metric) {
|
|
678
|
+
return [=](byte_t const* a, byte_t const* b, std::size_t bytes, std::size_t) -> float {
|
|
651
679
|
return metric((scalar_at const*)a, (scalar_at const*)b, bytes / sizeof(scalar_at));
|
|
652
680
|
};
|
|
653
681
|
}
|
|
654
682
|
|
|
655
|
-
static metric_and_meta_t
|
|
683
|
+
static metric_and_meta_t ip_metric_f32_(std::size_t dimensions) {
|
|
656
684
|
(void)dimensions;
|
|
657
|
-
#if defined(
|
|
658
|
-
#if defined(
|
|
685
|
+
#if defined(USEARCH_USE_SIMSIMD)
|
|
686
|
+
#if defined(USEARCH_IS_X86)
|
|
659
687
|
if (dimensions % 4 == 0)
|
|
660
688
|
return {
|
|
661
|
-
|
|
689
|
+
pun_metric_<simsimd_f32_t>([](simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
662
690
|
return 1.f - simsimd_dot_f32x4avx2(a, b, d);
|
|
663
691
|
}),
|
|
664
692
|
isa_t::avx2_k,
|
|
665
693
|
};
|
|
666
|
-
#elif defined(
|
|
694
|
+
#elif defined(USEARCH_IS_ARM)
|
|
667
695
|
if (supports_arm_sve())
|
|
668
696
|
return {
|
|
669
|
-
|
|
697
|
+
pun_metric_<simsimd_f32_t>([](simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
670
698
|
return 1.f - simsimd_dot_f32sve(a, b, d);
|
|
671
699
|
}),
|
|
672
700
|
isa_t::sve_k,
|
|
673
701
|
};
|
|
674
702
|
if (dimensions % 4 == 0)
|
|
675
703
|
return {
|
|
676
|
-
|
|
704
|
+
pun_metric_<simsimd_f32_t>([](simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
677
705
|
return 1.f - simsimd_dot_f32x4neon(a, b, d);
|
|
678
706
|
}),
|
|
679
707
|
isa_t::neon_k,
|
|
680
708
|
};
|
|
681
709
|
#endif
|
|
682
710
|
#endif
|
|
683
|
-
return {
|
|
711
|
+
return {pun_metric_<f32_t>(ip_gt<f32_t>{}), isa_t::auto_k};
|
|
684
712
|
}
|
|
685
713
|
|
|
686
|
-
static metric_and_meta_t
|
|
714
|
+
static metric_and_meta_t cos_metric_f16_(std::size_t dimensions) {
|
|
687
715
|
(void)dimensions;
|
|
688
|
-
#if defined(
|
|
689
|
-
#if defined(
|
|
716
|
+
#if defined(USEARCH_USE_SIMSIMD)
|
|
717
|
+
#if defined(USEARCH_IS_X86)
|
|
690
718
|
if (dimensions % 16 == 0)
|
|
691
719
|
return {
|
|
692
|
-
|
|
720
|
+
pun_metric_<simsimd_f16_t>([](simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
693
721
|
return 1.f - simsimd_cos_f16x16avx512(a, b, d);
|
|
694
722
|
}),
|
|
695
723
|
isa_t::avx512_k,
|
|
696
724
|
};
|
|
697
|
-
#elif defined(
|
|
725
|
+
#elif defined(USEARCH_IS_ARM)
|
|
698
726
|
if (dimensions % 4 == 0)
|
|
699
727
|
return {
|
|
700
|
-
|
|
728
|
+
pun_metric_<simsimd_f16_t>([](simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
701
729
|
return 1.f - simsimd_cos_f16x4neon(a, b, d);
|
|
702
730
|
}),
|
|
703
731
|
isa_t::neon_k,
|
|
704
732
|
};
|
|
705
733
|
#endif
|
|
706
734
|
#endif
|
|
707
|
-
return {
|
|
735
|
+
return {pun_metric_<f16_bits_t>(cos_gt<f16_bits_t, f32_t>{}), isa_t::auto_k};
|
|
708
736
|
}
|
|
709
737
|
|
|
710
|
-
static metric_and_meta_t
|
|
738
|
+
static metric_and_meta_t ip_metric_(std::size_t dimensions, accuracy_t accuracy) {
|
|
711
739
|
switch (accuracy) {
|
|
712
740
|
case accuracy_t::f32_k:
|
|
713
741
|
// The two most common numeric types for the most common metric have optimized versions
|
|
714
|
-
return
|
|
742
|
+
return ip_metric_f32_(dimensions);
|
|
715
743
|
case accuracy_t::f16_k:
|
|
716
744
|
// Dot-product accumulates error, Cosine-distance normalizes it
|
|
717
|
-
return
|
|
745
|
+
return cos_metric_f16_(dimensions);
|
|
718
746
|
|
|
719
|
-
case accuracy_t::f8_k: return {
|
|
720
|
-
case accuracy_t::f64_k: return {
|
|
747
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(cos_f8_t{}), isa_t::auto_k};
|
|
748
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(ip_gt<f64_t>{}), isa_t::auto_k};
|
|
721
749
|
default: return {};
|
|
722
750
|
}
|
|
723
751
|
}
|
|
724
752
|
|
|
725
|
-
static metric_and_meta_t
|
|
753
|
+
static metric_and_meta_t l2_metric_(std::size_t, accuracy_t accuracy) {
|
|
726
754
|
switch (accuracy) {
|
|
727
|
-
case accuracy_t::f8_k: return {
|
|
728
|
-
case accuracy_t::f16_k: return {
|
|
729
|
-
case accuracy_t::f32_k: return {
|
|
730
|
-
case accuracy_t::f64_k: return {
|
|
755
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(l2sq_f8_t{}), isa_t::auto_k};
|
|
756
|
+
case accuracy_t::f16_k: return {pun_metric_<f16_bits_t>(l2sq_gt<f16_bits_t, f32_t>{}), isa_t::auto_k};
|
|
757
|
+
case accuracy_t::f32_k: return {pun_metric_<f32_t>(l2sq_gt<f32_t>{}), isa_t::auto_k};
|
|
758
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(l2sq_gt<f64_t>{}), isa_t::auto_k};
|
|
731
759
|
default: return {};
|
|
732
760
|
}
|
|
733
761
|
}
|
|
734
762
|
|
|
735
|
-
static metric_and_meta_t
|
|
763
|
+
static metric_and_meta_t cos_metric_(std::size_t dimensions, accuracy_t accuracy) {
|
|
736
764
|
switch (accuracy) {
|
|
737
|
-
case accuracy_t::f8_k: return {
|
|
738
|
-
case accuracy_t::f16_k: return
|
|
739
|
-
case accuracy_t::f32_k: return {
|
|
740
|
-
case accuracy_t::f64_k: return {
|
|
765
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(cos_f8_t{}), isa_t::auto_k};
|
|
766
|
+
case accuracy_t::f16_k: return cos_metric_f16_(dimensions);
|
|
767
|
+
case accuracy_t::f32_k: return {pun_metric_<f32_t>(cos_gt<f32_t>{}), isa_t::auto_k};
|
|
768
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(cos_gt<f64_t>{}), isa_t::auto_k};
|
|
741
769
|
default: return {};
|
|
742
770
|
}
|
|
743
771
|
}
|
|
744
772
|
|
|
745
|
-
static metric_and_meta_t
|
|
773
|
+
static metric_and_meta_t haversine_metric_(accuracy_t accuracy) {
|
|
746
774
|
switch (accuracy) {
|
|
747
|
-
case accuracy_t::f8_k: return {
|
|
748
|
-
case accuracy_t::f16_k: return {
|
|
749
|
-
case accuracy_t::f32_k: return {
|
|
750
|
-
case accuracy_t::f64_k: return {
|
|
775
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(haversine_gt<f8_bits_t, f32_t>{}), isa_t::auto_k};
|
|
776
|
+
case accuracy_t::f16_k: return {pun_metric_<f16_bits_t>(haversine_gt<f16_bits_t, f32_t>{}), isa_t::auto_k};
|
|
777
|
+
case accuracy_t::f32_k: return {pun_metric_<f32_t>(haversine_gt<f32_t>{}), isa_t::auto_k};
|
|
778
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(haversine_gt<f64_t>{}), isa_t::auto_k};
|
|
751
779
|
default: return {};
|
|
752
780
|
}
|
|
753
781
|
}
|
|
754
782
|
};
|
|
755
783
|
|
|
756
|
-
using
|
|
757
|
-
using
|
|
784
|
+
using punned_small_t = punned_gt<std::int64_t, std::uint32_t>;
|
|
785
|
+
using punned_big_t = punned_gt<uuid_t, uint40_t>;
|
|
758
786
|
|
|
759
787
|
} // namespace usearch
|
|
760
788
|
} // namespace unum
|