usearch 0.2.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.clang-tidy +60 -0
- package/.github/ISSUE_TEMPLATE/bug_report.yml +90 -0
- package/.github/ISSUE_TEMPLATE/config.yml +5 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +64 -0
- package/.github/workflows/release.yml +47 -20
- package/.github/workflows/update_version.sh +1 -0
- package/.vscode/launch.json +22 -0
- package/.vscode/settings.json +9 -1
- package/.vscode/tasks.json +4 -4
- package/CITATION.cff +11 -0
- package/CMakeLists.txt +108 -95
- package/CODE_OF_CONDUCT.md +128 -0
- package/CONTRIBUTING.md +12 -0
- package/Cargo.toml +1 -1
- package/Dockerfile +14 -0
- package/README.md +21 -12
- package/VERSION +1 -1
- package/binding.gyp +13 -9
- package/build.rs +2 -1
- package/cmake/config.cmake.in +15 -0
- package/cmake/pkg-config.pc.in +4 -0
- package/cmake/usearchConfigVersion.cmake.in +20 -0
- package/conanfile.py +1 -1
- package/docs/{Benchmarking.md → benchmarks.md} +11 -2
- package/docs/benchmarks.rst +5 -0
- package/docs/{Compilation.md → compilation.md} +54 -27
- package/docs/compilation.rst +5 -0
- package/docs/conf.dox +2668 -0
- package/docs/index.rst +7 -0
- package/include/usearch/usearch.hpp +833 -439
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +17 -16
- package/javascript/lib.cpp +94 -20
- package/javascript/test.js +1 -1
- package/objc/USearchObjective.mm +47 -30
- package/objc/include/USearchObjective.h +18 -26
- package/package.json +1 -1
- package/pyproject.toml +3 -2
- package/python/lib.cpp +216 -130
- package/python/{bench.py → scripts/bench.py} +6 -1
- package/python/scripts/bench_params.py +112 -0
- package/python/{test.py → scripts/test.py} +32 -13
- package/python/usearch/client.py +100 -14
- package/python/usearch/server.py +104 -23
- package/rust/lib.cpp +20 -15
- package/rust/lib.hpp +5 -5
- package/rust/lib.rs +3 -3
- package/setup.py +7 -0
- package/src/CMakeLists.txt +107 -0
- package/src/bench.cpp +54 -19
- package/src/{advanced.hpp → punned.hpp} +235 -203
- package/src/test.cpp +62 -0
- package/swift/Index+Sugar.swift +1 -0
- package/swift/Test.swift +2 -1
- package/wolfram/lib.cpp +14 -14
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
#pragma once
|
|
2
|
-
|
|
3
|
-
#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
|
|
4
|
-
#define _USE_MATH_DEFINES
|
|
5
|
-
|
|
6
|
-
#define WINDOWS
|
|
7
|
-
#endif
|
|
8
|
-
|
|
9
2
|
#include <stdlib.h> // `aligned_alloc`
|
|
10
3
|
|
|
11
4
|
#include <functional> // `std::function`
|
|
@@ -14,25 +7,26 @@
|
|
|
14
7
|
#include <thread> // `std::thread`
|
|
15
8
|
#include <vector> // `std::vector`
|
|
16
9
|
|
|
17
|
-
#
|
|
10
|
+
#include <fp16/fp16.h>
|
|
11
|
+
|
|
12
|
+
#include <usearch/usearch.hpp>
|
|
13
|
+
|
|
14
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
18
15
|
#include <omp.h> // `omp_get_num_threads()`
|
|
19
16
|
#endif
|
|
20
17
|
|
|
21
|
-
#if
|
|
18
|
+
#if defined(USEARCH_IS_LINUX)
|
|
22
19
|
#include <sys/auxv.h>
|
|
23
20
|
#endif
|
|
24
21
|
|
|
25
|
-
#if defined(
|
|
22
|
+
#if defined(USEARCH_IS_ARM)
|
|
26
23
|
#include <arm_fp16.h>
|
|
27
24
|
#endif
|
|
28
25
|
|
|
29
|
-
#
|
|
30
|
-
#if defined(USEARCH_USE_SIMD)
|
|
26
|
+
#if defined(USEARCH_USE_SIMSIMD)
|
|
31
27
|
#include <simsimd/simsimd.h>
|
|
32
28
|
#endif
|
|
33
29
|
|
|
34
|
-
#include <usearch/usearch.hpp>
|
|
35
|
-
|
|
36
30
|
namespace unum {
|
|
37
31
|
namespace usearch {
|
|
38
32
|
|
|
@@ -41,8 +35,7 @@ using punned_distance_t = float;
|
|
|
41
35
|
using punned_metric_t = punned_distance_t (*)(byte_t const*, byte_t const*, std::size_t, std::size_t);
|
|
42
36
|
|
|
43
37
|
template <typename metric_at>
|
|
44
|
-
|
|
45
|
-
byte_t const* a, byte_t const* b, std::size_t a_bytes, std::size_t b_bytes) noexcept {
|
|
38
|
+
punned_distance_t punned_metric(byte_t const* a, byte_t const* b, std::size_t a_bytes, std::size_t b_bytes) {
|
|
46
39
|
using scalar_t = typename metric_at::scalar_t;
|
|
47
40
|
return metric_at{}((scalar_t const*)a, (scalar_t const*)b, a_bytes / sizeof(scalar_t), b_bytes / sizeof(scalar_t));
|
|
48
41
|
}
|
|
@@ -50,22 +43,20 @@ static punned_distance_t punned_metric( //
|
|
|
50
43
|
using punned_stateful_metric_t =
|
|
51
44
|
std::function<punned_distance_t(byte_t const*, byte_t const*, std::size_t, std::size_t)>;
|
|
52
45
|
|
|
53
|
-
template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp)
|
|
46
|
+
template <typename at, typename compare_at> inline at clamp(at v, at lo, at hi, compare_at comp) {
|
|
54
47
|
return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
|
|
55
48
|
}
|
|
56
|
-
template <typename at> inline at clamp(at v, at lo, at hi)
|
|
57
|
-
return usearch::clamp(v, lo, hi, std::less<at>{});
|
|
58
|
-
}
|
|
49
|
+
template <typename at> inline at clamp(at v, at lo, at hi) { return usearch::clamp(v, lo, hi, std::less<at>{}); }
|
|
59
50
|
|
|
60
51
|
class f8_bits_t;
|
|
61
52
|
class f16_bits_t;
|
|
62
53
|
|
|
63
|
-
inline float f16_to_f32(std::uint16_t u16)
|
|
54
|
+
inline float f16_to_f32(std::uint16_t u16) {
|
|
64
55
|
#if defined(__AVX512F__)
|
|
65
56
|
_Float16 f16;
|
|
66
57
|
std::memcpy(&f16, &u16, sizeof(std::uint16_t));
|
|
67
58
|
return float(f16);
|
|
68
|
-
#elif defined(
|
|
59
|
+
#elif defined(USEARCH_IS_ARM)
|
|
69
60
|
__fp16 f16;
|
|
70
61
|
std::memcpy(&f16, &u16, sizeof(std::uint16_t));
|
|
71
62
|
return float(f16);
|
|
@@ -74,13 +65,13 @@ inline float f16_to_f32(std::uint16_t u16) noexcept {
|
|
|
74
65
|
#endif
|
|
75
66
|
}
|
|
76
67
|
|
|
77
|
-
inline std::uint16_t f32_to_f16(float f32)
|
|
68
|
+
inline std::uint16_t f32_to_f16(float f32) {
|
|
78
69
|
#if defined(__AVX512F__)
|
|
79
70
|
_Float16 f16 = _Float16(f32);
|
|
80
71
|
std::uint16_t u16;
|
|
81
72
|
std::memcpy(&u16, &f16, sizeof(std::uint16_t));
|
|
82
73
|
return u16;
|
|
83
|
-
#elif defined(
|
|
74
|
+
#elif defined(USEARCH_IS_ARM)
|
|
84
75
|
__fp16 f16 = __fp16(f32);
|
|
85
76
|
std::uint16_t u16;
|
|
86
77
|
std::memcpy(&u16, &f16, sizeof(std::uint16_t));
|
|
@@ -94,47 +85,47 @@ class f16_bits_t {
|
|
|
94
85
|
std::uint16_t uint16_{};
|
|
95
86
|
|
|
96
87
|
public:
|
|
97
|
-
inline f16_bits_t()
|
|
88
|
+
inline f16_bits_t() : uint16_(0) {}
|
|
98
89
|
inline f16_bits_t(f16_bits_t&&) = default;
|
|
99
90
|
inline f16_bits_t& operator=(f16_bits_t&&) = default;
|
|
100
91
|
inline f16_bits_t(f16_bits_t const&) = default;
|
|
101
92
|
inline f16_bits_t& operator=(f16_bits_t const&) = default;
|
|
102
93
|
|
|
103
|
-
inline operator float() const
|
|
104
|
-
|
|
105
|
-
inline f16_bits_t(f8_bits_t)
|
|
106
|
-
inline f16_bits_t(float v)
|
|
107
|
-
inline f16_bits_t(double v)
|
|
108
|
-
|
|
109
|
-
inline f16_bits_t operator+(f16_bits_t other) const
|
|
110
|
-
inline f16_bits_t operator-(f16_bits_t other) const
|
|
111
|
-
inline f16_bits_t operator*(f16_bits_t other) const
|
|
112
|
-
inline f16_bits_t operator/(f16_bits_t other) const
|
|
113
|
-
inline f16_bits_t operator+(float other) const
|
|
114
|
-
inline f16_bits_t operator-(float other) const
|
|
115
|
-
inline f16_bits_t operator*(float other) const
|
|
116
|
-
inline f16_bits_t operator/(float other) const
|
|
117
|
-
inline f16_bits_t operator+(double other) const
|
|
118
|
-
inline f16_bits_t operator-(double other) const
|
|
119
|
-
inline f16_bits_t operator*(double other) const
|
|
120
|
-
inline f16_bits_t operator/(double other) const
|
|
121
|
-
|
|
122
|
-
inline f16_bits_t& operator+=(float v)
|
|
94
|
+
inline operator float() const { return f16_to_f32(uint16_); }
|
|
95
|
+
|
|
96
|
+
inline f16_bits_t(f8_bits_t);
|
|
97
|
+
inline f16_bits_t(float v) : uint16_(f32_to_f16(v)) {}
|
|
98
|
+
inline f16_bits_t(double v) : uint16_(f32_to_f16(v)) {}
|
|
99
|
+
|
|
100
|
+
inline f16_bits_t operator+(f16_bits_t other) const { return {float(*this) + float(other)}; }
|
|
101
|
+
inline f16_bits_t operator-(f16_bits_t other) const { return {float(*this) - float(other)}; }
|
|
102
|
+
inline f16_bits_t operator*(f16_bits_t other) const { return {float(*this) * float(other)}; }
|
|
103
|
+
inline f16_bits_t operator/(f16_bits_t other) const { return {float(*this) / float(other)}; }
|
|
104
|
+
inline f16_bits_t operator+(float other) const { return {float(*this) + other}; }
|
|
105
|
+
inline f16_bits_t operator-(float other) const { return {float(*this) - other}; }
|
|
106
|
+
inline f16_bits_t operator*(float other) const { return {float(*this) * other}; }
|
|
107
|
+
inline f16_bits_t operator/(float other) const { return {float(*this) / other}; }
|
|
108
|
+
inline f16_bits_t operator+(double other) const { return {float(*this) + other}; }
|
|
109
|
+
inline f16_bits_t operator-(double other) const { return {float(*this) - other}; }
|
|
110
|
+
inline f16_bits_t operator*(double other) const { return {float(*this) * other}; }
|
|
111
|
+
inline f16_bits_t operator/(double other) const { return {float(*this) / other}; }
|
|
112
|
+
|
|
113
|
+
inline f16_bits_t& operator+=(float v) {
|
|
123
114
|
uint16_ = f32_to_f16(v + f16_to_f32(uint16_));
|
|
124
115
|
return *this;
|
|
125
116
|
}
|
|
126
117
|
|
|
127
|
-
inline f16_bits_t& operator-=(float v)
|
|
118
|
+
inline f16_bits_t& operator-=(float v) {
|
|
128
119
|
uint16_ = f32_to_f16(v - f16_to_f32(uint16_));
|
|
129
120
|
return *this;
|
|
130
121
|
}
|
|
131
122
|
|
|
132
|
-
inline f16_bits_t& operator*=(float v)
|
|
123
|
+
inline f16_bits_t& operator*=(float v) {
|
|
133
124
|
uint16_ = f32_to_f16(v * f16_to_f32(uint16_));
|
|
134
125
|
return *this;
|
|
135
126
|
}
|
|
136
127
|
|
|
137
|
-
inline f16_bits_t& operator/=(float v)
|
|
128
|
+
inline f16_bits_t& operator/=(float v) {
|
|
138
129
|
uint16_ = f32_to_f16(v / f16_to_f32(uint16_));
|
|
139
130
|
return *this;
|
|
140
131
|
}
|
|
@@ -152,30 +143,30 @@ class f8_bits_t {
|
|
|
152
143
|
constexpr static std::int8_t min_k = -100;
|
|
153
144
|
constexpr static std::int8_t max_k = 100;
|
|
154
145
|
|
|
155
|
-
inline f8_bits_t()
|
|
146
|
+
inline f8_bits_t() : int8_(0) {}
|
|
156
147
|
inline f8_bits_t(f8_bits_t&&) = default;
|
|
157
148
|
inline f8_bits_t& operator=(f8_bits_t&&) = default;
|
|
158
149
|
inline f8_bits_t(f8_bits_t const&) = default;
|
|
159
150
|
inline f8_bits_t& operator=(f8_bits_t const&) = default;
|
|
160
151
|
|
|
161
|
-
inline operator float() const
|
|
162
|
-
inline operator f16_bits_t() const
|
|
163
|
-
inline operator double() const
|
|
164
|
-
inline explicit operator std::int8_t() const
|
|
165
|
-
inline explicit operator std::int16_t() const
|
|
166
|
-
inline explicit operator std::int32_t() const
|
|
167
|
-
inline explicit operator std::int64_t() const
|
|
152
|
+
inline operator float() const { return float(int8_) / divisor_k; }
|
|
153
|
+
inline operator f16_bits_t() const { return float(int8_) / divisor_k; }
|
|
154
|
+
inline operator double() const { return double(int8_) / divisor_k; }
|
|
155
|
+
inline explicit operator std::int8_t() const { return int8_; }
|
|
156
|
+
inline explicit operator std::int16_t() const { return int8_; }
|
|
157
|
+
inline explicit operator std::int32_t() const { return int8_; }
|
|
158
|
+
inline explicit operator std::int64_t() const { return int8_; }
|
|
168
159
|
|
|
169
|
-
inline f8_bits_t(float v)
|
|
160
|
+
inline f8_bits_t(float v)
|
|
170
161
|
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
171
|
-
inline f8_bits_t(double v)
|
|
162
|
+
inline f8_bits_t(double v)
|
|
172
163
|
: int8_(usearch::clamp<std::int8_t>(static_cast<std::int8_t>(v * divisor_k), min_k, max_k)) {}
|
|
173
164
|
};
|
|
174
165
|
|
|
175
|
-
inline f16_bits_t::f16_bits_t(f8_bits_t v)
|
|
166
|
+
inline f16_bits_t::f16_bits_t(f8_bits_t v) : f16_bits_t(float(v)) {}
|
|
176
167
|
|
|
177
168
|
struct cos_f8_t {
|
|
178
|
-
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const
|
|
169
|
+
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const {
|
|
179
170
|
std::int32_t ab{}, a2{}, b2{};
|
|
180
171
|
for (std::size_t i = 0; i != n; i++) {
|
|
181
172
|
std::int16_t ai{a[i]};
|
|
@@ -189,7 +180,7 @@ struct cos_f8_t {
|
|
|
189
180
|
};
|
|
190
181
|
|
|
191
182
|
struct l2sq_f8_t {
|
|
192
|
-
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const
|
|
183
|
+
punned_distance_t operator()(f8_bits_t const* a, f8_bits_t const* b, std::size_t n) const {
|
|
193
184
|
std::int32_t ab_deltas_sq{};
|
|
194
185
|
for (std::size_t i = 0; i != n; i++)
|
|
195
186
|
ab_deltas_sq += square(std::int16_t(a[i]) - std::int16_t(b[i]));
|
|
@@ -212,7 +203,7 @@ void multithreaded(std::size_t threads, std::size_t tasks, callback_at&& callbac
|
|
|
212
203
|
return;
|
|
213
204
|
}
|
|
214
205
|
|
|
215
|
-
#if defined(
|
|
206
|
+
#if defined(USEARCH_USE_OPENMP)
|
|
216
207
|
omp_set_num_threads(threads);
|
|
217
208
|
#pragma omp parallel for schedule(dynamic)
|
|
218
209
|
for (std::size_t i = 0; i < tasks; ++i)
|
|
@@ -246,13 +237,13 @@ class aligned_allocator_gt {
|
|
|
246
237
|
using other = aligned_allocator_gt<other_element_at>;
|
|
247
238
|
};
|
|
248
239
|
|
|
249
|
-
pointer allocate(size_type length) const
|
|
240
|
+
pointer allocate(size_type length) const {
|
|
250
241
|
void* result = nullptr;
|
|
251
242
|
int status = posix_memalign(&result, alignment_ak, ceil2(length * sizeof(value_type)));
|
|
252
243
|
return status == 0 ? (pointer)result : nullptr;
|
|
253
244
|
}
|
|
254
245
|
|
|
255
|
-
void deallocate(pointer begin, size_type) const
|
|
246
|
+
void deallocate(pointer begin, size_type) const { std::free(begin); }
|
|
256
247
|
};
|
|
257
248
|
|
|
258
249
|
using aligned_allocator_t = aligned_allocator_gt<>;
|
|
@@ -264,7 +255,7 @@ enum class accuracy_t {
|
|
|
264
255
|
f8_k,
|
|
265
256
|
};
|
|
266
257
|
|
|
267
|
-
inline char const* accuracy_name(accuracy_t accuracy)
|
|
258
|
+
inline char const* accuracy_name(accuracy_t accuracy) {
|
|
268
259
|
switch (accuracy) {
|
|
269
260
|
case accuracy_t::f32_k: return "f32";
|
|
270
261
|
case accuracy_t::f16_k: return "f16";
|
|
@@ -274,7 +265,7 @@ inline char const* accuracy_name(accuracy_t accuracy) noexcept {
|
|
|
274
265
|
}
|
|
275
266
|
}
|
|
276
267
|
|
|
277
|
-
inline bool str_equals(char const* begin, std::size_t len, char const* other_begin)
|
|
268
|
+
inline bool str_equals(char const* begin, std::size_t len, char const* other_begin) {
|
|
278
269
|
std::size_t other_len = strlen(other_begin);
|
|
279
270
|
return len == other_len && strncmp(begin, other_begin, len) == 0;
|
|
280
271
|
}
|
|
@@ -298,10 +289,10 @@ template <typename index_at>
|
|
|
298
289
|
inline index_at index_from_name( //
|
|
299
290
|
char const* name, std::size_t len, std::size_t dimensions, accuracy_t accuracy, config_t const& config) {
|
|
300
291
|
|
|
301
|
-
if (str_equals(name, len, "
|
|
292
|
+
if (str_equals(name, len, "l2sq") || str_equals(name, len, "euclidean_sq")) {
|
|
302
293
|
if (dimensions == 0)
|
|
303
294
|
throw std::invalid_argument("The number of dimensions must be positive");
|
|
304
|
-
return index_at::
|
|
295
|
+
return index_at::l2sq(dimensions, accuracy, config);
|
|
305
296
|
} else if (str_equals(name, len, "ip") || str_equals(name, len, "inner") || str_equals(name, len, "dot")) {
|
|
306
297
|
if (dimensions == 0)
|
|
307
298
|
throw std::invalid_argument("The number of dimensions must be positive");
|
|
@@ -315,7 +306,7 @@ inline index_at index_from_name( //
|
|
|
315
306
|
throw std::invalid_argument("The number of dimensions must be equal to two");
|
|
316
307
|
return index_at::haversine(accuracy, config);
|
|
317
308
|
} else
|
|
318
|
-
throw std::invalid_argument("Unknown distance, choose:
|
|
309
|
+
throw std::invalid_argument("Unknown distance, choose: l2sq, ip, cos, hamming, jaccard");
|
|
319
310
|
return {};
|
|
320
311
|
}
|
|
321
312
|
|
|
@@ -327,7 +318,7 @@ enum class isa_t {
|
|
|
327
318
|
avx512_k,
|
|
328
319
|
};
|
|
329
320
|
|
|
330
|
-
inline char const* isa_name(isa_t isa)
|
|
321
|
+
inline char const* isa_name(isa_t isa) {
|
|
331
322
|
switch (isa) {
|
|
332
323
|
case isa_t::auto_k: return "auto";
|
|
333
324
|
case isa_t::neon_k: return "neon";
|
|
@@ -338,7 +329,7 @@ inline char const* isa_name(isa_t isa) noexcept {
|
|
|
338
329
|
}
|
|
339
330
|
}
|
|
340
331
|
|
|
341
|
-
inline std::size_t bytes_per_scalar(accuracy_t accuracy)
|
|
332
|
+
inline std::size_t bytes_per_scalar(accuracy_t accuracy) {
|
|
342
333
|
switch (accuracy) {
|
|
343
334
|
case accuracy_t::f32_k: return 4;
|
|
344
335
|
case accuracy_t::f16_k: return 2;
|
|
@@ -349,10 +340,10 @@ inline std::size_t bytes_per_scalar(accuracy_t accuracy) noexcept {
|
|
|
349
340
|
}
|
|
350
341
|
|
|
351
342
|
inline bool supports_arm_sve() {
|
|
352
|
-
#if defined(
|
|
353
|
-
#if
|
|
354
|
-
unsigned long
|
|
355
|
-
if (
|
|
343
|
+
#if defined(USEARCH_IS_ARM)
|
|
344
|
+
#if defined(USEARCH_IS_LINUX)
|
|
345
|
+
unsigned long capabilities = getauxval(AT_HWCAP);
|
|
346
|
+
if (capabilities & HWCAP_SVE)
|
|
356
347
|
return true;
|
|
357
348
|
#endif
|
|
358
349
|
#endif
|
|
@@ -360,7 +351,7 @@ inline bool supports_arm_sve() {
|
|
|
360
351
|
}
|
|
361
352
|
|
|
362
353
|
template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
|
|
363
|
-
bool operator()(byte_t const* input, std::size_t bytes_in_input, byte_t* output)
|
|
354
|
+
bool operator()(byte_t const* input, std::size_t bytes_in_input, byte_t* output) const {
|
|
364
355
|
from_scalar_at const* typed_input = reinterpret_cast<from_scalar_at const*>(input);
|
|
365
356
|
to_scalar_at* typed_output = reinterpret_cast<to_scalar_at*>(output);
|
|
366
357
|
std::transform( //
|
|
@@ -371,15 +362,19 @@ template <typename from_scalar_at, typename to_scalar_at> struct cast_gt {
|
|
|
371
362
|
};
|
|
372
363
|
|
|
373
364
|
template <> struct cast_gt<f32_t, f32_t> {
|
|
374
|
-
bool operator()(byte_t const*, std::size_t, byte_t*)
|
|
365
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
375
366
|
};
|
|
376
367
|
|
|
377
368
|
template <> struct cast_gt<f64_t, f64_t> {
|
|
378
|
-
bool operator()(byte_t const*, std::size_t, byte_t*)
|
|
369
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
379
370
|
};
|
|
380
371
|
|
|
381
372
|
template <> struct cast_gt<f16_bits_t, f16_bits_t> {
|
|
382
|
-
bool operator()(byte_t const*, std::size_t, byte_t*)
|
|
373
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
template <> struct cast_gt<f8_bits_t, f8_bits_t> {
|
|
377
|
+
bool operator()(byte_t const*, std::size_t, byte_t*) const { return false; }
|
|
383
378
|
};
|
|
384
379
|
|
|
385
380
|
/**
|
|
@@ -387,15 +382,12 @@ template <> struct cast_gt<f16_bits_t, f16_bits_t> {
|
|
|
387
382
|
* vectors with automatic down-casting and hardware acceleration.
|
|
388
383
|
*/
|
|
389
384
|
template <typename label_at = std::int64_t, typename id_at = std::uint32_t> //
|
|
390
|
-
class
|
|
385
|
+
class punned_gt {
|
|
391
386
|
public:
|
|
392
387
|
using label_t = label_at;
|
|
393
388
|
using id_t = id_at;
|
|
394
389
|
using distance_t = punned_distance_t;
|
|
395
390
|
|
|
396
|
-
using f32_t = float;
|
|
397
|
-
using f64_t = double;
|
|
398
|
-
|
|
399
391
|
private:
|
|
400
392
|
/// @brief Schema: input buffer, bytes in input buffer, output buffer.
|
|
401
393
|
using cast_t = std::function<bool(byte_t const*, std::size_t, byte_t*)>;
|
|
@@ -422,22 +414,24 @@ class auto_index_gt {
|
|
|
422
414
|
} casts_;
|
|
423
415
|
|
|
424
416
|
punned_stateful_metric_t root_metric_;
|
|
425
|
-
config_t root_config_;
|
|
426
417
|
|
|
427
418
|
mutable std::vector<std::size_t> available_threads_;
|
|
428
419
|
mutable std::mutex available_threads_mutex_;
|
|
429
420
|
|
|
430
421
|
public:
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
422
|
+
using search_results_t = typename index_t::search_results_t;
|
|
423
|
+
using add_result_t = typename index_t::add_result_t;
|
|
424
|
+
|
|
425
|
+
punned_gt() = default;
|
|
426
|
+
punned_gt(punned_gt&& other) { swap(other); }
|
|
427
|
+
punned_gt& operator=(punned_gt&& other) {
|
|
434
428
|
swap(other);
|
|
435
429
|
return *this;
|
|
436
430
|
}
|
|
437
431
|
|
|
438
|
-
~
|
|
432
|
+
~punned_gt() { aligned_index_free_(index_); }
|
|
439
433
|
|
|
440
|
-
void swap(
|
|
434
|
+
void swap(punned_gt& other) {
|
|
441
435
|
std::swap(dimensions_, other.dimensions_);
|
|
442
436
|
std::swap(casted_vector_bytes_, other.casted_vector_bytes_);
|
|
443
437
|
std::swap(accuracy_, other.accuracy_);
|
|
@@ -446,22 +440,20 @@ class auto_index_gt {
|
|
|
446
440
|
std::swap(cast_buffer_, other.cast_buffer_);
|
|
447
441
|
std::swap(casts_, other.casts_);
|
|
448
442
|
std::swap(root_metric_, other.root_metric_);
|
|
449
|
-
std::swap(root_config_, other.root_config_);
|
|
450
443
|
std::swap(available_threads_, other.available_threads_);
|
|
451
444
|
}
|
|
452
445
|
|
|
453
|
-
|
|
454
|
-
std::size_t connectivity() const noexcept { return index_->connectivity(); }
|
|
455
|
-
std::size_t size() const noexcept { return index_->size(); }
|
|
456
|
-
std::size_t capacity() const noexcept { return index_->capacity(); }
|
|
457
|
-
config_t const& config() const noexcept { return root_config_; }
|
|
458
|
-
void clear() noexcept { return index_->clear(); }
|
|
446
|
+
static config_t optimize(config_t config) noexcept { return index_t::optimize(config); }
|
|
459
447
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
std::size_t
|
|
463
|
-
|
|
464
|
-
}
|
|
448
|
+
std::size_t dimensions() const { return dimensions_; }
|
|
449
|
+
std::size_t connectivity() const { return index_->connectivity(); }
|
|
450
|
+
std::size_t size() const { return index_->size(); }
|
|
451
|
+
std::size_t capacity() const { return index_->capacity(); }
|
|
452
|
+
config_t const& config() const { return index_->config(); }
|
|
453
|
+
void clear() { return index_->clear(); }
|
|
454
|
+
|
|
455
|
+
accuracy_t accuracy() const { return accuracy_; }
|
|
456
|
+
isa_t acceleration() const { return acceleration_; }
|
|
465
457
|
|
|
466
458
|
void save(char const* path) const { index_->save(path); }
|
|
467
459
|
void load(char const* path) { index_->load(path); }
|
|
@@ -469,36 +461,50 @@ class auto_index_gt {
|
|
|
469
461
|
void reserve(std::size_t capacity) { index_->reserve(capacity); }
|
|
470
462
|
|
|
471
463
|
// clang-format off
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
464
|
+
add_result_t add(label_t label, f8_bits_t const* vector) { return add_(label, vector, casts_.from_f8); }
|
|
465
|
+
add_result_t add(label_t label, f16_bits_t const* vector) { return add_(label, vector, casts_.from_f16); }
|
|
466
|
+
add_result_t add(label_t label, f32_t const* vector) { return add_(label, vector, casts_.from_f32); }
|
|
467
|
+
add_result_t add(label_t label, f64_t const* vector) { return add_(label, vector, casts_.from_f64); }
|
|
468
|
+
|
|
469
|
+
add_result_t add(label_t label, f8_bits_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f8); }
|
|
470
|
+
add_result_t add(label_t label, f16_bits_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f16); }
|
|
471
|
+
add_result_t add(label_t label, f32_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f32); }
|
|
472
|
+
add_result_t add(label_t label, f64_t const* vector, add_config_t config) { return add_(label, vector, config, casts_.from_f64); }
|
|
473
|
+
|
|
474
|
+
search_results_t search(f8_bits_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f8); }
|
|
475
|
+
search_results_t search(f16_bits_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f16); }
|
|
476
|
+
search_results_t search(f32_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f32); }
|
|
477
|
+
search_results_t search(f64_t const* vector, std::size_t wanted) const { return search_(vector, wanted, casts_.from_f64); }
|
|
478
|
+
|
|
479
|
+
search_results_t search(f8_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f8); }
|
|
480
|
+
search_results_t search(f16_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f16); }
|
|
481
|
+
search_results_t search(f32_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f32); }
|
|
482
|
+
search_results_t search(f64_t const* vector, std::size_t wanted, search_config_t config) const { return search_(vector, wanted, config, casts_.from_f64); }
|
|
483
|
+
|
|
484
|
+
search_results_t search_around(label_t hint, f8_bits_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f8); }
|
|
485
|
+
search_results_t search_around(label_t hint, f16_bits_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f16); }
|
|
486
|
+
search_results_t search_around(label_t hint, f32_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f32); }
|
|
487
|
+
search_results_t search_around(label_t hint, f64_t const* vector, std::size_t wanted) const { return search_around_(hint, vector, wanted, casts_.from_f64); }
|
|
488
|
+
|
|
489
|
+
search_results_t search_around(label_t hint, f8_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f8); }
|
|
490
|
+
search_results_t search_around(label_t hint, f16_bits_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f16); }
|
|
491
|
+
search_results_t search_around(label_t hint, f32_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f32); }
|
|
492
|
+
search_results_t search_around(label_t hint, f64_t const* vector, std::size_t wanted, search_config_t config) const { return search_around_(hint, vector, wanted, config, casts_.from_f64); }
|
|
493
|
+
|
|
494
|
+
static punned_gt ip(std::size_t dimensions, accuracy_t accuracy = accuracy_t::f16_k, config_t config = {}) { return make_(dimensions, accuracy, ip_metric_(dimensions, accuracy), make_casts_(accuracy), config); }
|
|
495
|
+
static punned_gt l2sq(std::size_t dimensions, accuracy_t accuracy = accuracy_t::f16_k, config_t config = {}) { return make_(dimensions, accuracy, l2_metric_(dimensions, accuracy), make_casts_(accuracy), config); }
|
|
496
|
+
static punned_gt cos(std::size_t dimensions, accuracy_t accuracy = accuracy_t::f32_k, config_t config = {}) { return make_(dimensions, accuracy, cos_metric_(dimensions, accuracy), make_casts_(accuracy), config); }
|
|
497
|
+
static punned_gt haversine(accuracy_t accuracy = accuracy_t::f32_k, config_t config = {}) { return make_(2, accuracy, haversine_metric_(accuracy), make_casts_(accuracy), config); }
|
|
492
498
|
// clang-format on
|
|
493
499
|
|
|
494
|
-
static
|
|
500
|
+
static punned_gt udf( //
|
|
495
501
|
std::size_t dimensions, punned_stateful_metric_t metric, //
|
|
496
502
|
accuracy_t accuracy = accuracy_t::f32_k, config_t config = {}) {
|
|
497
|
-
return
|
|
503
|
+
return make_(dimensions, accuracy, {metric, isa_t::auto_k}, make_casts_(accuracy), config);
|
|
498
504
|
}
|
|
499
505
|
|
|
500
|
-
|
|
501
|
-
|
|
506
|
+
punned_gt fork() const {
|
|
507
|
+
punned_gt result;
|
|
502
508
|
|
|
503
509
|
result.dimensions_ = dimensions_;
|
|
504
510
|
result.accuracy_ = accuracy_;
|
|
@@ -508,33 +514,32 @@ class auto_index_gt {
|
|
|
508
514
|
result.casts_ = casts_;
|
|
509
515
|
|
|
510
516
|
result.root_metric_ = root_metric_;
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
new (raw) index_t(root_config_, root_metric_);
|
|
517
|
+
index_t* raw = aligned_index_alloc_();
|
|
518
|
+
new (raw) index_t(config(), root_metric_);
|
|
514
519
|
result.index_ = raw;
|
|
515
520
|
|
|
516
521
|
return result;
|
|
517
522
|
}
|
|
518
523
|
|
|
519
524
|
private:
|
|
520
|
-
static index_t*
|
|
521
|
-
#if defined(
|
|
525
|
+
static index_t* aligned_index_alloc_() {
|
|
526
|
+
#if defined(USEARCH_IS_WINDOWS)
|
|
522
527
|
return (index_t*)_aligned_malloc(64, 64 * divide_round_up<64>(sizeof(index_t)));
|
|
523
528
|
#else
|
|
524
529
|
return (index_t*)aligned_alloc(64, 64 * divide_round_up<64>(sizeof(index_t)));
|
|
525
530
|
#endif
|
|
526
531
|
}
|
|
527
532
|
|
|
528
|
-
static void
|
|
533
|
+
static void aligned_index_free_(index_t* raw) { free(raw); }
|
|
529
534
|
|
|
530
535
|
struct thread_lock_t {
|
|
531
|
-
|
|
536
|
+
punned_gt const& parent;
|
|
532
537
|
std::size_t thread_id;
|
|
533
538
|
|
|
534
|
-
~thread_lock_t() { parent.
|
|
539
|
+
~thread_lock_t() { parent.thread_unlock_(thread_id); }
|
|
535
540
|
};
|
|
536
541
|
|
|
537
|
-
thread_lock_t
|
|
542
|
+
thread_lock_t thread_lock_() const {
|
|
538
543
|
available_threads_mutex_.lock();
|
|
539
544
|
std::size_t thread_id = available_threads_.back();
|
|
540
545
|
available_threads_.pop_back();
|
|
@@ -542,57 +547,85 @@ class auto_index_gt {
|
|
|
542
547
|
return {*this, thread_id};
|
|
543
548
|
}
|
|
544
549
|
|
|
545
|
-
void
|
|
550
|
+
void thread_unlock_(std::size_t thread_id) const {
|
|
546
551
|
available_threads_mutex_.lock();
|
|
547
552
|
available_threads_.push_back(thread_id);
|
|
548
553
|
available_threads_mutex_.unlock();
|
|
549
554
|
}
|
|
550
555
|
|
|
551
556
|
template <typename scalar_at>
|
|
552
|
-
|
|
557
|
+
add_result_t add_(label_t label, scalar_at const* vector, add_config_t config, cast_t const& cast) {
|
|
553
558
|
byte_t const* vector_data = reinterpret_cast<byte_t const*>(vector);
|
|
554
559
|
std::size_t vector_bytes = dimensions_ * sizeof(scalar_at);
|
|
555
560
|
|
|
556
|
-
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * thread;
|
|
557
|
-
bool casted = cast(vector_data,
|
|
561
|
+
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * config.thread;
|
|
562
|
+
bool casted = cast(vector_data, vector_bytes, casted_data);
|
|
558
563
|
if (casted)
|
|
559
|
-
vector_data = casted_data, vector_bytes = casted_vector_bytes_,
|
|
564
|
+
vector_data = casted_data, vector_bytes = casted_vector_bytes_, config.store_vector = true;
|
|
560
565
|
|
|
561
|
-
index_->add(label, {vector_data, vector_bytes},
|
|
566
|
+
return index_->add(label, {vector_data, vector_bytes}, config);
|
|
562
567
|
}
|
|
563
568
|
|
|
564
569
|
template <typename scalar_at>
|
|
565
|
-
|
|
570
|
+
search_results_t search_( //
|
|
566
571
|
scalar_at const* vector, std::size_t wanted, //
|
|
567
|
-
|
|
568
|
-
|
|
572
|
+
search_config_t config, cast_t const& cast) const {
|
|
573
|
+
|
|
574
|
+
byte_t const* vector_data = reinterpret_cast<byte_t const*>(vector);
|
|
575
|
+
std::size_t vector_bytes = dimensions_ * sizeof(scalar_at);
|
|
576
|
+
|
|
577
|
+
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * config.thread;
|
|
578
|
+
bool casted = cast(vector_data, vector_bytes, casted_data);
|
|
579
|
+
if (casted)
|
|
580
|
+
vector_data = casted_data, vector_bytes = casted_vector_bytes_;
|
|
581
|
+
|
|
582
|
+
return index_->search({vector_data, vector_bytes}, wanted, config);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
template <typename scalar_at>
|
|
586
|
+
search_results_t search_around_( //
|
|
587
|
+
label_t hint, scalar_at const* vector, std::size_t wanted, //
|
|
588
|
+
search_config_t config, cast_t const& cast) const {
|
|
569
589
|
|
|
570
590
|
byte_t const* vector_data = reinterpret_cast<byte_t const*>(vector);
|
|
571
591
|
std::size_t vector_bytes = dimensions_ * sizeof(scalar_at);
|
|
572
592
|
|
|
573
|
-
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * thread;
|
|
574
|
-
bool casted = cast(vector_data,
|
|
593
|
+
byte_t* casted_data = cast_buffer_.data() + casted_vector_bytes_ * config.thread;
|
|
594
|
+
bool casted = cast(vector_data, vector_bytes, casted_data);
|
|
575
595
|
if (casted)
|
|
576
596
|
vector_data = casted_data, vector_bytes = casted_vector_bytes_;
|
|
577
597
|
|
|
578
|
-
return index_->
|
|
598
|
+
return index_->search_around(hint, {vector_data, vector_bytes}, wanted, config);
|
|
579
599
|
}
|
|
580
600
|
|
|
581
|
-
template <typename scalar_at>
|
|
582
|
-
thread_lock_t lock =
|
|
583
|
-
|
|
601
|
+
template <typename scalar_at> add_result_t add_(label_t label, scalar_at const* vector, cast_t const& cast) {
|
|
602
|
+
thread_lock_t lock = thread_lock_();
|
|
603
|
+
add_config_t config;
|
|
604
|
+
config.thread = lock.thread_id;
|
|
605
|
+
return add_(label, vector, config, cast);
|
|
584
606
|
}
|
|
585
607
|
|
|
586
608
|
template <typename scalar_at>
|
|
587
|
-
|
|
609
|
+
search_results_t search_( //
|
|
588
610
|
scalar_at const* vector, std::size_t wanted, //
|
|
589
|
-
label_t* matches, distance_t* distances, //
|
|
590
611
|
cast_t const& cast) const {
|
|
591
|
-
thread_lock_t lock =
|
|
592
|
-
|
|
612
|
+
thread_lock_t lock = thread_lock_();
|
|
613
|
+
search_config_t config;
|
|
614
|
+
config.thread = lock.thread_id;
|
|
615
|
+
return search_(vector, wanted, config, cast);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
template <typename scalar_at>
|
|
619
|
+
search_results_t search_around_( //
|
|
620
|
+
label_t hint, scalar_at const* vector, std::size_t wanted, //
|
|
621
|
+
cast_t const& cast) const {
|
|
622
|
+
thread_lock_t lock = thread_lock_();
|
|
623
|
+
search_config_t config;
|
|
624
|
+
config.thread = lock.thread_id;
|
|
625
|
+
return search_around_(hint, vector, wanted, config, cast);
|
|
593
626
|
}
|
|
594
627
|
|
|
595
|
-
static
|
|
628
|
+
static punned_gt make_( //
|
|
596
629
|
std::size_t dimensions, accuracy_t accuracy, //
|
|
597
630
|
metric_and_meta_t metric_and_meta, casts_t casts, //
|
|
598
631
|
config_t config) {
|
|
@@ -601,7 +634,7 @@ class auto_index_gt {
|
|
|
601
634
|
config.max_threads_add = config.max_threads_add ? config.max_threads_add : hardware_threads;
|
|
602
635
|
config.max_threads_search = config.max_threads_search ? config.max_threads_search : hardware_threads;
|
|
603
636
|
std::size_t max_threads = (std::max)(config.max_threads_add, config.max_threads_search);
|
|
604
|
-
|
|
637
|
+
punned_gt result;
|
|
605
638
|
result.dimensions_ = dimensions;
|
|
606
639
|
result.accuracy_ = accuracy;
|
|
607
640
|
result.casted_vector_bytes_ = bytes_per_scalar(accuracy) * dimensions;
|
|
@@ -609,20 +642,19 @@ class auto_index_gt {
|
|
|
609
642
|
result.casts_ = casts;
|
|
610
643
|
result.acceleration_ = metric_and_meta.acceleration;
|
|
611
644
|
result.root_metric_ = metric_and_meta.metric;
|
|
612
|
-
result.root_config_ = config;
|
|
613
645
|
|
|
614
646
|
// Fill the thread IDs.
|
|
615
647
|
result.available_threads_.resize(max_threads);
|
|
616
648
|
std::iota(result.available_threads_.begin(), result.available_threads_.end(), 0ul);
|
|
617
649
|
|
|
618
|
-
//
|
|
619
|
-
index_t* raw =
|
|
650
|
+
// Available since C11, but only C++17, so we use the C version.
|
|
651
|
+
index_t* raw = aligned_index_alloc_();
|
|
620
652
|
new (raw) index_t(config, metric_and_meta.metric);
|
|
621
653
|
result.index_ = raw;
|
|
622
654
|
return result;
|
|
623
655
|
}
|
|
624
656
|
|
|
625
|
-
template <typename to_scalar_at> static casts_t
|
|
657
|
+
template <typename to_scalar_at> static casts_t make_casts_() {
|
|
626
658
|
casts_t result;
|
|
627
659
|
result.from_f8 = cast_gt<f8_bits_t, to_scalar_at>{};
|
|
628
660
|
result.from_f16 = cast_gt<f16_bits_t, to_scalar_at>{};
|
|
@@ -631,126 +663,126 @@ class auto_index_gt {
|
|
|
631
663
|
return result;
|
|
632
664
|
}
|
|
633
665
|
|
|
634
|
-
static casts_t
|
|
666
|
+
static casts_t make_casts_(accuracy_t accuracy) {
|
|
635
667
|
switch (accuracy) {
|
|
636
|
-
case accuracy_t::f64_k: return
|
|
637
|
-
case accuracy_t::f32_k: return
|
|
638
|
-
case accuracy_t::f16_k: return
|
|
639
|
-
case accuracy_t::f8_k: return
|
|
668
|
+
case accuracy_t::f64_k: return make_casts_<f64_t>();
|
|
669
|
+
case accuracy_t::f32_k: return make_casts_<f32_t>();
|
|
670
|
+
case accuracy_t::f16_k: return make_casts_<f16_bits_t>();
|
|
671
|
+
case accuracy_t::f8_k: return make_casts_<f8_bits_t>();
|
|
640
672
|
default: return {};
|
|
641
673
|
}
|
|
642
674
|
}
|
|
643
675
|
|
|
644
676
|
template <typename scalar_at, typename typed_at> //
|
|
645
|
-
static punned_stateful_metric_t
|
|
646
|
-
return [=](byte_t const* a, byte_t const* b, std::size_t bytes, std::size_t)
|
|
677
|
+
static punned_stateful_metric_t pun_metric_(typed_at metric) {
|
|
678
|
+
return [=](byte_t const* a, byte_t const* b, std::size_t bytes, std::size_t) -> float {
|
|
647
679
|
return metric((scalar_at const*)a, (scalar_at const*)b, bytes / sizeof(scalar_at));
|
|
648
680
|
};
|
|
649
681
|
}
|
|
650
682
|
|
|
651
|
-
static metric_and_meta_t
|
|
683
|
+
static metric_and_meta_t ip_metric_f32_(std::size_t dimensions) {
|
|
652
684
|
(void)dimensions;
|
|
653
|
-
#if defined(
|
|
654
|
-
#if defined(
|
|
685
|
+
#if defined(USEARCH_USE_SIMSIMD)
|
|
686
|
+
#if defined(USEARCH_IS_X86)
|
|
655
687
|
if (dimensions % 4 == 0)
|
|
656
688
|
return {
|
|
657
|
-
|
|
689
|
+
pun_metric_<simsimd_f32_t>([](simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
658
690
|
return 1.f - simsimd_dot_f32x4avx2(a, b, d);
|
|
659
691
|
}),
|
|
660
692
|
isa_t::avx2_k,
|
|
661
693
|
};
|
|
662
|
-
#elif defined(
|
|
694
|
+
#elif defined(USEARCH_IS_ARM)
|
|
663
695
|
if (supports_arm_sve())
|
|
664
696
|
return {
|
|
665
|
-
|
|
697
|
+
pun_metric_<simsimd_f32_t>([](simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
666
698
|
return 1.f - simsimd_dot_f32sve(a, b, d);
|
|
667
699
|
}),
|
|
668
700
|
isa_t::sve_k,
|
|
669
701
|
};
|
|
670
702
|
if (dimensions % 4 == 0)
|
|
671
703
|
return {
|
|
672
|
-
|
|
704
|
+
pun_metric_<simsimd_f32_t>([](simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
673
705
|
return 1.f - simsimd_dot_f32x4neon(a, b, d);
|
|
674
706
|
}),
|
|
675
707
|
isa_t::neon_k,
|
|
676
708
|
};
|
|
677
709
|
#endif
|
|
678
710
|
#endif
|
|
679
|
-
return {
|
|
711
|
+
return {pun_metric_<f32_t>(ip_gt<f32_t>{}), isa_t::auto_k};
|
|
680
712
|
}
|
|
681
713
|
|
|
682
|
-
static metric_and_meta_t
|
|
714
|
+
static metric_and_meta_t cos_metric_f16_(std::size_t dimensions) {
|
|
683
715
|
(void)dimensions;
|
|
684
|
-
#if defined(
|
|
685
|
-
#if defined(
|
|
716
|
+
#if defined(USEARCH_USE_SIMSIMD)
|
|
717
|
+
#if defined(USEARCH_IS_X86)
|
|
686
718
|
if (dimensions % 16 == 0)
|
|
687
719
|
return {
|
|
688
|
-
|
|
720
|
+
pun_metric_<simsimd_f16_t>([](simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
689
721
|
return 1.f - simsimd_cos_f16x16avx512(a, b, d);
|
|
690
722
|
}),
|
|
691
723
|
isa_t::avx512_k,
|
|
692
724
|
};
|
|
693
|
-
#elif defined(
|
|
725
|
+
#elif defined(USEARCH_IS_ARM)
|
|
694
726
|
if (dimensions % 4 == 0)
|
|
695
727
|
return {
|
|
696
|
-
|
|
728
|
+
pun_metric_<simsimd_f16_t>([](simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
697
729
|
return 1.f - simsimd_cos_f16x4neon(a, b, d);
|
|
698
730
|
}),
|
|
699
731
|
isa_t::neon_k,
|
|
700
732
|
};
|
|
701
733
|
#endif
|
|
702
734
|
#endif
|
|
703
|
-
return {
|
|
735
|
+
return {pun_metric_<f16_bits_t>(cos_gt<f16_bits_t, f32_t>{}), isa_t::auto_k};
|
|
704
736
|
}
|
|
705
737
|
|
|
706
|
-
static metric_and_meta_t
|
|
738
|
+
static metric_and_meta_t ip_metric_(std::size_t dimensions, accuracy_t accuracy) {
|
|
707
739
|
switch (accuracy) {
|
|
708
740
|
case accuracy_t::f32_k:
|
|
709
741
|
// The two most common numeric types for the most common metric have optimized versions
|
|
710
|
-
return
|
|
742
|
+
return ip_metric_f32_(dimensions);
|
|
711
743
|
case accuracy_t::f16_k:
|
|
712
744
|
// Dot-product accumulates error, Cosine-distance normalizes it
|
|
713
|
-
return
|
|
745
|
+
return cos_metric_f16_(dimensions);
|
|
714
746
|
|
|
715
|
-
case accuracy_t::f8_k: return {
|
|
716
|
-
case accuracy_t::f64_k: return {
|
|
747
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(cos_f8_t{}), isa_t::auto_k};
|
|
748
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(ip_gt<f64_t>{}), isa_t::auto_k};
|
|
717
749
|
default: return {};
|
|
718
750
|
}
|
|
719
751
|
}
|
|
720
752
|
|
|
721
|
-
static metric_and_meta_t
|
|
753
|
+
static metric_and_meta_t l2_metric_(std::size_t, accuracy_t accuracy) {
|
|
722
754
|
switch (accuracy) {
|
|
723
|
-
case accuracy_t::f8_k: return {
|
|
724
|
-
case accuracy_t::f16_k: return {
|
|
725
|
-
case accuracy_t::f32_k: return {
|
|
726
|
-
case accuracy_t::f64_k: return {
|
|
755
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(l2sq_f8_t{}), isa_t::auto_k};
|
|
756
|
+
case accuracy_t::f16_k: return {pun_metric_<f16_bits_t>(l2sq_gt<f16_bits_t, f32_t>{}), isa_t::auto_k};
|
|
757
|
+
case accuracy_t::f32_k: return {pun_metric_<f32_t>(l2sq_gt<f32_t>{}), isa_t::auto_k};
|
|
758
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(l2sq_gt<f64_t>{}), isa_t::auto_k};
|
|
727
759
|
default: return {};
|
|
728
760
|
}
|
|
729
761
|
}
|
|
730
762
|
|
|
731
|
-
static metric_and_meta_t
|
|
763
|
+
static metric_and_meta_t cos_metric_(std::size_t dimensions, accuracy_t accuracy) {
|
|
732
764
|
switch (accuracy) {
|
|
733
|
-
case accuracy_t::f8_k: return {
|
|
734
|
-
case accuracy_t::f16_k: return
|
|
735
|
-
case accuracy_t::f32_k: return {
|
|
736
|
-
case accuracy_t::f64_k: return {
|
|
765
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(cos_f8_t{}), isa_t::auto_k};
|
|
766
|
+
case accuracy_t::f16_k: return cos_metric_f16_(dimensions);
|
|
767
|
+
case accuracy_t::f32_k: return {pun_metric_<f32_t>(cos_gt<f32_t>{}), isa_t::auto_k};
|
|
768
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(cos_gt<f64_t>{}), isa_t::auto_k};
|
|
737
769
|
default: return {};
|
|
738
770
|
}
|
|
739
771
|
}
|
|
740
772
|
|
|
741
|
-
static metric_and_meta_t
|
|
773
|
+
static metric_and_meta_t haversine_metric_(accuracy_t accuracy) {
|
|
742
774
|
switch (accuracy) {
|
|
743
|
-
case accuracy_t::f8_k: return {
|
|
744
|
-
case accuracy_t::f16_k: return {
|
|
745
|
-
case accuracy_t::f32_k: return {
|
|
746
|
-
case accuracy_t::f64_k: return {
|
|
775
|
+
case accuracy_t::f8_k: return {pun_metric_<f8_bits_t>(haversine_gt<f8_bits_t, f32_t>{}), isa_t::auto_k};
|
|
776
|
+
case accuracy_t::f16_k: return {pun_metric_<f16_bits_t>(haversine_gt<f16_bits_t, f32_t>{}), isa_t::auto_k};
|
|
777
|
+
case accuracy_t::f32_k: return {pun_metric_<f32_t>(haversine_gt<f32_t>{}), isa_t::auto_k};
|
|
778
|
+
case accuracy_t::f64_k: return {pun_metric_<f64_t>(haversine_gt<f64_t>{}), isa_t::auto_k};
|
|
747
779
|
default: return {};
|
|
748
780
|
}
|
|
749
781
|
}
|
|
750
782
|
};
|
|
751
783
|
|
|
752
|
-
using
|
|
753
|
-
using
|
|
784
|
+
using punned_small_t = punned_gt<std::int64_t, std::uint32_t>;
|
|
785
|
+
using punned_big_t = punned_gt<uuid_t, uint40_t>;
|
|
754
786
|
|
|
755
787
|
} // namespace usearch
|
|
756
788
|
} // namespace unum
|