usearch 2.15.2 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.gyp +1 -1
- package/c/CMakeLists.txt +10 -3
- package/cpp/README.md +1 -1
- package/cpp/bench.cpp +2 -2
- package/cpp/test.cpp +16 -0
- package/include/usearch/index.hpp +44 -17
- package/include/usearch/index_dense.hpp +51 -89
- package/include/usearch/index_plugins.hpp +468 -43
- package/javascript/lib.cpp +17 -5
- package/javascript/usearch.test.js +205 -34
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.git-blame-ignore-revs +2 -0
- package/simsimd/CODE_OF_CONDUCT.md +128 -0
- package/simsimd/CONTRIBUTING.md +84 -26
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +4 -4
- package/simsimd/MANIFEST.in +3 -0
- package/simsimd/Package.swift +15 -3
- package/simsimd/README.md +534 -177
- package/simsimd/binding.gyp +6 -2
- package/simsimd/build.rs +13 -1
- package/simsimd/c/lib.c +266 -53
- package/simsimd/include/simsimd/binary.h +321 -112
- package/simsimd/include/simsimd/curved.h +803 -0
- package/simsimd/include/simsimd/dot.h +694 -521
- package/simsimd/include/simsimd/elementwise.h +1437 -0
- package/simsimd/include/simsimd/geospatial.h +9 -3
- package/simsimd/include/simsimd/mesh.h +69 -0
- package/simsimd/include/simsimd/probability.h +234 -206
- package/simsimd/include/simsimd/simsimd.h +1476 -617
- package/simsimd/include/simsimd/sparse.h +1428 -0
- package/simsimd/include/simsimd/spatial.h +1556 -742
- package/simsimd/include/simsimd/types.h +276 -132
- package/simsimd/javascript/lib.c +1 -2
- package/simsimd/javascript/simsimd.ts +4 -4
- package/simsimd/package.json +4 -4
- package/simsimd/rust/lib.rs +179 -1
- package/simsimd/scripts/bench.cxx +1070 -0
- package/simsimd/scripts/bench.ipynb +187 -0
- package/simsimd/{javascript → scripts}/bench.js +1 -1
- package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
- package/simsimd/scripts/bench_vectors.py +700 -0
- package/simsimd/scripts/bench_vectors_live.py +213 -0
- package/simsimd/scripts/test.c +155 -0
- package/simsimd/{javascript → scripts}/test.mjs +2 -2
- package/simsimd/scripts/test.py +1410 -0
- package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
- package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
- /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
- /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
|
@@ -11,10 +11,11 @@
|
|
|
11
11
|
* For datatypes:
|
|
12
12
|
* - 32-bit floating point numbers
|
|
13
13
|
* - 16-bit floating point numbers
|
|
14
|
+
* - 16-bit brain-floating point numbers
|
|
14
15
|
*
|
|
15
16
|
* For hardware architectures:
|
|
16
|
-
* - Arm
|
|
17
|
-
* - x86
|
|
17
|
+
* - Arm: NEON
|
|
18
|
+
* - x86: Haswell, Skylake, Sapphire
|
|
18
19
|
*
|
|
19
20
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
20
21
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -82,62 +83,62 @@ SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_
|
|
|
82
83
|
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* divergence);
|
|
83
84
|
// clang-format on
|
|
84
85
|
|
|
85
|
-
#define SIMSIMD_MAKE_KL(name, input_type, accumulator_type,
|
|
86
|
-
SIMSIMD_PUBLIC void simsimd_kl_##input_type##_##name(simsimd_##input_type##_t const*
|
|
87
|
-
simsimd_##input_type##_t const*
|
|
88
|
-
simsimd_distance_t*
|
|
89
|
-
simsimd_##accumulator_type##_t d = 0;
|
|
90
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
91
|
-
simsimd_##accumulator_type##_t ai =
|
|
92
|
-
simsimd_##accumulator_type##_t bi =
|
|
93
|
-
d += ai * SIMSIMD_LOG((ai + epsilon) / (bi + epsilon));
|
|
94
|
-
}
|
|
95
|
-
*result = (simsimd_distance_t)d;
|
|
86
|
+
#define SIMSIMD_MAKE_KL(name, input_type, accumulator_type, load_and_convert, epsilon) \
|
|
87
|
+
SIMSIMD_PUBLIC void simsimd_kl_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
88
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
89
|
+
simsimd_distance_t *result) { \
|
|
90
|
+
simsimd_##accumulator_type##_t d = 0; \
|
|
91
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
92
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
93
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
94
|
+
d += ai * SIMSIMD_LOG((ai + epsilon) / (bi + epsilon)); \
|
|
95
|
+
} \
|
|
96
|
+
*result = (simsimd_distance_t)d; \
|
|
96
97
|
}
|
|
97
98
|
|
|
98
|
-
#define SIMSIMD_MAKE_JS(name, input_type, accumulator_type,
|
|
99
|
-
SIMSIMD_PUBLIC void simsimd_js_##input_type##_##name(simsimd_##input_type##_t const*
|
|
100
|
-
simsimd_##input_type##_t const*
|
|
101
|
-
simsimd_distance_t*
|
|
102
|
-
simsimd_##accumulator_type##_t d = 0;
|
|
103
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
104
|
-
simsimd_##accumulator_type##_t ai =
|
|
105
|
-
simsimd_##accumulator_type##_t bi =
|
|
106
|
-
simsimd_##accumulator_type##_t mi = (ai + bi) / 2;
|
|
107
|
-
d += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
108
|
-
d += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
109
|
-
}
|
|
110
|
-
*result = (simsimd_distance_t)d / 2;
|
|
99
|
+
#define SIMSIMD_MAKE_JS(name, input_type, accumulator_type, load_and_convert, epsilon) \
|
|
100
|
+
SIMSIMD_PUBLIC void simsimd_js_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
101
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
102
|
+
simsimd_distance_t *result) { \
|
|
103
|
+
simsimd_##accumulator_type##_t d = 0; \
|
|
104
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
105
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
106
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
107
|
+
simsimd_##accumulator_type##_t mi = (ai + bi) / 2; \
|
|
108
|
+
d += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon)); \
|
|
109
|
+
d += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon)); \
|
|
110
|
+
} \
|
|
111
|
+
*result = (simsimd_distance_t)d / 2; \
|
|
111
112
|
}
|
|
112
113
|
|
|
113
|
-
SIMSIMD_MAKE_KL(serial, f64, f64,
|
|
114
|
-
SIMSIMD_MAKE_JS(serial, f64, f64,
|
|
114
|
+
SIMSIMD_MAKE_KL(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f64_serial
|
|
115
|
+
SIMSIMD_MAKE_JS(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f64_serial
|
|
115
116
|
|
|
116
|
-
SIMSIMD_MAKE_KL(serial, f32, f32,
|
|
117
|
-
SIMSIMD_MAKE_JS(serial, f32, f32,
|
|
117
|
+
SIMSIMD_MAKE_KL(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f32_serial
|
|
118
|
+
SIMSIMD_MAKE_JS(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f32_serial
|
|
118
119
|
|
|
119
|
-
SIMSIMD_MAKE_KL(serial, f16, f32,
|
|
120
|
-
SIMSIMD_MAKE_JS(serial, f16, f32,
|
|
120
|
+
SIMSIMD_MAKE_KL(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f16_serial
|
|
121
|
+
SIMSIMD_MAKE_JS(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f16_serial
|
|
121
122
|
|
|
122
|
-
SIMSIMD_MAKE_KL(serial, bf16, f32,
|
|
123
|
-
SIMSIMD_MAKE_JS(serial, bf16, f32,
|
|
123
|
+
SIMSIMD_MAKE_KL(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_bf16_serial
|
|
124
|
+
SIMSIMD_MAKE_JS(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_bf16_serial
|
|
124
125
|
|
|
125
|
-
SIMSIMD_MAKE_KL(accurate, f32, f64,
|
|
126
|
-
SIMSIMD_MAKE_JS(accurate, f32, f64,
|
|
126
|
+
SIMSIMD_MAKE_KL(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f32_accurate
|
|
127
|
+
SIMSIMD_MAKE_JS(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f32_accurate
|
|
127
128
|
|
|
128
|
-
SIMSIMD_MAKE_KL(accurate, f16, f64,
|
|
129
|
-
SIMSIMD_MAKE_JS(accurate, f16, f64,
|
|
129
|
+
SIMSIMD_MAKE_KL(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f16_accurate
|
|
130
|
+
SIMSIMD_MAKE_JS(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_f16_accurate
|
|
130
131
|
|
|
131
|
-
SIMSIMD_MAKE_KL(accurate, bf16, f64,
|
|
132
|
-
SIMSIMD_MAKE_JS(accurate, bf16, f64,
|
|
132
|
+
SIMSIMD_MAKE_KL(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_bf16_accurate
|
|
133
|
+
SIMSIMD_MAKE_JS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_bf16_accurate
|
|
133
134
|
|
|
134
|
-
#if
|
|
135
|
+
#if _SIMSIMD_TARGET_ARM
|
|
135
136
|
#if SIMSIMD_TARGET_NEON
|
|
136
137
|
#pragma GCC push_options
|
|
137
|
-
#pragma GCC target("+simd")
|
|
138
|
-
#pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
|
|
138
|
+
#pragma GCC target("arch=armv8.2-a+simd")
|
|
139
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
139
140
|
|
|
140
|
-
SIMSIMD_PUBLIC float32x4_t
|
|
141
|
+
SIMSIMD_PUBLIC float32x4_t _simsimd_log2_f32_neon(float32x4_t x) {
|
|
141
142
|
// Extracting the exponent
|
|
142
143
|
int32x4_t i = vreinterpretq_s32_f32(x);
|
|
143
144
|
int32x4_t e = vsubq_s32(vshrq_n_s32(vandq_s32(i, vdupq_n_s32(0x7F800000)), 23), vdupq_n_s32(127));
|
|
@@ -162,126 +163,155 @@ SIMSIMD_PUBLIC float32x4_t simsimd_log2_f32_neon(float32x4_t x) {
|
|
|
162
163
|
return result;
|
|
163
164
|
}
|
|
164
165
|
|
|
165
|
-
SIMSIMD_PUBLIC void simsimd_kl_f32_neon(simsimd_f32_t const*
|
|
166
|
-
simsimd_distance_t*
|
|
167
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
166
|
+
SIMSIMD_PUBLIC void simsimd_kl_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
167
|
+
simsimd_distance_t *result) {
|
|
168
168
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
169
169
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
170
|
+
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
171
|
+
float32x4_t a_vec, b_vec;
|
|
172
|
+
|
|
173
|
+
simsimd_kl_f32_neon_cycle:
|
|
174
|
+
if (n < 4) {
|
|
175
|
+
a_vec = _simsimd_partial_load_f32x4_neon(a, n);
|
|
176
|
+
b_vec = _simsimd_partial_load_f32x4_neon(b, n);
|
|
177
|
+
n = 0;
|
|
178
178
|
}
|
|
179
|
+
else {
|
|
180
|
+
a_vec = vld1q_f32(a);
|
|
181
|
+
b_vec = vld1q_f32(b);
|
|
182
|
+
n -= 4, a += 4, b += 4;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
|
|
186
|
+
float32x4_t log_ratio_vec = _simsimd_log2_f32_neon(ratio_vec);
|
|
187
|
+
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
188
|
+
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
189
|
+
if (n != 0) goto simsimd_kl_f32_neon_cycle;
|
|
190
|
+
|
|
179
191
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
180
192
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
181
|
-
for (; i < n; ++i)
|
|
182
|
-
sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (b[i] + epsilon));
|
|
183
193
|
*result = sum;
|
|
184
194
|
}
|
|
185
195
|
|
|
186
|
-
SIMSIMD_PUBLIC void simsimd_js_f32_neon(simsimd_f32_t const*
|
|
187
|
-
simsimd_distance_t*
|
|
188
|
-
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
196
|
+
SIMSIMD_PUBLIC void simsimd_js_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
197
|
+
simsimd_distance_t *result) {
|
|
189
198
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
190
199
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
200
|
+
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
201
|
+
float32x4_t a_vec, b_vec;
|
|
202
|
+
|
|
203
|
+
simsimd_js_f32_neon_cycle:
|
|
204
|
+
if (n < 4) {
|
|
205
|
+
a_vec = _simsimd_partial_load_f32x4_neon(a, n);
|
|
206
|
+
b_vec = _simsimd_partial_load_f32x4_neon(b, n);
|
|
207
|
+
n = 0;
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
a_vec = vld1q_f32(a);
|
|
211
|
+
b_vec = vld1q_f32(b);
|
|
212
|
+
n -= 4, a += 4, b += 4;
|
|
203
213
|
}
|
|
214
|
+
|
|
215
|
+
float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
|
|
216
|
+
float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
217
|
+
float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
218
|
+
float32x4_t log_ratio_a_vec = _simsimd_log2_f32_neon(ratio_a_vec);
|
|
219
|
+
float32x4_t log_ratio_b_vec = _simsimd_log2_f32_neon(ratio_b_vec);
|
|
220
|
+
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
221
|
+
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
222
|
+
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
223
|
+
if (n != 0) goto simsimd_js_f32_neon_cycle;
|
|
224
|
+
|
|
204
225
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
205
226
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
206
|
-
|
|
207
|
-
simsimd_f32_t mi = 0.5f * (a[i] + b[i]);
|
|
208
|
-
sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (mi + epsilon));
|
|
209
|
-
sum += b[i] * SIMSIMD_LOG((b[i] + epsilon) / (mi + epsilon));
|
|
210
|
-
}
|
|
211
|
-
*result = sum;
|
|
227
|
+
*result = sum / 2;
|
|
212
228
|
}
|
|
213
229
|
|
|
214
230
|
#pragma clang attribute pop
|
|
215
231
|
#pragma GCC pop_options
|
|
232
|
+
#endif // SIMSIMD_TARGET_NEON
|
|
216
233
|
|
|
234
|
+
#if SIMSIMD_TARGET_NEON_F16
|
|
217
235
|
#pragma GCC push_options
|
|
218
|
-
#pragma GCC target("+simd+fp16")
|
|
219
|
-
#pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
|
|
236
|
+
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
237
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
220
238
|
|
|
221
|
-
SIMSIMD_PUBLIC void simsimd_kl_f16_neon(simsimd_f16_t const*
|
|
222
|
-
simsimd_distance_t*
|
|
239
|
+
SIMSIMD_PUBLIC void simsimd_kl_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
240
|
+
simsimd_distance_t *result) {
|
|
223
241
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
224
242
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
225
243
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
244
|
+
float32x4_t a_vec, b_vec;
|
|
245
|
+
|
|
246
|
+
simsimd_kl_f16_neon_cycle:
|
|
247
|
+
if (n < 4) {
|
|
248
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
249
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
250
|
+
n = 0;
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
254
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
255
|
+
n -= 4, a += 4, b += 4;
|
|
234
256
|
}
|
|
257
|
+
|
|
258
|
+
float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
|
|
259
|
+
float32x4_t log_ratio_vec = _simsimd_log2_f32_neon(ratio_vec);
|
|
260
|
+
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
261
|
+
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
262
|
+
if (n) goto simsimd_kl_f16_neon_cycle;
|
|
263
|
+
|
|
235
264
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
236
265
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
237
|
-
for (; i < n; ++i)
|
|
238
|
-
sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
|
|
239
|
-
SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
|
|
240
266
|
*result = sum;
|
|
241
267
|
}
|
|
242
268
|
|
|
243
|
-
SIMSIMD_PUBLIC void simsimd_js_f16_neon(simsimd_f16_t const*
|
|
244
|
-
simsimd_distance_t*
|
|
269
|
+
SIMSIMD_PUBLIC void simsimd_js_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
270
|
+
simsimd_distance_t *result) {
|
|
245
271
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
246
272
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
247
273
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
float32x4_t log_ratio_a_vec = simsimd_log2_f32_neon(ratio_a_vec);
|
|
256
|
-
float32x4_t log_ratio_b_vec = simsimd_log2_f32_neon(ratio_b_vec);
|
|
257
|
-
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
258
|
-
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
259
|
-
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
274
|
+
float32x4_t a_vec, b_vec;
|
|
275
|
+
|
|
276
|
+
simsimd_js_f16_neon_cycle:
|
|
277
|
+
if (n < 4) {
|
|
278
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
279
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
280
|
+
n = 0;
|
|
260
281
|
}
|
|
282
|
+
else {
|
|
283
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
284
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
285
|
+
n -= 4, a += 4, b += 4;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
|
|
289
|
+
float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
290
|
+
float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
291
|
+
float32x4_t log_ratio_a_vec = _simsimd_log2_f32_neon(ratio_a_vec);
|
|
292
|
+
float32x4_t log_ratio_b_vec = _simsimd_log2_f32_neon(ratio_b_vec);
|
|
293
|
+
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
294
|
+
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
295
|
+
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
296
|
+
if (n) goto simsimd_js_f16_neon_cycle;
|
|
297
|
+
|
|
261
298
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
262
299
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
263
|
-
|
|
264
|
-
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
|
|
265
|
-
simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
|
|
266
|
-
simsimd_f32_t mi = 0.5f * (ai + bi);
|
|
267
|
-
sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
268
|
-
sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
269
|
-
}
|
|
270
|
-
*result = sum;
|
|
300
|
+
*result = sum / 2;
|
|
271
301
|
}
|
|
272
302
|
|
|
273
303
|
#pragma clang attribute pop
|
|
274
304
|
#pragma GCC pop_options
|
|
275
|
-
#endif //
|
|
276
|
-
#endif //
|
|
305
|
+
#endif // SIMSIMD_TARGET_NEON_F16
|
|
306
|
+
#endif // _SIMSIMD_TARGET_ARM
|
|
277
307
|
|
|
278
|
-
#if
|
|
308
|
+
#if _SIMSIMD_TARGET_X86
|
|
279
309
|
#if SIMSIMD_TARGET_HASWELL
|
|
280
310
|
#pragma GCC push_options
|
|
281
311
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
282
312
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
283
313
|
|
|
284
|
-
|
|
314
|
+
SIMSIMD_INTERNAL __m256 _simsimd_log2_f32_haswell(__m256 x) {
|
|
285
315
|
// Extracting the exponent
|
|
286
316
|
__m256i i = _mm256_castps_si256(x);
|
|
287
317
|
__m256i e = _mm256_srli_epi32(_mm256_and_si256(i, _mm256_set1_epi32(0x7F800000)), 23);
|
|
@@ -308,74 +338,72 @@ inline __m256 simsimd_log2_f32_haswell(__m256 x) {
|
|
|
308
338
|
return result;
|
|
309
339
|
}
|
|
310
340
|
|
|
311
|
-
SIMSIMD_PUBLIC void simsimd_kl_f16_haswell(simsimd_f16_t const*
|
|
312
|
-
simsimd_distance_t*
|
|
341
|
+
SIMSIMD_PUBLIC void simsimd_kl_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
342
|
+
simsimd_distance_t *result) {
|
|
313
343
|
__m256 sum_vec = _mm256_setzero_ps();
|
|
314
344
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
315
345
|
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
316
|
-
|
|
317
|
-
for (; i + 8 <= n; i += 8) {
|
|
318
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
319
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
320
|
-
__m256 ratio_vec = _mm256_div_ps(_mm256_add_ps(a_vec, epsilon_vec), _mm256_add_ps(b_vec, epsilon_vec));
|
|
321
|
-
__m256 log_ratio_vec = simsimd_log2_f32_haswell(ratio_vec);
|
|
322
|
-
__m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
|
|
323
|
-
sum_vec = _mm256_add_ps(sum_vec, prod_vec);
|
|
324
|
-
}
|
|
346
|
+
__m256 a_vec, b_vec;
|
|
325
347
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
348
|
+
simsimd_kl_f16_haswell_cycle:
|
|
349
|
+
if (n < 8) {
|
|
350
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
351
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
352
|
+
n = 0;
|
|
353
|
+
}
|
|
354
|
+
else {
|
|
355
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
356
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
357
|
+
n -= 8, a += 8, b += 8;
|
|
358
|
+
}
|
|
359
|
+
a_vec = _mm256_add_ps(a_vec, epsilon_vec);
|
|
360
|
+
b_vec = _mm256_add_ps(b_vec, epsilon_vec);
|
|
361
|
+
__m256 ratio_vec = _mm256_div_ps(a_vec, b_vec);
|
|
362
|
+
__m256 log_ratio_vec = _simsimd_log2_f32_haswell(ratio_vec);
|
|
363
|
+
__m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
|
|
364
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_vec);
|
|
365
|
+
if (n) goto simsimd_kl_f16_haswell_cycle;
|
|
329
366
|
|
|
330
367
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
331
|
-
simsimd_f32_t sum;
|
|
332
|
-
_mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
|
|
368
|
+
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
333
369
|
sum *= log2_normalizer;
|
|
334
|
-
|
|
335
|
-
// Accumulate the tail:
|
|
336
|
-
for (; i < n; ++i)
|
|
337
|
-
sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
|
|
338
|
-
SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
|
|
339
370
|
*result = sum;
|
|
340
371
|
}
|
|
341
372
|
|
|
342
|
-
SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const*
|
|
343
|
-
simsimd_distance_t*
|
|
344
|
-
__m256 sum_vec = _mm256_setzero_ps();
|
|
373
|
+
SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
374
|
+
simsimd_distance_t *result) {
|
|
345
375
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
346
376
|
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
__m256 a_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i))), epsilon_vec);
|
|
350
|
-
__m256 b_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i))), epsilon_vec);
|
|
351
|
-
__m256 m_vec = _mm256_mul_ps(_mm256_add_ps(a_vec, b_vec), _mm256_set1_ps(0.5f)); // M = (P + Q) / 2
|
|
352
|
-
__m256 ratio_a_vec = _mm256_div_ps(a_vec, m_vec);
|
|
353
|
-
__m256 ratio_b_vec = _mm256_div_ps(b_vec, m_vec);
|
|
354
|
-
__m256 log_ratio_a_vec = simsimd_log2_f32_haswell(ratio_a_vec);
|
|
355
|
-
__m256 log_ratio_b_vec = simsimd_log2_f32_haswell(ratio_b_vec);
|
|
356
|
-
__m256 prod_a_vec = _mm256_mul_ps(a_vec, log_ratio_a_vec);
|
|
357
|
-
__m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
|
|
358
|
-
sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
|
|
359
|
-
sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
|
|
360
|
-
}
|
|
377
|
+
__m256 sum_vec = _mm256_setzero_ps();
|
|
378
|
+
__m256 a_vec, b_vec;
|
|
361
379
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
380
|
+
simsimd_js_f16_haswell_cycle:
|
|
381
|
+
if (n < 8) {
|
|
382
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
383
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
384
|
+
n = 0;
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
388
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
389
|
+
n -= 8, a += 8, b += 8;
|
|
390
|
+
}
|
|
391
|
+
a_vec = _mm256_add_ps(a_vec, epsilon_vec);
|
|
392
|
+
b_vec = _mm256_add_ps(b_vec, epsilon_vec);
|
|
393
|
+
__m256 m_vec = _mm256_mul_ps(_mm256_add_ps(a_vec, b_vec), _mm256_set1_ps(0.5f)); // M = (P + Q) / 2
|
|
394
|
+
__m256 ratio_a_vec = _mm256_div_ps(a_vec, m_vec);
|
|
395
|
+
__m256 ratio_b_vec = _mm256_div_ps(b_vec, m_vec);
|
|
396
|
+
__m256 log_ratio_a_vec = _simsimd_log2_f32_haswell(ratio_a_vec);
|
|
397
|
+
__m256 log_ratio_b_vec = _simsimd_log2_f32_haswell(ratio_b_vec);
|
|
398
|
+
__m256 prod_a_vec = _mm256_mul_ps(a_vec, log_ratio_a_vec);
|
|
399
|
+
__m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
|
|
400
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
|
|
401
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
|
|
402
|
+
if (n) goto simsimd_js_f16_haswell_cycle;
|
|
365
403
|
|
|
366
404
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
367
|
-
simsimd_f32_t sum;
|
|
368
|
-
_mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
|
|
405
|
+
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
369
406
|
sum *= log2_normalizer;
|
|
370
|
-
|
|
371
|
-
// Accumulate the tail:
|
|
372
|
-
for (; i < n; ++i) {
|
|
373
|
-
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
|
|
374
|
-
simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
|
|
375
|
-
simsimd_f32_t mi = ai + bi;
|
|
376
|
-
sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
377
|
-
sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
378
|
-
}
|
|
379
407
|
*result = sum / 2;
|
|
380
408
|
}
|
|
381
409
|
|
|
@@ -385,10 +413,10 @@ SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
385
413
|
|
|
386
414
|
#if SIMSIMD_TARGET_SKYLAKE
|
|
387
415
|
#pragma GCC push_options
|
|
388
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2")
|
|
389
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
416
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2")
|
|
417
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
390
418
|
|
|
391
|
-
|
|
419
|
+
SIMSIMD_INTERNAL __m512 _simsimd_log2_f32_skylake(__m512 x) {
|
|
392
420
|
// Extract the exponent and mantissa
|
|
393
421
|
__m512 one = _mm512_set1_ps(1.0f);
|
|
394
422
|
__m512 e = _mm512_getexp_ps(x);
|
|
@@ -405,8 +433,8 @@ inline __m512 simsimd_log2_f32_skylake(__m512 x) {
|
|
|
405
433
|
return _mm512_add_ps(_mm512_mul_ps(p, _mm512_sub_ps(m, one)), e);
|
|
406
434
|
}
|
|
407
435
|
|
|
408
|
-
SIMSIMD_PUBLIC void simsimd_kl_f32_skylake(simsimd_f32_t const*
|
|
409
|
-
simsimd_distance_t*
|
|
436
|
+
SIMSIMD_PUBLIC void simsimd_kl_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
437
|
+
simsimd_distance_t *result) {
|
|
410
438
|
__m512 sum_vec = _mm512_setzero();
|
|
411
439
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
412
440
|
__m512 epsilon_vec = _mm512_set1_ps(epsilon);
|
|
@@ -418,24 +446,24 @@ simsimd_kl_f32_skylake_cycle:
|
|
|
418
446
|
a_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, a), epsilon_vec);
|
|
419
447
|
b_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, b), epsilon_vec);
|
|
420
448
|
n = 0;
|
|
421
|
-
}
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
422
451
|
a_vec = _mm512_add_ps(_mm512_loadu_ps(a), epsilon_vec);
|
|
423
452
|
b_vec = _mm512_add_ps(_mm512_loadu_ps(b), epsilon_vec);
|
|
424
453
|
a += 16, b += 16, n -= 16;
|
|
425
454
|
}
|
|
426
455
|
__m512 ratio_vec = _mm512_div_ps(a_vec, b_vec);
|
|
427
|
-
__m512 log_ratio_vec =
|
|
456
|
+
__m512 log_ratio_vec = _simsimd_log2_f32_skylake(ratio_vec);
|
|
428
457
|
__m512 prod_vec = _mm512_mul_ps(a_vec, log_ratio_vec);
|
|
429
458
|
sum_vec = _mm512_add_ps(sum_vec, prod_vec);
|
|
430
|
-
if (n)
|
|
431
|
-
goto simsimd_kl_f32_skylake_cycle;
|
|
459
|
+
if (n) goto simsimd_kl_f32_skylake_cycle;
|
|
432
460
|
|
|
433
461
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
434
462
|
*result = _mm512_reduce_add_ps(sum_vec) * log2_normalizer;
|
|
435
463
|
}
|
|
436
464
|
|
|
437
|
-
SIMSIMD_PUBLIC void simsimd_js_f32_skylake(simsimd_f32_t const*
|
|
438
|
-
simsimd_distance_t*
|
|
465
|
+
SIMSIMD_PUBLIC void simsimd_js_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
466
|
+
simsimd_distance_t *result) {
|
|
439
467
|
__m512 sum_a_vec = _mm512_setzero();
|
|
440
468
|
__m512 sum_b_vec = _mm512_setzero();
|
|
441
469
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
@@ -448,7 +476,8 @@ simsimd_js_f32_skylake_cycle:
|
|
|
448
476
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
449
477
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
450
478
|
n = 0;
|
|
451
|
-
}
|
|
479
|
+
}
|
|
480
|
+
else {
|
|
452
481
|
a_vec = _mm512_loadu_ps(a);
|
|
453
482
|
b_vec = _mm512_loadu_ps(b);
|
|
454
483
|
a += 16, b += 16, n -= 16;
|
|
@@ -460,15 +489,14 @@ simsimd_js_f32_skylake_cycle:
|
|
|
460
489
|
__m512 m_recip_approx = _mm512_rcp14_ps(m_vec);
|
|
461
490
|
__m512 ratio_a_vec = _mm512_mul_ps(a_vec, m_recip_approx);
|
|
462
491
|
__m512 ratio_b_vec = _mm512_mul_ps(b_vec, m_recip_approx);
|
|
463
|
-
__m512 log_ratio_a_vec =
|
|
464
|
-
__m512 log_ratio_b_vec =
|
|
492
|
+
__m512 log_ratio_a_vec = _simsimd_log2_f32_skylake(ratio_a_vec);
|
|
493
|
+
__m512 log_ratio_b_vec = _simsimd_log2_f32_skylake(ratio_b_vec);
|
|
465
494
|
sum_a_vec = _mm512_maskz_fmadd_ps(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
466
495
|
sum_b_vec = _mm512_maskz_fmadd_ps(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
467
|
-
if (n)
|
|
468
|
-
goto simsimd_js_f32_skylake_cycle;
|
|
496
|
+
if (n) goto simsimd_js_f32_skylake_cycle;
|
|
469
497
|
|
|
470
498
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
471
|
-
*result = _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) *
|
|
499
|
+
*result = _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) * log2_normalizer / 2;
|
|
472
500
|
}
|
|
473
501
|
|
|
474
502
|
#pragma clang attribute pop
|
|
@@ -477,10 +505,10 @@ simsimd_js_f32_skylake_cycle:
|
|
|
477
505
|
|
|
478
506
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
479
507
|
#pragma GCC push_options
|
|
480
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
481
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
508
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
509
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
482
510
|
|
|
483
|
-
|
|
511
|
+
SIMSIMD_INTERNAL __m512h _simsimd_log2_f16_sapphire(__m512h x) {
|
|
484
512
|
// Extract the exponent and mantissa
|
|
485
513
|
__m512h one = _mm512_set1_ph((simsimd_f16_t)1);
|
|
486
514
|
__m512h e = _mm512_getexp_ph(x);
|
|
@@ -497,8 +525,8 @@ inline __m512h simsimd_log2_f16_sapphire(__m512h x) {
|
|
|
497
525
|
return _mm512_add_ph(_mm512_mul_ph(p, _mm512_sub_ph(m, one)), e);
|
|
498
526
|
}
|
|
499
527
|
|
|
500
|
-
SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const*
|
|
501
|
-
simsimd_distance_t*
|
|
528
|
+
SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
529
|
+
simsimd_distance_t *result) {
|
|
502
530
|
__m512h sum_vec = _mm512_setzero_ph();
|
|
503
531
|
__m512h epsilon_vec = _mm512_set1_ph((simsimd_f16_t)SIMSIMD_F16_DIVISION_EPSILON);
|
|
504
532
|
__m512h a_vec, b_vec;
|
|
@@ -509,24 +537,24 @@ simsimd_kl_f16_sapphire_cycle:
|
|
|
509
537
|
a_vec = _mm512_maskz_add_ph(mask, _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a)), epsilon_vec);
|
|
510
538
|
b_vec = _mm512_maskz_add_ph(mask, _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b)), epsilon_vec);
|
|
511
539
|
n = 0;
|
|
512
|
-
}
|
|
540
|
+
}
|
|
541
|
+
else {
|
|
513
542
|
a_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_loadu_epi16(a)), epsilon_vec);
|
|
514
543
|
b_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_loadu_epi16(b)), epsilon_vec);
|
|
515
544
|
a += 32, b += 32, n -= 32;
|
|
516
545
|
}
|
|
517
546
|
__m512h ratio_vec = _mm512_div_ph(a_vec, b_vec);
|
|
518
|
-
__m512h log_ratio_vec =
|
|
547
|
+
__m512h log_ratio_vec = _simsimd_log2_f16_sapphire(ratio_vec);
|
|
519
548
|
__m512h prod_vec = _mm512_mul_ph(a_vec, log_ratio_vec);
|
|
520
549
|
sum_vec = _mm512_add_ph(sum_vec, prod_vec);
|
|
521
|
-
if (n)
|
|
522
|
-
goto simsimd_kl_f16_sapphire_cycle;
|
|
550
|
+
if (n) goto simsimd_kl_f16_sapphire_cycle;
|
|
523
551
|
|
|
524
552
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
525
553
|
*result = _mm512_reduce_add_ph(sum_vec) * log2_normalizer;
|
|
526
554
|
}
|
|
527
555
|
|
|
528
|
-
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const*
|
|
529
|
-
simsimd_distance_t*
|
|
556
|
+
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
557
|
+
simsimd_distance_t *result) {
|
|
530
558
|
__m512h sum_a_vec = _mm512_setzero_ph();
|
|
531
559
|
__m512h sum_b_vec = _mm512_setzero_ph();
|
|
532
560
|
__m512h epsilon_vec = _mm512_set1_ph((simsimd_f16_t)SIMSIMD_F16_DIVISION_EPSILON);
|
|
@@ -538,7 +566,8 @@ simsimd_js_f16_sapphire_cycle:
|
|
|
538
566
|
a_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
539
567
|
b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
540
568
|
n = 0;
|
|
541
|
-
}
|
|
569
|
+
}
|
|
570
|
+
else {
|
|
542
571
|
a_vec = _mm512_castsi512_ph(_mm512_loadu_epi16(a));
|
|
543
572
|
b_vec = _mm512_castsi512_ph(_mm512_loadu_epi16(b));
|
|
544
573
|
a += 32, b += 32, n -= 32;
|
|
@@ -550,21 +579,20 @@ simsimd_js_f16_sapphire_cycle:
|
|
|
550
579
|
__m512h m_recip_approx = _mm512_rcp_ph(m_vec);
|
|
551
580
|
__m512h ratio_a_vec = _mm512_mul_ph(a_vec, m_recip_approx);
|
|
552
581
|
__m512h ratio_b_vec = _mm512_mul_ph(b_vec, m_recip_approx);
|
|
553
|
-
__m512h log_ratio_a_vec =
|
|
554
|
-
__m512h log_ratio_b_vec =
|
|
582
|
+
__m512h log_ratio_a_vec = _simsimd_log2_f16_sapphire(ratio_a_vec);
|
|
583
|
+
__m512h log_ratio_b_vec = _simsimd_log2_f16_sapphire(ratio_b_vec);
|
|
555
584
|
sum_a_vec = _mm512_maskz_fmadd_ph(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
556
585
|
sum_b_vec = _mm512_maskz_fmadd_ph(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
557
|
-
if (n)
|
|
558
|
-
goto simsimd_js_f16_sapphire_cycle;
|
|
586
|
+
if (n) goto simsimd_js_f16_sapphire_cycle;
|
|
559
587
|
|
|
560
588
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
561
|
-
*result = _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) *
|
|
589
|
+
*result = _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) * log2_normalizer / 2;
|
|
562
590
|
}
|
|
563
591
|
|
|
564
592
|
#pragma clang attribute pop
|
|
565
593
|
#pragma GCC pop_options
|
|
566
594
|
#endif // SIMSIMD_TARGET_SAPPHIRE
|
|
567
|
-
#endif //
|
|
595
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
568
596
|
|
|
569
597
|
#ifdef __cplusplus
|
|
570
598
|
}
|