usearch 2.15.3 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/cpp/test.cpp +16 -0
- package/include/usearch/index.hpp +21 -5
- package/include/usearch/index_dense.hpp +6 -1
- package/include/usearch/index_plugins.hpp +345 -8
- package/javascript/lib.cpp +17 -5
- package/javascript/usearch.test.js +190 -21
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.git-blame-ignore-revs +2 -0
- package/simsimd/CONTRIBUTING.md +71 -23
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +4 -4
- package/simsimd/MANIFEST.in +2 -1
- package/simsimd/Package.swift +15 -3
- package/simsimd/README.md +104 -35
- package/simsimd/binding.gyp +6 -2
- package/simsimd/build.rs +2 -0
- package/simsimd/c/lib.c +180 -99
- package/simsimd/include/simsimd/binary.h +311 -102
- package/simsimd/include/simsimd/curved.h +134 -135
- package/simsimd/include/simsimd/dot.h +448 -274
- package/simsimd/include/simsimd/elementwise.h +1437 -0
- package/simsimd/include/simsimd/geospatial.h +2 -2
- package/simsimd/include/simsimd/mesh.h +2 -2
- package/simsimd/include/simsimd/probability.h +88 -88
- package/simsimd/include/simsimd/simsimd.h +1157 -772
- package/simsimd/include/simsimd/sparse.h +768 -147
- package/simsimd/include/simsimd/spatial.h +955 -376
- package/simsimd/include/simsimd/types.h +145 -71
- package/simsimd/javascript/lib.c +1 -2
- package/simsimd/package.json +4 -4
- package/simsimd/rust/lib.rs +14 -1
- package/simsimd/scripts/bench.cxx +1070 -0
- package/simsimd/scripts/bench.ipynb +187 -0
- package/simsimd/{javascript → scripts}/bench.js +1 -1
- package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
- package/simsimd/scripts/bench_vectors.py +700 -0
- package/simsimd/scripts/bench_vectors_live.py +213 -0
- package/simsimd/scripts/test.c +155 -0
- package/simsimd/{javascript → scripts}/test.mjs +2 -2
- package/simsimd/scripts/test.py +1410 -0
- package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
- package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
- /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
- /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* - 64-bit IEEE-754 floating point
|
|
14
14
|
*
|
|
15
15
|
* For hardware architectures:
|
|
16
|
-
* - Arm
|
|
17
|
-
* - x86
|
|
16
|
+
* - Arm: NEON
|
|
17
|
+
* - x86: Haswell
|
|
18
18
|
*
|
|
19
19
|
* In most cases, for distance computations, we don't need the exact Haversine formula.
|
|
20
20
|
* The very last part of the computation applies `asin(sqrt(x))` non-linear transformation.
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
* - 16-bit brain-floating point
|
|
16
16
|
*
|
|
17
17
|
* For hardware architectures:
|
|
18
|
-
* - Arm
|
|
19
|
-
* - x86
|
|
18
|
+
* - Arm: Neon
|
|
19
|
+
* - x86: Genoa, Sapphire
|
|
20
20
|
*
|
|
21
21
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
22
22
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
* - 16-bit brain-floating point numbers
|
|
15
15
|
*
|
|
16
16
|
* For hardware architectures:
|
|
17
|
-
* - Arm
|
|
18
|
-
* - x86
|
|
17
|
+
* - Arm: NEON
|
|
18
|
+
* - x86: Haswell, Skylake, Sapphire
|
|
19
19
|
*
|
|
20
20
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
21
21
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -83,32 +83,32 @@ SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_
|
|
|
83
83
|
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* divergence);
|
|
84
84
|
// clang-format on
|
|
85
85
|
|
|
86
|
-
#define SIMSIMD_MAKE_KL(name, input_type, accumulator_type, load_and_convert, epsilon)
|
|
87
|
-
SIMSIMD_PUBLIC void simsimd_kl_##input_type##_##name(simsimd_##input_type##_t const*
|
|
88
|
-
simsimd_##input_type##_t const*
|
|
89
|
-
simsimd_distance_t*
|
|
90
|
-
simsimd_##accumulator_type##_t d = 0;
|
|
91
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
92
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i);
|
|
93
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i);
|
|
94
|
-
d += ai * SIMSIMD_LOG((ai + epsilon) / (bi + epsilon));
|
|
95
|
-
}
|
|
96
|
-
*result = (simsimd_distance_t)d;
|
|
86
|
+
#define SIMSIMD_MAKE_KL(name, input_type, accumulator_type, load_and_convert, epsilon) \
|
|
87
|
+
SIMSIMD_PUBLIC void simsimd_kl_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
88
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
89
|
+
simsimd_distance_t *result) { \
|
|
90
|
+
simsimd_##accumulator_type##_t d = 0; \
|
|
91
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
92
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
93
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
94
|
+
d += ai * SIMSIMD_LOG((ai + epsilon) / (bi + epsilon)); \
|
|
95
|
+
} \
|
|
96
|
+
*result = (simsimd_distance_t)d; \
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
#define SIMSIMD_MAKE_JS(name, input_type, accumulator_type, load_and_convert, epsilon)
|
|
100
|
-
SIMSIMD_PUBLIC void simsimd_js_##input_type##_##name(simsimd_##input_type##_t const*
|
|
101
|
-
simsimd_##input_type##_t const*
|
|
102
|
-
simsimd_distance_t*
|
|
103
|
-
simsimd_##accumulator_type##_t d = 0;
|
|
104
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
105
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i);
|
|
106
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i);
|
|
107
|
-
simsimd_##accumulator_type##_t mi = (ai + bi) / 2;
|
|
108
|
-
d += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
109
|
-
d += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
110
|
-
}
|
|
111
|
-
*result = (simsimd_distance_t)d / 2;
|
|
99
|
+
#define SIMSIMD_MAKE_JS(name, input_type, accumulator_type, load_and_convert, epsilon) \
|
|
100
|
+
SIMSIMD_PUBLIC void simsimd_js_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
101
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
102
|
+
simsimd_distance_t *result) { \
|
|
103
|
+
simsimd_##accumulator_type##_t d = 0; \
|
|
104
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
105
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
106
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
107
|
+
simsimd_##accumulator_type##_t mi = (ai + bi) / 2; \
|
|
108
|
+
d += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon)); \
|
|
109
|
+
d += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon)); \
|
|
110
|
+
} \
|
|
111
|
+
*result = (simsimd_distance_t)d / 2; \
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
SIMSIMD_MAKE_KL(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_f64_serial
|
|
@@ -132,7 +132,7 @@ SIMSIMD_MAKE_JS(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_DIVISION_EPS
|
|
|
132
132
|
SIMSIMD_MAKE_KL(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_kl_bf16_accurate
|
|
133
133
|
SIMSIMD_MAKE_JS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_DIVISION_EPSILON) // simsimd_js_bf16_accurate
|
|
134
134
|
|
|
135
|
-
#if
|
|
135
|
+
#if _SIMSIMD_TARGET_ARM
|
|
136
136
|
#if SIMSIMD_TARGET_NEON
|
|
137
137
|
#pragma GCC push_options
|
|
138
138
|
#pragma GCC target("arch=armv8.2-a+simd")
|
|
@@ -163,8 +163,8 @@ SIMSIMD_PUBLIC float32x4_t _simsimd_log2_f32_neon(float32x4_t x) {
|
|
|
163
163
|
return result;
|
|
164
164
|
}
|
|
165
165
|
|
|
166
|
-
SIMSIMD_PUBLIC void simsimd_kl_f32_neon(simsimd_f32_t const*
|
|
167
|
-
simsimd_distance_t*
|
|
166
|
+
SIMSIMD_PUBLIC void simsimd_kl_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
167
|
+
simsimd_distance_t *result) {
|
|
168
168
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
169
169
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
170
170
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
@@ -175,7 +175,8 @@ simsimd_kl_f32_neon_cycle:
|
|
|
175
175
|
a_vec = _simsimd_partial_load_f32x4_neon(a, n);
|
|
176
176
|
b_vec = _simsimd_partial_load_f32x4_neon(b, n);
|
|
177
177
|
n = 0;
|
|
178
|
-
}
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
179
180
|
a_vec = vld1q_f32(a);
|
|
180
181
|
b_vec = vld1q_f32(b);
|
|
181
182
|
n -= 4, a += 4, b += 4;
|
|
@@ -185,16 +186,15 @@ simsimd_kl_f32_neon_cycle:
|
|
|
185
186
|
float32x4_t log_ratio_vec = _simsimd_log2_f32_neon(ratio_vec);
|
|
186
187
|
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
187
188
|
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
188
|
-
if (n != 0)
|
|
189
|
-
goto simsimd_kl_f32_neon_cycle;
|
|
189
|
+
if (n != 0) goto simsimd_kl_f32_neon_cycle;
|
|
190
190
|
|
|
191
191
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
192
192
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
193
193
|
*result = sum;
|
|
194
194
|
}
|
|
195
195
|
|
|
196
|
-
SIMSIMD_PUBLIC void simsimd_js_f32_neon(simsimd_f32_t const*
|
|
197
|
-
simsimd_distance_t*
|
|
196
|
+
SIMSIMD_PUBLIC void simsimd_js_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
197
|
+
simsimd_distance_t *result) {
|
|
198
198
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
199
199
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
200
200
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
@@ -205,7 +205,8 @@ simsimd_js_f32_neon_cycle:
|
|
|
205
205
|
a_vec = _simsimd_partial_load_f32x4_neon(a, n);
|
|
206
206
|
b_vec = _simsimd_partial_load_f32x4_neon(b, n);
|
|
207
207
|
n = 0;
|
|
208
|
-
}
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
209
210
|
a_vec = vld1q_f32(a);
|
|
210
211
|
b_vec = vld1q_f32(b);
|
|
211
212
|
n -= 4, a += 4, b += 4;
|
|
@@ -219,8 +220,7 @@ simsimd_js_f32_neon_cycle:
|
|
|
219
220
|
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
220
221
|
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
221
222
|
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
222
|
-
if (n != 0)
|
|
223
|
-
goto simsimd_js_f32_neon_cycle;
|
|
223
|
+
if (n != 0) goto simsimd_js_f32_neon_cycle;
|
|
224
224
|
|
|
225
225
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
226
226
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
@@ -236,8 +236,8 @@ simsimd_js_f32_neon_cycle:
|
|
|
236
236
|
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
237
237
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
238
238
|
|
|
239
|
-
SIMSIMD_PUBLIC void simsimd_kl_f16_neon(simsimd_f16_t const*
|
|
240
|
-
simsimd_distance_t*
|
|
239
|
+
SIMSIMD_PUBLIC void simsimd_kl_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
240
|
+
simsimd_distance_t *result) {
|
|
241
241
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
242
242
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
243
243
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
@@ -248,9 +248,10 @@ simsimd_kl_f16_neon_cycle:
|
|
|
248
248
|
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
249
249
|
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
250
250
|
n = 0;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
254
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
254
255
|
n -= 4, a += 4, b += 4;
|
|
255
256
|
}
|
|
256
257
|
|
|
@@ -258,16 +259,15 @@ simsimd_kl_f16_neon_cycle:
|
|
|
258
259
|
float32x4_t log_ratio_vec = _simsimd_log2_f32_neon(ratio_vec);
|
|
259
260
|
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
260
261
|
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
261
|
-
if (n)
|
|
262
|
-
goto simsimd_kl_f16_neon_cycle;
|
|
262
|
+
if (n) goto simsimd_kl_f16_neon_cycle;
|
|
263
263
|
|
|
264
264
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
265
265
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
266
266
|
*result = sum;
|
|
267
267
|
}
|
|
268
268
|
|
|
269
|
-
SIMSIMD_PUBLIC void simsimd_js_f16_neon(simsimd_f16_t const*
|
|
270
|
-
simsimd_distance_t*
|
|
269
|
+
SIMSIMD_PUBLIC void simsimd_js_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
270
|
+
simsimd_distance_t *result) {
|
|
271
271
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
272
272
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
273
273
|
float32x4_t epsilon_vec = vdupq_n_f32(epsilon);
|
|
@@ -278,9 +278,10 @@ simsimd_js_f16_neon_cycle:
|
|
|
278
278
|
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
279
279
|
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
280
280
|
n = 0;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
|
|
281
|
+
}
|
|
282
|
+
else {
|
|
283
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
284
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
284
285
|
n -= 4, a += 4, b += 4;
|
|
285
286
|
}
|
|
286
287
|
|
|
@@ -292,8 +293,7 @@ simsimd_js_f16_neon_cycle:
|
|
|
292
293
|
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
293
294
|
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
294
295
|
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
295
|
-
if (n)
|
|
296
|
-
goto simsimd_js_f16_neon_cycle;
|
|
296
|
+
if (n) goto simsimd_js_f16_neon_cycle;
|
|
297
297
|
|
|
298
298
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
299
299
|
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
@@ -303,9 +303,9 @@ simsimd_js_f16_neon_cycle:
|
|
|
303
303
|
#pragma clang attribute pop
|
|
304
304
|
#pragma GCC pop_options
|
|
305
305
|
#endif // SIMSIMD_TARGET_NEON_F16
|
|
306
|
-
#endif //
|
|
306
|
+
#endif // _SIMSIMD_TARGET_ARM
|
|
307
307
|
|
|
308
|
-
#if
|
|
308
|
+
#if _SIMSIMD_TARGET_X86
|
|
309
309
|
#if SIMSIMD_TARGET_HASWELL
|
|
310
310
|
#pragma GCC push_options
|
|
311
311
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
@@ -338,8 +338,8 @@ SIMSIMD_INTERNAL __m256 _simsimd_log2_f32_haswell(__m256 x) {
|
|
|
338
338
|
return result;
|
|
339
339
|
}
|
|
340
340
|
|
|
341
|
-
SIMSIMD_PUBLIC void simsimd_kl_f16_haswell(simsimd_f16_t const*
|
|
342
|
-
simsimd_distance_t*
|
|
341
|
+
SIMSIMD_PUBLIC void simsimd_kl_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
342
|
+
simsimd_distance_t *result) {
|
|
343
343
|
__m256 sum_vec = _mm256_setzero_ps();
|
|
344
344
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
345
345
|
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
@@ -350,9 +350,10 @@ simsimd_kl_f16_haswell_cycle:
|
|
|
350
350
|
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
351
351
|
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
352
352
|
n = 0;
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
|
|
353
|
+
}
|
|
354
|
+
else {
|
|
355
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
356
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
356
357
|
n -= 8, a += 8, b += 8;
|
|
357
358
|
}
|
|
358
359
|
a_vec = _mm256_add_ps(a_vec, epsilon_vec);
|
|
@@ -361,8 +362,7 @@ simsimd_kl_f16_haswell_cycle:
|
|
|
361
362
|
__m256 log_ratio_vec = _simsimd_log2_f32_haswell(ratio_vec);
|
|
362
363
|
__m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
|
|
363
364
|
sum_vec = _mm256_add_ps(sum_vec, prod_vec);
|
|
364
|
-
if (n)
|
|
365
|
-
goto simsimd_kl_f16_haswell_cycle;
|
|
365
|
+
if (n) goto simsimd_kl_f16_haswell_cycle;
|
|
366
366
|
|
|
367
367
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
368
368
|
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
@@ -370,8 +370,8 @@ simsimd_kl_f16_haswell_cycle:
|
|
|
370
370
|
*result = sum;
|
|
371
371
|
}
|
|
372
372
|
|
|
373
|
-
SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const*
|
|
374
|
-
simsimd_distance_t*
|
|
373
|
+
SIMSIMD_PUBLIC void simsimd_js_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
374
|
+
simsimd_distance_t *result) {
|
|
375
375
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
376
376
|
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
377
377
|
__m256 sum_vec = _mm256_setzero_ps();
|
|
@@ -382,9 +382,10 @@ simsimd_js_f16_haswell_cycle:
|
|
|
382
382
|
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
383
383
|
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
384
384
|
n = 0;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
388
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
388
389
|
n -= 8, a += 8, b += 8;
|
|
389
390
|
}
|
|
390
391
|
a_vec = _mm256_add_ps(a_vec, epsilon_vec);
|
|
@@ -398,8 +399,7 @@ simsimd_js_f16_haswell_cycle:
|
|
|
398
399
|
__m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
|
|
399
400
|
sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
|
|
400
401
|
sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
|
|
401
|
-
if (n)
|
|
402
|
-
goto simsimd_js_f16_haswell_cycle;
|
|
402
|
+
if (n) goto simsimd_js_f16_haswell_cycle;
|
|
403
403
|
|
|
404
404
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
405
405
|
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
@@ -433,8 +433,8 @@ SIMSIMD_INTERNAL __m512 _simsimd_log2_f32_skylake(__m512 x) {
|
|
|
433
433
|
return _mm512_add_ps(_mm512_mul_ps(p, _mm512_sub_ps(m, one)), e);
|
|
434
434
|
}
|
|
435
435
|
|
|
436
|
-
SIMSIMD_PUBLIC void simsimd_kl_f32_skylake(simsimd_f32_t const*
|
|
437
|
-
simsimd_distance_t*
|
|
436
|
+
SIMSIMD_PUBLIC void simsimd_kl_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
437
|
+
simsimd_distance_t *result) {
|
|
438
438
|
__m512 sum_vec = _mm512_setzero();
|
|
439
439
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
440
440
|
__m512 epsilon_vec = _mm512_set1_ps(epsilon);
|
|
@@ -446,7 +446,8 @@ simsimd_kl_f32_skylake_cycle:
|
|
|
446
446
|
a_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, a), epsilon_vec);
|
|
447
447
|
b_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, b), epsilon_vec);
|
|
448
448
|
n = 0;
|
|
449
|
-
}
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
450
451
|
a_vec = _mm512_add_ps(_mm512_loadu_ps(a), epsilon_vec);
|
|
451
452
|
b_vec = _mm512_add_ps(_mm512_loadu_ps(b), epsilon_vec);
|
|
452
453
|
a += 16, b += 16, n -= 16;
|
|
@@ -455,15 +456,14 @@ simsimd_kl_f32_skylake_cycle:
|
|
|
455
456
|
__m512 log_ratio_vec = _simsimd_log2_f32_skylake(ratio_vec);
|
|
456
457
|
__m512 prod_vec = _mm512_mul_ps(a_vec, log_ratio_vec);
|
|
457
458
|
sum_vec = _mm512_add_ps(sum_vec, prod_vec);
|
|
458
|
-
if (n)
|
|
459
|
-
goto simsimd_kl_f32_skylake_cycle;
|
|
459
|
+
if (n) goto simsimd_kl_f32_skylake_cycle;
|
|
460
460
|
|
|
461
461
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
462
462
|
*result = _mm512_reduce_add_ps(sum_vec) * log2_normalizer;
|
|
463
463
|
}
|
|
464
464
|
|
|
465
|
-
SIMSIMD_PUBLIC void simsimd_js_f32_skylake(simsimd_f32_t const*
|
|
466
|
-
simsimd_distance_t*
|
|
465
|
+
SIMSIMD_PUBLIC void simsimd_js_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
466
|
+
simsimd_distance_t *result) {
|
|
467
467
|
__m512 sum_a_vec = _mm512_setzero();
|
|
468
468
|
__m512 sum_b_vec = _mm512_setzero();
|
|
469
469
|
simsimd_f32_t epsilon = SIMSIMD_F32_DIVISION_EPSILON;
|
|
@@ -476,7 +476,8 @@ simsimd_js_f32_skylake_cycle:
|
|
|
476
476
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
477
477
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
478
478
|
n = 0;
|
|
479
|
-
}
|
|
479
|
+
}
|
|
480
|
+
else {
|
|
480
481
|
a_vec = _mm512_loadu_ps(a);
|
|
481
482
|
b_vec = _mm512_loadu_ps(b);
|
|
482
483
|
a += 16, b += 16, n -= 16;
|
|
@@ -492,8 +493,7 @@ simsimd_js_f32_skylake_cycle:
|
|
|
492
493
|
__m512 log_ratio_b_vec = _simsimd_log2_f32_skylake(ratio_b_vec);
|
|
493
494
|
sum_a_vec = _mm512_maskz_fmadd_ps(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
494
495
|
sum_b_vec = _mm512_maskz_fmadd_ps(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
495
|
-
if (n)
|
|
496
|
-
goto simsimd_js_f32_skylake_cycle;
|
|
496
|
+
if (n) goto simsimd_js_f32_skylake_cycle;
|
|
497
497
|
|
|
498
498
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
499
499
|
*result = _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) * log2_normalizer / 2;
|
|
@@ -525,8 +525,8 @@ SIMSIMD_INTERNAL __m512h _simsimd_log2_f16_sapphire(__m512h x) {
|
|
|
525
525
|
return _mm512_add_ph(_mm512_mul_ph(p, _mm512_sub_ph(m, one)), e);
|
|
526
526
|
}
|
|
527
527
|
|
|
528
|
-
SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const*
|
|
529
|
-
simsimd_distance_t*
|
|
528
|
+
SIMSIMD_PUBLIC void simsimd_kl_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
529
|
+
simsimd_distance_t *result) {
|
|
530
530
|
__m512h sum_vec = _mm512_setzero_ph();
|
|
531
531
|
__m512h epsilon_vec = _mm512_set1_ph((simsimd_f16_t)SIMSIMD_F16_DIVISION_EPSILON);
|
|
532
532
|
__m512h a_vec, b_vec;
|
|
@@ -537,7 +537,8 @@ simsimd_kl_f16_sapphire_cycle:
|
|
|
537
537
|
a_vec = _mm512_maskz_add_ph(mask, _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a)), epsilon_vec);
|
|
538
538
|
b_vec = _mm512_maskz_add_ph(mask, _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b)), epsilon_vec);
|
|
539
539
|
n = 0;
|
|
540
|
-
}
|
|
540
|
+
}
|
|
541
|
+
else {
|
|
541
542
|
a_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_loadu_epi16(a)), epsilon_vec);
|
|
542
543
|
b_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_loadu_epi16(b)), epsilon_vec);
|
|
543
544
|
a += 32, b += 32, n -= 32;
|
|
@@ -546,15 +547,14 @@ simsimd_kl_f16_sapphire_cycle:
|
|
|
546
547
|
__m512h log_ratio_vec = _simsimd_log2_f16_sapphire(ratio_vec);
|
|
547
548
|
__m512h prod_vec = _mm512_mul_ph(a_vec, log_ratio_vec);
|
|
548
549
|
sum_vec = _mm512_add_ph(sum_vec, prod_vec);
|
|
549
|
-
if (n)
|
|
550
|
-
goto simsimd_kl_f16_sapphire_cycle;
|
|
550
|
+
if (n) goto simsimd_kl_f16_sapphire_cycle;
|
|
551
551
|
|
|
552
552
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
553
553
|
*result = _mm512_reduce_add_ph(sum_vec) * log2_normalizer;
|
|
554
554
|
}
|
|
555
555
|
|
|
556
|
-
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const*
|
|
557
|
-
simsimd_distance_t*
|
|
556
|
+
SIMSIMD_PUBLIC void simsimd_js_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
557
|
+
simsimd_distance_t *result) {
|
|
558
558
|
__m512h sum_a_vec = _mm512_setzero_ph();
|
|
559
559
|
__m512h sum_b_vec = _mm512_setzero_ph();
|
|
560
560
|
__m512h epsilon_vec = _mm512_set1_ph((simsimd_f16_t)SIMSIMD_F16_DIVISION_EPSILON);
|
|
@@ -566,7 +566,8 @@ simsimd_js_f16_sapphire_cycle:
|
|
|
566
566
|
a_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
567
567
|
b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
568
568
|
n = 0;
|
|
569
|
-
}
|
|
569
|
+
}
|
|
570
|
+
else {
|
|
570
571
|
a_vec = _mm512_castsi512_ph(_mm512_loadu_epi16(a));
|
|
571
572
|
b_vec = _mm512_castsi512_ph(_mm512_loadu_epi16(b));
|
|
572
573
|
a += 32, b += 32, n -= 32;
|
|
@@ -582,8 +583,7 @@ simsimd_js_f16_sapphire_cycle:
|
|
|
582
583
|
__m512h log_ratio_b_vec = _simsimd_log2_f16_sapphire(ratio_b_vec);
|
|
583
584
|
sum_a_vec = _mm512_maskz_fmadd_ph(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
584
585
|
sum_b_vec = _mm512_maskz_fmadd_ph(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
585
|
-
if (n)
|
|
586
|
-
goto simsimd_js_f16_sapphire_cycle;
|
|
586
|
+
if (n) goto simsimd_js_f16_sapphire_cycle;
|
|
587
587
|
|
|
588
588
|
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
589
589
|
*result = _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) * log2_normalizer / 2;
|
|
@@ -592,7 +592,7 @@ simsimd_js_f16_sapphire_cycle:
|
|
|
592
592
|
#pragma clang attribute pop
|
|
593
593
|
#pragma GCC pop_options
|
|
594
594
|
#endif // SIMSIMD_TARGET_SAPPHIRE
|
|
595
|
-
#endif //
|
|
595
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
596
596
|
|
|
597
597
|
#ifdef __cplusplus
|
|
598
598
|
}
|