usearch 2.15.2 → 2.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.gyp +1 -1
- package/c/CMakeLists.txt +10 -3
- package/cpp/README.md +1 -1
- package/cpp/bench.cpp +2 -2
- package/include/usearch/index.hpp +24 -13
- package/include/usearch/index_dense.hpp +45 -88
- package/include/usearch/index_plugins.hpp +126 -38
- package/javascript/usearch.test.js +15 -13
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/CODE_OF_CONDUCT.md +128 -0
- package/simsimd/CONTRIBUTING.md +18 -8
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +1 -1
- package/simsimd/MANIFEST.in +2 -0
- package/simsimd/README.md +446 -158
- package/simsimd/build.rs +11 -1
- package/simsimd/c/lib.c +172 -40
- package/simsimd/include/simsimd/binary.h +11 -11
- package/simsimd/include/simsimd/curved.h +804 -0
- package/simsimd/include/simsimd/dot.h +333 -334
- package/simsimd/include/simsimd/geospatial.h +9 -3
- package/simsimd/include/simsimd/mesh.h +69 -0
- package/simsimd/include/simsimd/probability.h +178 -150
- package/simsimd/include/simsimd/simsimd.h +581 -107
- package/simsimd/include/simsimd/sparse.h +807 -0
- package/simsimd/include/simsimd/spatial.h +737 -502
- package/simsimd/include/simsimd/types.h +145 -75
- package/simsimd/javascript/simsimd.ts +4 -4
- package/simsimd/package.json +1 -1
- package/simsimd/rust/lib.rs +165 -0
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* - 8-bit signed integers
|
|
17
17
|
*
|
|
18
18
|
* For hardware architectures:
|
|
19
|
-
* - Arm (NEON, SVE
|
|
20
|
-
* - x86 (AVX2
|
|
19
|
+
* - Arm (NEON, SVE)
|
|
20
|
+
* - x86 (AVX2, AVX512)
|
|
21
21
|
*
|
|
22
22
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
23
23
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -82,7 +82,11 @@ SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t c
|
|
|
82
82
|
SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
83
83
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
84
84
|
|
|
85
|
-
SIMSIMD_PUBLIC void
|
|
85
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
86
|
+
|
|
87
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
88
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
89
|
+
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
86
90
|
|
|
87
91
|
/* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
|
|
88
92
|
* Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
|
|
@@ -131,7 +135,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_
|
|
|
131
135
|
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
132
136
|
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
133
137
|
|
|
134
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
|
|
138
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
135
139
|
|
|
136
140
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
137
141
|
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
@@ -142,29 +146,29 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f1
|
|
|
142
146
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
143
147
|
// clang-format on
|
|
144
148
|
|
|
145
|
-
#define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type,
|
|
149
|
+
#define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, load_and_convert) \
|
|
146
150
|
SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const* a, \
|
|
147
151
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
148
152
|
simsimd_distance_t* result) { \
|
|
149
153
|
simsimd_##accumulator_type##_t ab = 0; \
|
|
150
154
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
151
|
-
simsimd_##accumulator_type##_t ai =
|
|
152
|
-
simsimd_##accumulator_type##_t bi =
|
|
155
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
156
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
153
157
|
ab += ai * bi; \
|
|
154
158
|
} \
|
|
155
159
|
*result = ab; \
|
|
156
160
|
}
|
|
157
161
|
|
|
158
|
-
#define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type,
|
|
162
|
+
#define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, load_and_convert) \
|
|
159
163
|
SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const* a, \
|
|
160
164
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
161
165
|
simsimd_distance_t* results) { \
|
|
162
166
|
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
|
|
163
167
|
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
|
|
164
|
-
simsimd_##accumulator_type##_t ar =
|
|
165
|
-
simsimd_##accumulator_type##_t br =
|
|
166
|
-
simsimd_##accumulator_type##_t ai =
|
|
167
|
-
simsimd_##accumulator_type##_t bi =
|
|
168
|
+
simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
|
|
169
|
+
simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
|
|
170
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
|
|
171
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
|
|
168
172
|
ab_real += ar * br - ai * bi; \
|
|
169
173
|
ab_imag += ar * bi + ai * br; \
|
|
170
174
|
} \
|
|
@@ -172,16 +176,16 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f
|
|
|
172
176
|
results[1] = ab_imag; \
|
|
173
177
|
}
|
|
174
178
|
|
|
175
|
-
#define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type,
|
|
179
|
+
#define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, load_and_convert) \
|
|
176
180
|
SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const* a, \
|
|
177
181
|
simsimd_##input_type##_t const* b, simsimd_size_t n, \
|
|
178
182
|
simsimd_distance_t* results) { \
|
|
179
183
|
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
|
|
180
184
|
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
|
|
181
|
-
simsimd_##accumulator_type##_t ar =
|
|
182
|
-
simsimd_##accumulator_type##_t br =
|
|
183
|
-
simsimd_##accumulator_type##_t ai =
|
|
184
|
-
simsimd_##accumulator_type##_t bi =
|
|
185
|
+
simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
|
|
186
|
+
simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
|
|
187
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
|
|
188
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
|
|
185
189
|
ab_real += ar * br + ai * bi; \
|
|
186
190
|
ab_imag += ar * bi - ai * br; \
|
|
187
191
|
} \
|
|
@@ -189,41 +193,54 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f
|
|
|
189
193
|
results[1] = ab_imag; \
|
|
190
194
|
}
|
|
191
195
|
|
|
192
|
-
SIMSIMD_MAKE_DOT(serial, f64, f64,
|
|
193
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64,
|
|
194
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64,
|
|
196
|
+
SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64_serial
|
|
197
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64c_serial
|
|
198
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f64c_serial
|
|
195
199
|
|
|
196
|
-
SIMSIMD_MAKE_DOT(serial, f32, f32,
|
|
197
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32,
|
|
198
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32,
|
|
200
|
+
SIMSIMD_MAKE_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_serial
|
|
201
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_serial
|
|
202
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_serial
|
|
199
203
|
|
|
200
|
-
SIMSIMD_MAKE_DOT(serial, f16, f32,
|
|
201
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32,
|
|
202
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32,
|
|
204
|
+
SIMSIMD_MAKE_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_serial
|
|
205
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_serial
|
|
206
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_serial
|
|
203
207
|
|
|
204
|
-
SIMSIMD_MAKE_DOT(serial, bf16, f32,
|
|
205
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32,
|
|
206
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32,
|
|
208
|
+
SIMSIMD_MAKE_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_serial
|
|
209
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_serial
|
|
210
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_serial
|
|
207
211
|
|
|
208
|
-
SIMSIMD_MAKE_DOT(serial, i8, i64,
|
|
212
|
+
SIMSIMD_MAKE_DOT(serial, i8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_i8_serial
|
|
209
213
|
|
|
210
|
-
SIMSIMD_MAKE_DOT(accurate, f32, f64,
|
|
211
|
-
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64,
|
|
212
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64,
|
|
214
|
+
SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_accurate
|
|
215
|
+
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_accurate
|
|
216
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_accurate
|
|
213
217
|
|
|
214
|
-
SIMSIMD_MAKE_DOT(accurate, f16, f64,
|
|
215
|
-
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64,
|
|
216
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64,
|
|
218
|
+
SIMSIMD_MAKE_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_accurate
|
|
219
|
+
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_accurate
|
|
220
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_accurate
|
|
217
221
|
|
|
218
|
-
SIMSIMD_MAKE_DOT(accurate, bf16, f64,
|
|
219
|
-
SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64,
|
|
220
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64,
|
|
222
|
+
SIMSIMD_MAKE_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_accurate
|
|
223
|
+
SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_accurate
|
|
224
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_accurate
|
|
221
225
|
|
|
222
226
|
#if SIMSIMD_TARGET_ARM
|
|
223
227
|
#if SIMSIMD_TARGET_NEON
|
|
224
228
|
#pragma GCC push_options
|
|
225
|
-
#pragma GCC target("+simd")
|
|
226
|
-
#pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
|
|
229
|
+
#pragma GCC target("arch=armv8.2-a+simd")
|
|
230
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
231
|
+
|
|
232
|
+
SIMSIMD_INTERNAL float32x4_t _simsimd_partial_load_f32x4_neon(simsimd_f32_t const* a, simsimd_size_t n) {
|
|
233
|
+
union {
|
|
234
|
+
float32x4_t vec;
|
|
235
|
+
simsimd_f32_t scalars[4];
|
|
236
|
+
} result;
|
|
237
|
+
simsimd_size_t i = 0;
|
|
238
|
+
for (; i < n; ++i)
|
|
239
|
+
result.scalars[i] = a[i];
|
|
240
|
+
for (; i < 4; ++i)
|
|
241
|
+
result.scalars[i] = 0;
|
|
242
|
+
return result.vec;
|
|
243
|
+
}
|
|
227
244
|
|
|
228
245
|
SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
229
246
|
simsimd_distance_t* result) {
|
|
@@ -349,36 +366,48 @@ SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
349
366
|
|
|
350
367
|
#pragma clang attribute pop
|
|
351
368
|
#pragma GCC pop_options
|
|
369
|
+
#endif
|
|
352
370
|
|
|
371
|
+
#if SIMSIMD_TARGET_NEON_F16
|
|
353
372
|
#pragma GCC push_options
|
|
354
|
-
#pragma GCC target("+simd+fp16")
|
|
355
|
-
#pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
|
|
373
|
+
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
374
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
375
|
+
|
|
376
|
+
SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t const* a, simsimd_size_t n) {
|
|
377
|
+
// In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
|
|
378
|
+
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
379
|
+
// loads and stores with vectorized math.
|
|
380
|
+
union {
|
|
381
|
+
float16x4_t vec;
|
|
382
|
+
simsimd_f16_t scalars[4];
|
|
383
|
+
} result;
|
|
384
|
+
simsimd_size_t i = 0;
|
|
385
|
+
for (; i < n; ++i)
|
|
386
|
+
result.scalars[i] = a[i];
|
|
387
|
+
for (; i < 4; ++i)
|
|
388
|
+
result.scalars[i] = 0;
|
|
389
|
+
return result.vec;
|
|
390
|
+
}
|
|
356
391
|
|
|
357
392
|
SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
358
393
|
simsimd_distance_t* result) {
|
|
359
394
|
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
395
|
+
float32x4_t a_vec, b_vec;
|
|
360
396
|
simsimd_size_t i = 0;
|
|
361
|
-
for (; i + 4 <= n; i += 4) {
|
|
362
|
-
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
|
|
363
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
|
|
364
|
-
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
365
|
-
}
|
|
366
397
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
for (; i < n; ++i, ++j)
|
|
377
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
378
|
-
for (; j < 4; ++j)
|
|
379
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
380
|
-
ab_vec = vfmaq_f32(ab_vec, vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
|
|
398
|
+
simsimd_dot_f16_neon_cycle:
|
|
399
|
+
if (n < 4) {
|
|
400
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
401
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
402
|
+
n = 0;
|
|
403
|
+
} else {
|
|
404
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
|
|
405
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
|
|
406
|
+
a += 4, b += 4, n -= 4;
|
|
381
407
|
}
|
|
408
|
+
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
409
|
+
if (n)
|
|
410
|
+
goto simsimd_dot_f16_neon_cycle;
|
|
382
411
|
*result = vaddvq_f32(ab_vec);
|
|
383
412
|
}
|
|
384
413
|
|
|
@@ -390,13 +419,13 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
390
419
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
391
420
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
392
421
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
393
|
-
|
|
394
|
-
|
|
422
|
+
|
|
423
|
+
while (n >= 8) {
|
|
395
424
|
// Unpack the input arrays into real and imaginary parts.
|
|
396
425
|
// MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
|
|
397
426
|
// integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
|
|
398
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
399
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
427
|
+
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
428
|
+
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
400
429
|
float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
|
|
401
430
|
float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
|
|
402
431
|
float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
|
|
@@ -407,20 +436,14 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
407
436
|
ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
408
437
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
409
438
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
// Reduce horizontal sums:
|
|
413
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
414
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
415
439
|
|
|
416
|
-
|
|
417
|
-
for (; i + 2 <= n; i += 2) {
|
|
418
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
419
|
-
ab_real += ar * br - ai * bi;
|
|
420
|
-
ab_imag += ar * bi + ai * br;
|
|
440
|
+
n -= 8, a += 8, b += 8;
|
|
421
441
|
}
|
|
422
|
-
|
|
423
|
-
|
|
442
|
+
|
|
443
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
444
|
+
simsimd_dot_f16c_serial(a, b, n, results);
|
|
445
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
446
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
424
447
|
}
|
|
425
448
|
|
|
426
449
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, //
|
|
@@ -431,13 +454,13 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
431
454
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
432
455
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
433
456
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
434
|
-
|
|
435
|
-
|
|
457
|
+
|
|
458
|
+
while (n >= 8) {
|
|
436
459
|
// Unpack the input arrays into real and imaginary parts.
|
|
437
460
|
// MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
|
|
438
461
|
// integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
|
|
439
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
440
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
462
|
+
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
463
|
+
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
441
464
|
float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
|
|
442
465
|
float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
|
|
443
466
|
float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
|
|
@@ -448,62 +471,61 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
448
471
|
ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
449
472
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
450
473
|
ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
451
|
-
}
|
|
452
474
|
|
|
453
|
-
|
|
454
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
455
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
456
|
-
|
|
457
|
-
// Handle the tail:
|
|
458
|
-
for (; i + 2 <= n; i += 2) {
|
|
459
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
460
|
-
ab_real += ar * br + ai * bi;
|
|
461
|
-
ab_imag += ar * bi - ai * br;
|
|
475
|
+
n -= 8, a += 8, b += 8;
|
|
462
476
|
}
|
|
463
|
-
|
|
464
|
-
|
|
477
|
+
|
|
478
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
479
|
+
simsimd_vdot_f16c_serial(a, b, n, results);
|
|
480
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
481
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
465
482
|
}
|
|
466
483
|
|
|
467
484
|
#pragma clang attribute pop
|
|
468
485
|
#pragma GCC pop_options
|
|
469
|
-
#endif //
|
|
486
|
+
#endif // SIMSIMD_TARGET_NEON_F16
|
|
470
487
|
|
|
471
488
|
#if SIMSIMD_TARGET_NEON_BF16
|
|
472
489
|
#pragma GCC push_options
|
|
473
490
|
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
474
491
|
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
475
492
|
|
|
493
|
+
SIMSIMD_INTERNAL bfloat16x8_t _simsimd_partial_load_bf16x8_neon(simsimd_bf16_t const* a, simsimd_size_t n) {
|
|
494
|
+
union {
|
|
495
|
+
bfloat16x8_t vec;
|
|
496
|
+
simsimd_bf16_t scalars[8];
|
|
497
|
+
} result;
|
|
498
|
+
simsimd_size_t i = 0;
|
|
499
|
+
for (; i < n; ++i)
|
|
500
|
+
result.scalars[i] = a[i];
|
|
501
|
+
for (; i < 8; ++i)
|
|
502
|
+
result.scalars[i] = 0;
|
|
503
|
+
return result.vec;
|
|
504
|
+
}
|
|
505
|
+
|
|
476
506
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
477
507
|
simsimd_distance_t* result) {
|
|
478
|
-
float32x4_t
|
|
479
|
-
|
|
480
|
-
for (; i + 8 <= n; i += 8) {
|
|
481
|
-
bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
|
|
482
|
-
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
|
|
483
|
-
ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
|
|
484
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
|
|
485
|
-
}
|
|
508
|
+
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
509
|
+
bfloat16x8_t a_vec, b_vec;
|
|
486
510
|
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
for (; i < n; ++i, ++j)
|
|
497
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
498
|
-
for (; j < 8; ++j)
|
|
499
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
500
|
-
ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
501
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
511
|
+
simsimd_dot_bf16_neon_cycle:
|
|
512
|
+
if (n < 8) {
|
|
513
|
+
a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
514
|
+
b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
515
|
+
n = 0;
|
|
516
|
+
} else {
|
|
517
|
+
a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
|
|
518
|
+
b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
|
|
519
|
+
a += 8, b += 8, n -= 8;
|
|
502
520
|
}
|
|
503
|
-
|
|
521
|
+
ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
|
|
522
|
+
if (n)
|
|
523
|
+
goto simsimd_dot_bf16_neon_cycle;
|
|
524
|
+
|
|
525
|
+
*result = vaddvq_f32(ab_vec);
|
|
504
526
|
}
|
|
505
527
|
|
|
506
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
528
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
507
529
|
simsimd_distance_t* results) {
|
|
508
530
|
|
|
509
531
|
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
@@ -511,13 +533,13 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
511
533
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
512
534
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
513
535
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
514
|
-
|
|
515
|
-
|
|
536
|
+
|
|
537
|
+
while (n >= 8) {
|
|
516
538
|
// Unpack the input arrays into real and imaginary parts.
|
|
517
539
|
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
518
540
|
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
519
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
520
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
541
|
+
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
542
|
+
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
521
543
|
float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
|
|
522
544
|
float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
|
|
523
545
|
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
@@ -528,20 +550,14 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
528
550
|
ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
529
551
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
530
552
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
// Reduce horizontal sums:
|
|
534
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
535
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
536
553
|
|
|
537
|
-
|
|
538
|
-
for (; i + 2 <= n; i += 2) {
|
|
539
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
540
|
-
ab_real += ar * br - ai * bi;
|
|
541
|
-
ab_imag += ar * bi + ai * br;
|
|
554
|
+
n -= 8, a += 8, b += 8;
|
|
542
555
|
}
|
|
543
|
-
|
|
544
|
-
|
|
556
|
+
|
|
557
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
558
|
+
simsimd_dot_bf16c_serial(a, b, n, results);
|
|
559
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
560
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
545
561
|
}
|
|
546
562
|
|
|
547
563
|
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, //
|
|
@@ -552,13 +568,13 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
552
568
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
553
569
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
554
570
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
555
|
-
|
|
556
|
-
|
|
571
|
+
|
|
572
|
+
while (n >= 8) {
|
|
557
573
|
// Unpack the input arrays into real and imaginary parts.
|
|
558
574
|
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
559
575
|
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
560
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
561
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
576
|
+
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
577
|
+
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
562
578
|
float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
|
|
563
579
|
float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
|
|
564
580
|
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
@@ -569,20 +585,14 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
569
585
|
ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
570
586
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
571
587
|
ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
572
|
-
}
|
|
573
588
|
|
|
574
|
-
|
|
575
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
576
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
577
|
-
|
|
578
|
-
// Handle the tail:
|
|
579
|
-
for (; i + 2 <= n; i += 2) {
|
|
580
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
581
|
-
ab_real += ar * br + ai * bi;
|
|
582
|
-
ab_imag += ar * bi - ai * br;
|
|
589
|
+
n -= 8, a += 8, b += 8;
|
|
583
590
|
}
|
|
584
|
-
|
|
585
|
-
|
|
591
|
+
|
|
592
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
593
|
+
simsimd_vdot_bf16c_serial(a, b, n, results);
|
|
594
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
595
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
586
596
|
}
|
|
587
597
|
|
|
588
598
|
#pragma clang attribute pop
|
|
@@ -592,8 +602,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
592
602
|
#if SIMSIMD_TARGET_SVE
|
|
593
603
|
|
|
594
604
|
#pragma GCC push_options
|
|
595
|
-
#pragma GCC target("+sve")
|
|
596
|
-
#pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
|
|
605
|
+
#pragma GCC target("arch=armv8.2-a+sve")
|
|
606
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
|
|
597
607
|
|
|
598
608
|
SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
599
609
|
simsimd_distance_t* result) {
|
|
@@ -719,8 +729,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t
|
|
|
719
729
|
#pragma GCC pop_options
|
|
720
730
|
|
|
721
731
|
#pragma GCC push_options
|
|
722
|
-
#pragma GCC target("+sve+fp16")
|
|
723
|
-
#pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
|
|
732
|
+
#pragma GCC target("arch=armv8.2-a+sve+fp16")
|
|
733
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
|
|
724
734
|
|
|
725
735
|
SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
|
|
726
736
|
simsimd_distance_t* result) {
|
|
@@ -796,44 +806,7 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
796
806
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
797
807
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
798
808
|
|
|
799
|
-
|
|
800
|
-
simsimd_distance_t* result) {
|
|
801
|
-
__m256 ab_vec = _mm256_setzero_ps();
|
|
802
|
-
simsimd_size_t i = 0;
|
|
803
|
-
for (; i + 8 <= n; i += 8) {
|
|
804
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
805
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
806
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
// In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
|
|
810
|
-
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
811
|
-
// loads and stores with vectorized math.
|
|
812
|
-
if (i < n) {
|
|
813
|
-
union {
|
|
814
|
-
__m128i f16_vec;
|
|
815
|
-
simsimd_f16_t f16[8];
|
|
816
|
-
} a_padded_tail, b_padded_tail;
|
|
817
|
-
simsimd_size_t j = 0;
|
|
818
|
-
for (; i < n; ++i, ++j)
|
|
819
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
820
|
-
for (; j < 8; ++j)
|
|
821
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
822
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
823
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
824
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
825
|
-
}
|
|
826
|
-
|
|
827
|
-
ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
|
|
828
|
-
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
829
|
-
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
830
|
-
|
|
831
|
-
simsimd_f32_t f32_result;
|
|
832
|
-
_mm_store_ss(&f32_result, _mm256_castps256_ps128(ab_vec));
|
|
833
|
-
*result = f32_result;
|
|
834
|
-
}
|
|
835
|
-
|
|
836
|
-
inline simsimd_f64_t _mm256_reduce_add_ps_dbl(__m256 vec) {
|
|
809
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x8_haswell(__m256 vec) {
|
|
837
810
|
// Convert the lower and higher 128-bit lanes of the input vector to double precision
|
|
838
811
|
__m128 low_f32 = _mm256_castps256_ps128(vec);
|
|
839
812
|
__m128 high_f32 = _mm256_extractf128_ps(vec, 1);
|
|
@@ -868,7 +841,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
868
841
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
869
842
|
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
870
843
|
}
|
|
871
|
-
simsimd_f64_t ab =
|
|
844
|
+
simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
872
845
|
for (; i < n; ++i)
|
|
873
846
|
ab += a[i] * b[i];
|
|
874
847
|
*results = ab;
|
|
@@ -903,7 +876,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
903
876
|
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
904
877
|
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
905
878
|
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
906
|
-
// one of the vectors.
|
|
879
|
+
// one of the vectors. Moreover, `XOR` can be placed after the primary loop.
|
|
907
880
|
// Both operations are quite cheap, and the throughput doubles from 2.5 GB/s to 5 GB/s.
|
|
908
881
|
__m256 ab_real_vec = _mm256_setzero_ps();
|
|
909
882
|
__m256 ab_imag_vec = _mm256_setzero_ps();
|
|
@@ -922,15 +895,17 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
922
895
|
for (; i + 8 <= n; i += 8) {
|
|
923
896
|
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
924
897
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
925
|
-
__m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
|
|
926
898
|
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
927
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec,
|
|
899
|
+
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
928
900
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
929
901
|
}
|
|
930
902
|
|
|
903
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
904
|
+
ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
|
|
905
|
+
|
|
931
906
|
// Reduce horizontal sums:
|
|
932
|
-
simsimd_distance_t ab_real =
|
|
933
|
-
simsimd_distance_t ab_imag =
|
|
907
|
+
simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
908
|
+
simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
934
909
|
|
|
935
910
|
// Handle the tail:
|
|
936
911
|
for (; i + 2 <= n; i += 2) {
|
|
@@ -963,14 +938,16 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
|
|
|
963
938
|
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
964
939
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
965
940
|
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
966
|
-
a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
|
|
967
941
|
b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
968
942
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
969
943
|
}
|
|
970
944
|
|
|
945
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
946
|
+
ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
|
|
947
|
+
|
|
971
948
|
// Reduce horizontal sums:
|
|
972
|
-
simsimd_distance_t ab_real =
|
|
973
|
-
simsimd_distance_t ab_imag =
|
|
949
|
+
simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
950
|
+
simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
974
951
|
|
|
975
952
|
// Handle the tail:
|
|
976
953
|
for (; i + 2 <= n; i += 2) {
|
|
@@ -982,6 +959,44 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
|
|
|
982
959
|
results[1] = ab_imag;
|
|
983
960
|
}
|
|
984
961
|
|
|
962
|
+
SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const* a, simsimd_size_t n) {
|
|
963
|
+
// In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
|
|
964
|
+
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
965
|
+
// loads and stores with vectorized math.
|
|
966
|
+
union {
|
|
967
|
+
__m128i vec;
|
|
968
|
+
simsimd_f16_t scalars[8];
|
|
969
|
+
} result;
|
|
970
|
+
simsimd_size_t i = 0;
|
|
971
|
+
for (; i < n; ++i)
|
|
972
|
+
result.scalars[i] = a[i];
|
|
973
|
+
for (; i < 8; ++i)
|
|
974
|
+
result.scalars[i] = 0;
|
|
975
|
+
return _mm256_cvtph_ps(result.vec);
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
979
|
+
simsimd_distance_t* result) {
|
|
980
|
+
__m256 a_vec, b_vec;
|
|
981
|
+
__m256 ab_vec = _mm256_setzero_ps();
|
|
982
|
+
|
|
983
|
+
simsimd_dot_f16_haswell_cycle:
|
|
984
|
+
if (n < 8) {
|
|
985
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
986
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
987
|
+
n = 0;
|
|
988
|
+
} else {
|
|
989
|
+
a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
990
|
+
b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
991
|
+
n -= 8, a += 8, b += 8;
|
|
992
|
+
}
|
|
993
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
994
|
+
if (n)
|
|
995
|
+
goto simsimd_dot_f16_haswell_cycle;
|
|
996
|
+
|
|
997
|
+
*result = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
998
|
+
}
|
|
999
|
+
|
|
985
1000
|
SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
986
1001
|
simsimd_distance_t* results) {
|
|
987
1002
|
// Ideally the implementation would load 256 bits worth of vector data at a time,
|
|
@@ -1004,40 +1019,24 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16
|
|
|
1004
1019
|
3, 2, 1, 0, // Points to the first f32 in 128-bit lane
|
|
1005
1020
|
7, 6, 5, 4 // Points to the second f32 in 128-bit lane
|
|
1006
1021
|
);
|
|
1007
|
-
simsimd_size_t i = 0;
|
|
1008
|
-
for (; i + 8 <= n; i += 8) {
|
|
1009
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
1010
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
1011
|
-
__m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
|
|
1012
|
-
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1013
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
|
|
1014
|
-
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
1015
|
-
}
|
|
1016
1022
|
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
if (i < n) {
|
|
1021
|
-
union {
|
|
1022
|
-
__m128i f16_vec;
|
|
1023
|
-
simsimd_f16_t f16[8];
|
|
1024
|
-
} a_padded_tail, b_padded_tail;
|
|
1025
|
-
simsimd_size_t j = 0;
|
|
1026
|
-
for (; i < n; ++i, ++j)
|
|
1027
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
1028
|
-
for (; j < 8; ++j)
|
|
1029
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
1030
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
1031
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
1032
|
-
__m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
|
|
1023
|
+
while (n >= 8) {
|
|
1024
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
1025
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
1033
1026
|
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1034
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec,
|
|
1027
|
+
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1035
1028
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
1029
|
+
|
|
1030
|
+
n -= 8, a += 8, b += 8;
|
|
1036
1031
|
}
|
|
1037
1032
|
|
|
1038
|
-
//
|
|
1039
|
-
|
|
1040
|
-
|
|
1033
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1034
|
+
ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
|
|
1035
|
+
|
|
1036
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
1037
|
+
simsimd_dot_f16c_serial(a, b, n, results);
|
|
1038
|
+
results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
1039
|
+
results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
1041
1040
|
}
|
|
1042
1041
|
|
|
1043
1042
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
@@ -1056,40 +1055,24 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
|
|
|
1056
1055
|
3, 2, 1, 0, // Points to the first f32 in 128-bit lane
|
|
1057
1056
|
7, 6, 5, 4 // Points to the second f32 in 128-bit lane
|
|
1058
1057
|
);
|
|
1059
|
-
simsimd_size_t i = 0;
|
|
1060
|
-
for (; i + 8 <= n; i += 8) {
|
|
1061
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
1062
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
1063
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1064
|
-
a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
|
|
1065
|
-
b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1066
|
-
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
1067
|
-
}
|
|
1068
1058
|
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
if (i < n) {
|
|
1073
|
-
union {
|
|
1074
|
-
__m128i f16_vec;
|
|
1075
|
-
simsimd_f16_t f16[8];
|
|
1076
|
-
} a_padded_tail, b_padded_tail;
|
|
1077
|
-
simsimd_size_t j = 0;
|
|
1078
|
-
for (; i < n; ++i, ++j)
|
|
1079
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
1080
|
-
for (; j < 8; ++j)
|
|
1081
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
1082
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
1083
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
1059
|
+
while (n >= 8) {
|
|
1060
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
|
|
1061
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
|
|
1084
1062
|
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1085
|
-
a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
|
|
1086
1063
|
b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1087
1064
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
1065
|
+
|
|
1066
|
+
n -= 8, a += 8, b += 8;
|
|
1088
1067
|
}
|
|
1089
1068
|
|
|
1090
|
-
//
|
|
1091
|
-
|
|
1092
|
-
|
|
1069
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1070
|
+
ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
|
|
1071
|
+
|
|
1072
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
1073
|
+
simsimd_dot_f16c_serial(a, b, n, results);
|
|
1074
|
+
results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
1075
|
+
results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
1093
1076
|
}
|
|
1094
1077
|
|
|
1095
1078
|
SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
@@ -1123,49 +1106,51 @@ SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
|
|
|
1123
1106
|
// Take care of the tail:
|
|
1124
1107
|
int ab = _mm_extract_epi32(ab_sum, 0);
|
|
1125
1108
|
for (; i < n; ++i)
|
|
1126
|
-
ab += a[i] * b[i];
|
|
1109
|
+
ab += (int)(a[i]) * b[i];
|
|
1127
1110
|
*result = ab;
|
|
1128
1111
|
}
|
|
1129
1112
|
|
|
1113
|
+
SIMSIMD_INTERNAL __m256 _simsimd_bf16x8_to_f32x8_haswell(__m128i a) {
|
|
1114
|
+
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
1115
|
+
return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a), 16));
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
SIMSIMD_INTERNAL __m128i _simsimd_partial_load_bf16x8_haswell(simsimd_bf16_t const* a, simsimd_size_t n) {
|
|
1119
|
+
// In case the software emulation for `bf16` scalars is enabled, the `simsimd_bf16_to_f32`
|
|
1120
|
+
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
1121
|
+
// loads and stores with vectorized math.
|
|
1122
|
+
union {
|
|
1123
|
+
__m128i vec;
|
|
1124
|
+
simsimd_bf16_t scalars[8];
|
|
1125
|
+
} result;
|
|
1126
|
+
simsimd_size_t i = 0;
|
|
1127
|
+
for (; i < n; ++i)
|
|
1128
|
+
result.scalars[i] = a[i];
|
|
1129
|
+
for (; i < 8; ++i)
|
|
1130
|
+
result.scalars[i] = 0;
|
|
1131
|
+
return result.vec;
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1130
1134
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1131
1135
|
simsimd_distance_t* result) {
|
|
1136
|
+
__m128i a_vec, b_vec;
|
|
1132
1137
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
1133
|
-
simsimd_size_t i = 0;
|
|
1134
|
-
for (; i + 8 <= n; i += 8) {
|
|
1135
|
-
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
1136
|
-
// x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
|
|
1137
|
-
__m256 a_vec =
|
|
1138
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
|
|
1139
|
-
__m256 b_vec =
|
|
1140
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
|
|
1141
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1142
|
-
}
|
|
1143
1138
|
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
for (; i < n; ++i, ++j)
|
|
1154
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
1155
|
-
for (; j < 8; ++j)
|
|
1156
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
1157
|
-
__m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
|
|
1158
|
-
__m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
|
|
1159
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1139
|
+
simsimd_dot_bf16_haswell_cycle:
|
|
1140
|
+
if (n < 8) {
|
|
1141
|
+
a_vec = _simsimd_partial_load_bf16x8_haswell(a, n);
|
|
1142
|
+
b_vec = _simsimd_partial_load_bf16x8_haswell(b, n);
|
|
1143
|
+
n = 0;
|
|
1144
|
+
} else {
|
|
1145
|
+
a_vec = _mm_loadu_si128((__m128i const*)a);
|
|
1146
|
+
b_vec = _mm_loadu_si128((__m128i const*)b);
|
|
1147
|
+
a += 8, b += 8, n -= 8;
|
|
1160
1148
|
}
|
|
1149
|
+
ab_vec = _mm256_fmadd_ps(_simsimd_bf16x8_to_f32x8_haswell(a_vec), _simsimd_bf16x8_to_f32x8_haswell(b_vec), ab_vec);
|
|
1150
|
+
if (n)
|
|
1151
|
+
goto simsimd_dot_bf16_haswell_cycle;
|
|
1161
1152
|
|
|
1162
|
-
|
|
1163
|
-
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
1164
|
-
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
1165
|
-
|
|
1166
|
-
simsimd_f32_t f32_result;
|
|
1167
|
-
_mm_store_ss(&f32_result, _mm256_castps256_ps128(ab_vec));
|
|
1168
|
-
*result = f32_result;
|
|
1153
|
+
*result = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1169
1154
|
}
|
|
1170
1155
|
|
|
1171
1156
|
#pragma clang attribute pop
|
|
@@ -1174,8 +1159,15 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf
|
|
|
1174
1159
|
|
|
1175
1160
|
#if SIMSIMD_TARGET_SKYLAKE
|
|
1176
1161
|
#pragma GCC push_options
|
|
1177
|
-
#pragma GCC target("avx512f", "avx512vl", "avx512bw", "bmi2")
|
|
1178
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
|
|
1162
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
|
|
1163
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
|
|
1164
|
+
|
|
1165
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x16_skylake(__m512 a) {
|
|
1166
|
+
__m512 x = _mm512_add_ps(a, _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 0, 3, 2)));
|
|
1167
|
+
__m128 r = _mm512_castps512_ps128(_mm512_add_ps(x, _mm512_shuffle_f32x4(x, x, _MM_SHUFFLE(0, 0, 0, 1))));
|
|
1168
|
+
r = _mm_hadd_ps(r, r);
|
|
1169
|
+
return _mm_cvtss_f32(_mm_hadd_ps(r, r));
|
|
1170
|
+
}
|
|
1179
1171
|
|
|
1180
1172
|
SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1181
1173
|
simsimd_distance_t* result) {
|
|
@@ -1197,7 +1189,7 @@ simsimd_dot_f32_skylake_cycle:
|
|
|
1197
1189
|
if (n)
|
|
1198
1190
|
goto simsimd_dot_f32_skylake_cycle;
|
|
1199
1191
|
|
|
1200
|
-
*result =
|
|
1192
|
+
*result = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1201
1193
|
}
|
|
1202
1194
|
|
|
1203
1195
|
SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
@@ -1225,11 +1217,10 @@ simsimd_dot_f64_skylake_cycle:
|
|
|
1225
1217
|
|
|
1226
1218
|
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1227
1219
|
simsimd_distance_t* results) {
|
|
1228
|
-
|
|
1229
1220
|
__m512 ab_real_vec = _mm512_setzero();
|
|
1230
1221
|
__m512 ab_imag_vec = _mm512_setzero();
|
|
1231
1222
|
__m512 a_vec;
|
|
1232
|
-
|
|
1223
|
+
__m512 b_vec;
|
|
1233
1224
|
|
|
1234
1225
|
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1235
1226
|
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
@@ -1247,27 +1238,29 @@ simsimd_dot_f32c_skylake_cycle:
|
|
|
1247
1238
|
if (n < 16) {
|
|
1248
1239
|
__mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1249
1240
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1250
|
-
b_vec =
|
|
1241
|
+
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1251
1242
|
n = 0;
|
|
1252
1243
|
} else {
|
|
1253
1244
|
a_vec = _mm512_loadu_ps(a);
|
|
1254
|
-
b_vec =
|
|
1245
|
+
b_vec = _mm512_loadu_ps(b);
|
|
1255
1246
|
a += 16, b += 16, n -= 16;
|
|
1256
1247
|
}
|
|
1257
|
-
ab_real_vec = _mm512_fmadd_ps(
|
|
1258
|
-
ab_imag_vec =
|
|
1259
|
-
|
|
1248
|
+
ab_real_vec = _mm512_fmadd_ps(b_vec, a_vec, ab_real_vec);
|
|
1249
|
+
ab_imag_vec = _mm512_fmadd_ps(
|
|
1250
|
+
_mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
|
|
1260
1251
|
if (n)
|
|
1261
1252
|
goto simsimd_dot_f32c_skylake_cycle;
|
|
1262
1253
|
|
|
1254
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1255
|
+
ab_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_real_vec), sign_flip_vec));
|
|
1256
|
+
|
|
1263
1257
|
// Reduce horizontal sums:
|
|
1264
|
-
results[0] =
|
|
1265
|
-
results[1] =
|
|
1258
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1259
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1266
1260
|
}
|
|
1267
1261
|
|
|
1268
1262
|
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
|
|
1269
1263
|
simsimd_distance_t* results) {
|
|
1270
|
-
|
|
1271
1264
|
__m512 ab_real_vec = _mm512_setzero();
|
|
1272
1265
|
__m512 ab_imag_vec = _mm512_setzero();
|
|
1273
1266
|
__m512 a_vec;
|
|
@@ -1297,24 +1290,25 @@ simsimd_vdot_f32c_skylake_cycle:
|
|
|
1297
1290
|
a += 16, b += 16, n -= 16;
|
|
1298
1291
|
}
|
|
1299
1292
|
ab_real_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1300
|
-
a_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(a_vec), sign_flip_vec));
|
|
1301
1293
|
b_vec = _mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec));
|
|
1302
1294
|
ab_imag_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
1303
1295
|
if (n)
|
|
1304
1296
|
goto simsimd_vdot_f32c_skylake_cycle;
|
|
1305
1297
|
|
|
1298
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1299
|
+
ab_imag_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_imag_vec), sign_flip_vec));
|
|
1300
|
+
|
|
1306
1301
|
// Reduce horizontal sums:
|
|
1307
|
-
results[0] =
|
|
1308
|
-
results[1] =
|
|
1302
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1303
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1309
1304
|
}
|
|
1310
1305
|
|
|
1311
1306
|
SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
1312
1307
|
simsimd_distance_t* results) {
|
|
1313
|
-
|
|
1314
1308
|
__m512d ab_real_vec = _mm512_setzero_pd();
|
|
1315
1309
|
__m512d ab_imag_vec = _mm512_setzero_pd();
|
|
1316
1310
|
__m512d a_vec;
|
|
1317
|
-
|
|
1311
|
+
__m512d b_vec;
|
|
1318
1312
|
|
|
1319
1313
|
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1320
1314
|
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
@@ -1335,19 +1329,22 @@ simsimd_dot_f64c_skylake_cycle:
|
|
|
1335
1329
|
if (n < 8) {
|
|
1336
1330
|
__mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1337
1331
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1338
|
-
b_vec =
|
|
1332
|
+
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1339
1333
|
n = 0;
|
|
1340
1334
|
} else {
|
|
1341
1335
|
a_vec = _mm512_loadu_pd(a);
|
|
1342
|
-
b_vec =
|
|
1336
|
+
b_vec = _mm512_loadu_pd(b);
|
|
1343
1337
|
a += 8, b += 8, n -= 8;
|
|
1344
1338
|
}
|
|
1345
|
-
ab_real_vec = _mm512_fmadd_pd(
|
|
1346
|
-
ab_imag_vec =
|
|
1347
|
-
|
|
1339
|
+
ab_real_vec = _mm512_fmadd_pd(b_vec, a_vec, ab_real_vec);
|
|
1340
|
+
ab_imag_vec = _mm512_fmadd_pd(
|
|
1341
|
+
_mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
|
|
1348
1342
|
if (n)
|
|
1349
1343
|
goto simsimd_dot_f64c_skylake_cycle;
|
|
1350
1344
|
|
|
1345
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1346
|
+
ab_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_real_vec), sign_flip_vec));
|
|
1347
|
+
|
|
1351
1348
|
// Reduce horizontal sums:
|
|
1352
1349
|
results[0] = _mm512_reduce_add_pd(ab_real_vec);
|
|
1353
1350
|
results[1] = _mm512_reduce_add_pd(ab_imag_vec);
|
|
@@ -1355,7 +1352,6 @@ simsimd_dot_f64c_skylake_cycle:
|
|
|
1355
1352
|
|
|
1356
1353
|
SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
|
|
1357
1354
|
simsimd_distance_t* results) {
|
|
1358
|
-
|
|
1359
1355
|
__m512d ab_real_vec = _mm512_setzero_pd();
|
|
1360
1356
|
__m512d ab_imag_vec = _mm512_setzero_pd();
|
|
1361
1357
|
__m512d a_vec;
|
|
@@ -1388,12 +1384,14 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1388
1384
|
a += 8, b += 8, n -= 8;
|
|
1389
1385
|
}
|
|
1390
1386
|
ab_real_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_real_vec);
|
|
1391
|
-
a_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(a_vec), sign_flip_vec));
|
|
1392
1387
|
b_vec = _mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec));
|
|
1393
1388
|
ab_imag_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_imag_vec);
|
|
1394
1389
|
if (n)
|
|
1395
1390
|
goto simsimd_vdot_f64c_skylake_cycle;
|
|
1396
1391
|
|
|
1392
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1393
|
+
ab_imag_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_imag_vec), sign_flip_vec));
|
|
1394
|
+
|
|
1397
1395
|
// Reduce horizontal sums:
|
|
1398
1396
|
results[0] = _mm512_reduce_add_pd(ab_real_vec);
|
|
1399
1397
|
results[1] = _mm512_reduce_add_pd(ab_imag_vec);
|
|
@@ -1405,8 +1403,9 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1405
1403
|
|
|
1406
1404
|
#if SIMSIMD_TARGET_GENOA
|
|
1407
1405
|
#pragma GCC push_options
|
|
1408
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1409
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))),
|
|
1406
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1407
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1408
|
+
apply_to = function)
|
|
1410
1409
|
|
|
1411
1410
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1412
1411
|
simsimd_distance_t* result) {
|
|
@@ -1428,12 +1427,11 @@ simsimd_dot_bf16_genoa_cycle:
|
|
|
1428
1427
|
if (n)
|
|
1429
1428
|
goto simsimd_dot_bf16_genoa_cycle;
|
|
1430
1429
|
|
|
1431
|
-
*result =
|
|
1430
|
+
*result = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1432
1431
|
}
|
|
1433
1432
|
|
|
1434
1433
|
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1435
1434
|
simsimd_distance_t* results) {
|
|
1436
|
-
|
|
1437
1435
|
__m512 ab_real_vec = _mm512_setzero_ps();
|
|
1438
1436
|
__m512 ab_imag_vec = _mm512_setzero_ps();
|
|
1439
1437
|
__m512i a_vec;
|
|
@@ -1470,13 +1468,12 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1470
1468
|
goto simsimd_dot_bf16c_genoa_cycle;
|
|
1471
1469
|
|
|
1472
1470
|
// Reduce horizontal sums:
|
|
1473
|
-
results[0] =
|
|
1474
|
-
results[1] =
|
|
1471
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1472
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1475
1473
|
}
|
|
1476
1474
|
|
|
1477
1475
|
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1478
1476
|
simsimd_distance_t* results) {
|
|
1479
|
-
|
|
1480
1477
|
__m512 ab_real_vec = _mm512_setzero_ps();
|
|
1481
1478
|
__m512 ab_imag_vec = _mm512_setzero_ps();
|
|
1482
1479
|
__m512i a_vec;
|
|
@@ -1514,8 +1511,8 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1514
1511
|
goto simsimd_dot_bf16c_genoa_cycle;
|
|
1515
1512
|
|
|
1516
1513
|
// Reduce horizontal sums:
|
|
1517
|
-
results[0] =
|
|
1518
|
-
results[1] =
|
|
1514
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1515
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1519
1516
|
}
|
|
1520
1517
|
|
|
1521
1518
|
#pragma clang attribute pop
|
|
@@ -1524,8 +1521,9 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1524
1521
|
|
|
1525
1522
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1526
1523
|
#pragma GCC push_options
|
|
1527
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1528
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))),
|
|
1524
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1525
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
|
|
1526
|
+
apply_to = function)
|
|
1529
1527
|
|
|
1530
1528
|
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
1531
1529
|
simsimd_distance_t* result) {
|
|
@@ -1552,7 +1550,6 @@ simsimd_dot_f16_sapphire_cycle:
|
|
|
1552
1550
|
|
|
1553
1551
|
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
1554
1552
|
simsimd_distance_t* results) {
|
|
1555
|
-
|
|
1556
1553
|
__m512h ab_real_vec = _mm512_setzero_ph();
|
|
1557
1554
|
__m512h ab_imag_vec = _mm512_setzero_ph();
|
|
1558
1555
|
__m512i a_vec;
|
|
@@ -1582,6 +1579,7 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1582
1579
|
b_vec = _mm512_loadu_epi16(b);
|
|
1583
1580
|
a += 32, b += 32, n -= 32;
|
|
1584
1581
|
}
|
|
1582
|
+
// TODO: Consider using `_mm512_fmaddsub`
|
|
1585
1583
|
ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_xor_si512(b_vec, sign_flip_vec)),
|
|
1586
1584
|
_mm512_castsi512_ph(a_vec), ab_real_vec);
|
|
1587
1585
|
ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)),
|
|
@@ -1590,13 +1588,13 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1590
1588
|
goto simsimd_dot_f16c_sapphire_cycle;
|
|
1591
1589
|
|
|
1592
1590
|
// Reduce horizontal sums:
|
|
1591
|
+
// TODO: Optimize this with tree-like reductions
|
|
1593
1592
|
results[0] = _mm512_reduce_add_ph(ab_real_vec);
|
|
1594
1593
|
results[1] = _mm512_reduce_add_ph(ab_imag_vec);
|
|
1595
1594
|
}
|
|
1596
1595
|
|
|
1597
1596
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
|
|
1598
1597
|
simsimd_distance_t* results) {
|
|
1599
|
-
|
|
1600
1598
|
__m512h ab_real_vec = _mm512_setzero_ph();
|
|
1601
1599
|
__m512h ab_imag_vec = _mm512_setzero_ph();
|
|
1602
1600
|
__m512i a_vec;
|
|
@@ -1644,8 +1642,9 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1644
1642
|
|
|
1645
1643
|
#if SIMSIMD_TARGET_ICE
|
|
1646
1644
|
#pragma GCC push_options
|
|
1647
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1648
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))),
|
|
1645
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1646
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
|
|
1647
|
+
apply_to = function)
|
|
1649
1648
|
|
|
1650
1649
|
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
|
|
1651
1650
|
simsimd_distance_t* result) {
|