usearch 2.15.2 → 2.15.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,8 +16,8 @@
16
16
  * - 8-bit signed integers
17
17
  *
18
18
  * For hardware architectures:
19
- * - Arm (NEON, SVE?)
20
- * - x86 (AVX2?, AVX512?)
19
+ * - Arm (NEON, SVE)
20
+ * - x86 (AVX2, AVX512)
21
21
  *
22
22
  * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
23
23
  * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
@@ -82,7 +82,11 @@ SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t c
82
82
  SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
83
83
  SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
84
84
 
85
- SIMSIMD_PUBLIC void simsimd_dot_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
85
+ SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
86
+
87
+ SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
88
+ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
89
+ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
86
90
 
87
91
  /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
88
92
  * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
@@ -131,7 +135,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_
131
135
  SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
132
136
  SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
133
137
 
134
- SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
138
+ SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
135
139
 
136
140
  SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
137
141
  SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
@@ -142,29 +146,29 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f1
142
146
  SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
143
147
  // clang-format on
144
148
 
145
- #define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, converter) \
149
+ #define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, load_and_convert) \
146
150
  SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const* a, \
147
151
  simsimd_##input_type##_t const* b, simsimd_size_t n, \
148
152
  simsimd_distance_t* result) { \
149
153
  simsimd_##accumulator_type##_t ab = 0; \
150
154
  for (simsimd_size_t i = 0; i != n; ++i) { \
151
- simsimd_##accumulator_type##_t ai = converter(a[i]); \
152
- simsimd_##accumulator_type##_t bi = converter(b[i]); \
155
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
156
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
153
157
  ab += ai * bi; \
154
158
  } \
155
159
  *result = ab; \
156
160
  }
157
161
 
158
- #define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, converter) \
162
+ #define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, load_and_convert) \
159
163
  SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const* a, \
160
164
  simsimd_##input_type##_t const* b, simsimd_size_t n, \
161
165
  simsimd_distance_t* results) { \
162
166
  simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
163
167
  for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
164
- simsimd_##accumulator_type##_t ar = converter(a[i]); \
165
- simsimd_##accumulator_type##_t br = converter(b[i]); \
166
- simsimd_##accumulator_type##_t ai = converter(a[i + 1]); \
167
- simsimd_##accumulator_type##_t bi = converter(b[i + 1]); \
168
+ simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
169
+ simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
170
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
171
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
168
172
  ab_real += ar * br - ai * bi; \
169
173
  ab_imag += ar * bi + ai * br; \
170
174
  } \
@@ -172,16 +176,16 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f
172
176
  results[1] = ab_imag; \
173
177
  }
174
178
 
175
- #define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, converter) \
179
+ #define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, load_and_convert) \
176
180
  SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const* a, \
177
181
  simsimd_##input_type##_t const* b, simsimd_size_t n, \
178
182
  simsimd_distance_t* results) { \
179
183
  simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
180
184
  for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
181
- simsimd_##accumulator_type##_t ar = converter(a[i]); \
182
- simsimd_##accumulator_type##_t br = converter(b[i]); \
183
- simsimd_##accumulator_type##_t ai = converter(a[i + 1]); \
184
- simsimd_##accumulator_type##_t bi = converter(b[i + 1]); \
185
+ simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
186
+ simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
187
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
188
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
185
189
  ab_real += ar * br + ai * bi; \
186
190
  ab_imag += ar * bi - ai * br; \
187
191
  } \
@@ -189,41 +193,54 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f
189
193
  results[1] = ab_imag; \
190
194
  }
191
195
 
192
- SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f64_serial
193
- SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f64c_serial
194
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_vdot_f64c_serial
196
+ SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64_serial
197
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64c_serial
198
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f64c_serial
195
199
 
196
- SIMSIMD_MAKE_DOT(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_dot_f32_serial
197
- SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_dot_f32c_serial
198
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_vdot_f32c_serial
200
+ SIMSIMD_MAKE_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_serial
201
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_serial
202
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_serial
199
203
 
200
- SIMSIMD_MAKE_DOT(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16_serial
201
- SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16c_serial
202
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_vdot_f16c_serial
204
+ SIMSIMD_MAKE_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_serial
205
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_serial
206
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_serial
203
207
 
204
- SIMSIMD_MAKE_DOT(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_dot_bf16_serial
205
- SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_dot_bf16c_serial
206
- SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_vdot_bf16c_serial
208
+ SIMSIMD_MAKE_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_serial
209
+ SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_serial
210
+ SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_serial
207
211
 
208
- SIMSIMD_MAKE_DOT(serial, i8, i64, SIMSIMD_IDENTIFY) // simsimd_dot_i8_serial
212
+ SIMSIMD_MAKE_DOT(serial, i8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_i8_serial
209
213
 
210
- SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f32_accurate
211
- SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_dot_f32c_accurate
212
- SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_vdot_f32c_accurate
214
+ SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_accurate
215
+ SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_accurate
216
+ SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_accurate
213
217
 
214
- SIMSIMD_MAKE_DOT(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16_accurate
215
- SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_dot_f16c_accurate
216
- SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_vdot_f16c_accurate
218
+ SIMSIMD_MAKE_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_accurate
219
+ SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_accurate
220
+ SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_accurate
217
221
 
218
- SIMSIMD_MAKE_DOT(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_dot_bf16_accurate
219
- SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_dot_bf16c_accurate
220
- SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_vdot_bf16c_accurate
222
+ SIMSIMD_MAKE_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_accurate
223
+ SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_accurate
224
+ SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_accurate
221
225
 
222
226
  #if SIMSIMD_TARGET_ARM
223
227
  #if SIMSIMD_TARGET_NEON
224
228
  #pragma GCC push_options
225
- #pragma GCC target("+simd")
226
- #pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
229
+ #pragma GCC target("arch=armv8.2-a+simd")
230
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
231
+
232
+ SIMSIMD_INTERNAL float32x4_t _simsimd_partial_load_f32x4_neon(simsimd_f32_t const* a, simsimd_size_t n) {
233
+ union {
234
+ float32x4_t vec;
235
+ simsimd_f32_t scalars[4];
236
+ } result;
237
+ simsimd_size_t i = 0;
238
+ for (; i < n; ++i)
239
+ result.scalars[i] = a[i];
240
+ for (; i < 4; ++i)
241
+ result.scalars[i] = 0;
242
+ return result.vec;
243
+ }
227
244
 
228
245
  SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
229
246
  simsimd_distance_t* result) {
@@ -349,36 +366,48 @@ SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
349
366
 
350
367
  #pragma clang attribute pop
351
368
  #pragma GCC pop_options
369
+ #endif
352
370
 
371
+ #if SIMSIMD_TARGET_NEON_F16
353
372
  #pragma GCC push_options
354
- #pragma GCC target("+simd+fp16")
355
- #pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
373
+ #pragma GCC target("arch=armv8.2-a+simd+fp16")
374
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
375
+
376
+ SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t const* a, simsimd_size_t n) {
377
+ // In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
378
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
379
+ // loads and stores with vectorized math.
380
+ union {
381
+ float16x4_t vec;
382
+ simsimd_f16_t scalars[4];
383
+ } result;
384
+ simsimd_size_t i = 0;
385
+ for (; i < n; ++i)
386
+ result.scalars[i] = a[i];
387
+ for (; i < 4; ++i)
388
+ result.scalars[i] = 0;
389
+ return result.vec;
390
+ }
356
391
 
357
392
  SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
358
393
  simsimd_distance_t* result) {
359
394
  float32x4_t ab_vec = vdupq_n_f32(0);
395
+ float32x4_t a_vec, b_vec;
360
396
  simsimd_size_t i = 0;
361
- for (; i + 4 <= n; i += 4) {
362
- float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
363
- float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
364
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
365
- }
366
397
 
367
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
368
- // function will run. It is extremely slow, so even for the tail, let's combine serial
369
- // loads and stores with vectorized math.
370
- if (i < n) {
371
- union {
372
- float16x4_t f16_vec;
373
- simsimd_f16_t f16[4];
374
- } a_padded_tail, b_padded_tail;
375
- simsimd_size_t j = 0;
376
- for (; i < n; ++i, ++j)
377
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
378
- for (; j < 4; ++j)
379
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
380
- ab_vec = vfmaq_f32(ab_vec, vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
398
+ simsimd_dot_f16_neon_cycle:
399
+ if (n < 4) {
400
+ a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
401
+ b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
402
+ n = 0;
403
+ } else {
404
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
405
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
406
+ a += 4, b += 4, n -= 4;
381
407
  }
408
+ ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
409
+ if (n)
410
+ goto simsimd_dot_f16_neon_cycle;
382
411
  *result = vaddvq_f32(ab_vec);
383
412
  }
384
413
 
@@ -390,13 +419,13 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
390
419
  // at once. That's how the original implementation worked, but compiling it was a nightmare :)
391
420
  float32x4_t ab_real_vec = vdupq_n_f32(0);
392
421
  float32x4_t ab_imag_vec = vdupq_n_f32(0);
393
- simsimd_size_t i = 0;
394
- for (; i + 8 <= n; i += 8) {
422
+
423
+ while (n >= 8) {
395
424
  // Unpack the input arrays into real and imaginary parts.
396
425
  // MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
397
426
  // integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
398
- int16x4x2_t a_vec = vld2_s16((short*)a + i);
399
- int16x4x2_t b_vec = vld2_s16((short*)b + i);
427
+ int16x4x2_t a_vec = vld2_s16((short*)a);
428
+ int16x4x2_t b_vec = vld2_s16((short*)b);
400
429
  float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
401
430
  float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
402
431
  float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
@@ -407,20 +436,14 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
407
436
  ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
408
437
  ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
409
438
  ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
410
- }
411
-
412
- // Reduce horizontal sums:
413
- simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
414
- simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
415
439
 
416
- // Handle the tail:
417
- for (; i + 2 <= n; i += 2) {
418
- simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
419
- ab_real += ar * br - ai * bi;
420
- ab_imag += ar * bi + ai * br;
440
+ n -= 8, a += 8, b += 8;
421
441
  }
422
- results[0] = ab_real;
423
- results[1] = ab_imag;
442
+
443
+ // Reduce horizontal sums and aggregate with the tail:
444
+ simsimd_dot_f16c_serial(a, b, n, results);
445
+ results[0] += vaddvq_f32(ab_real_vec);
446
+ results[1] += vaddvq_f32(ab_imag_vec);
424
447
  }
425
448
 
426
449
  SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, //
@@ -431,13 +454,13 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
431
454
  // at once. That's how the original implementation worked, but compiling it was a nightmare :)
432
455
  float32x4_t ab_real_vec = vdupq_n_f32(0);
433
456
  float32x4_t ab_imag_vec = vdupq_n_f32(0);
434
- simsimd_size_t i = 0;
435
- for (; i + 8 <= n; i += 8) {
457
+
458
+ while (n >= 8) {
436
459
  // Unpack the input arrays into real and imaginary parts.
437
460
  // MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
438
461
  // integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
439
- int16x4x2_t a_vec = vld2_s16((short*)a + i);
440
- int16x4x2_t b_vec = vld2_s16((short*)b + i);
462
+ int16x4x2_t a_vec = vld2_s16((short*)a);
463
+ int16x4x2_t b_vec = vld2_s16((short*)b);
441
464
  float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
442
465
  float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
443
466
  float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
@@ -448,62 +471,61 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
448
471
  ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
449
472
  ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
450
473
  ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
451
- }
452
474
 
453
- // Reduce horizontal sums:
454
- simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
455
- simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
456
-
457
- // Handle the tail:
458
- for (; i + 2 <= n; i += 2) {
459
- simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
460
- ab_real += ar * br + ai * bi;
461
- ab_imag += ar * bi - ai * br;
475
+ n -= 8, a += 8, b += 8;
462
476
  }
463
- results[0] = ab_real;
464
- results[1] = ab_imag;
477
+
478
+ // Reduce horizontal sums and aggregate with the tail:
479
+ simsimd_vdot_f16c_serial(a, b, n, results);
480
+ results[0] += vaddvq_f32(ab_real_vec);
481
+ results[1] += vaddvq_f32(ab_imag_vec);
465
482
  }
466
483
 
467
484
  #pragma clang attribute pop
468
485
  #pragma GCC pop_options
469
- #endif // SIMSIMD_TARGET_NEON
486
+ #endif // SIMSIMD_TARGET_NEON_F16
470
487
 
471
488
  #if SIMSIMD_TARGET_NEON_BF16
472
489
  #pragma GCC push_options
473
490
  #pragma GCC target("arch=armv8.6-a+simd+bf16")
474
491
  #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
475
492
 
493
+ SIMSIMD_INTERNAL bfloat16x8_t _simsimd_partial_load_bf16x8_neon(simsimd_bf16_t const* a, simsimd_size_t n) {
494
+ union {
495
+ bfloat16x8_t vec;
496
+ simsimd_bf16_t scalars[8];
497
+ } result;
498
+ simsimd_size_t i = 0;
499
+ for (; i < n; ++i)
500
+ result.scalars[i] = a[i];
501
+ for (; i < 8; ++i)
502
+ result.scalars[i] = 0;
503
+ return result.vec;
504
+ }
505
+
476
506
  SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
477
507
  simsimd_distance_t* result) {
478
- float32x4_t ab_high_vec = vdupq_n_f32(0), ab_low_vec = vdupq_n_f32(0);
479
- simsimd_size_t i = 0;
480
- for (; i + 8 <= n; i += 8) {
481
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
482
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
483
- ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
484
- ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
485
- }
508
+ float32x4_t ab_vec = vdupq_n_f32(0);
509
+ bfloat16x8_t a_vec, b_vec;
486
510
 
487
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
488
- // function will run. It is extremely slow, so even for the tail, let's combine serial
489
- // loads and stores with vectorized math.
490
- if (i < n) {
491
- union {
492
- bfloat16x8_t bf16_vec;
493
- simsimd_bf16_t bf16[8];
494
- } a_padded_tail, b_padded_tail;
495
- simsimd_size_t j = 0;
496
- for (; i < n; ++i, ++j)
497
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
498
- for (; j < 8; ++j)
499
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
500
- ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
501
- ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
511
+ simsimd_dot_bf16_neon_cycle:
512
+ if (n < 8) {
513
+ a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
514
+ b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
515
+ n = 0;
516
+ } else {
517
+ a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
518
+ b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
519
+ a += 8, b += 8, n -= 8;
502
520
  }
503
- *result = vaddvq_f32(ab_high_vec) + vaddvq_f32(ab_low_vec);
521
+ ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
522
+ if (n)
523
+ goto simsimd_dot_bf16_neon_cycle;
524
+
525
+ *result = vaddvq_f32(ab_vec);
504
526
  }
505
527
 
506
- SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, //
528
+ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
507
529
  simsimd_distance_t* results) {
508
530
 
509
531
  // A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
@@ -511,13 +533,13 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
511
533
  // at once. That's how the original implementation worked, but compiling it was a nightmare :)
512
534
  float32x4_t ab_real_vec = vdupq_n_f32(0);
513
535
  float32x4_t ab_imag_vec = vdupq_n_f32(0);
514
- simsimd_size_t i = 0;
515
- for (; i + 8 <= n; i += 8) {
536
+
537
+ while (n >= 8) {
516
538
  // Unpack the input arrays into real and imaginary parts.
517
539
  // MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
518
540
  // integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
519
- int16x4x2_t a_vec = vld2_s16((short*)a + i);
520
- int16x4x2_t b_vec = vld2_s16((short*)b + i);
541
+ int16x4x2_t a_vec = vld2_s16((short*)a);
542
+ int16x4x2_t b_vec = vld2_s16((short*)b);
521
543
  float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
522
544
  float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
523
545
  float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
@@ -528,20 +550,14 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
528
550
  ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
529
551
  ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
530
552
  ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
531
- }
532
-
533
- // Reduce horizontal sums:
534
- simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
535
- simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
536
553
 
537
- // Handle the tail:
538
- for (; i + 2 <= n; i += 2) {
539
- simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
540
- ab_real += ar * br - ai * bi;
541
- ab_imag += ar * bi + ai * br;
554
+ n -= 8, a += 8, b += 8;
542
555
  }
543
- results[0] = ab_real;
544
- results[1] = ab_imag;
556
+
557
+ // Reduce horizontal sums and aggregate with the tail:
558
+ simsimd_dot_bf16c_serial(a, b, n, results);
559
+ results[0] += vaddvq_f32(ab_real_vec);
560
+ results[1] += vaddvq_f32(ab_imag_vec);
545
561
  }
546
562
 
547
563
  SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, //
@@ -552,13 +568,13 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
552
568
  // at once. That's how the original implementation worked, but compiling it was a nightmare :)
553
569
  float32x4_t ab_real_vec = vdupq_n_f32(0);
554
570
  float32x4_t ab_imag_vec = vdupq_n_f32(0);
555
- simsimd_size_t i = 0;
556
- for (; i + 8 <= n; i += 8) {
571
+
572
+ while (n >= 8) {
557
573
  // Unpack the input arrays into real and imaginary parts.
558
574
  // MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
559
575
  // integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
560
- int16x4x2_t a_vec = vld2_s16((short*)a + i);
561
- int16x4x2_t b_vec = vld2_s16((short*)b + i);
576
+ int16x4x2_t a_vec = vld2_s16((short*)a);
577
+ int16x4x2_t b_vec = vld2_s16((short*)b);
562
578
  float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
563
579
  float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
564
580
  float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
@@ -569,20 +585,14 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
569
585
  ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
570
586
  ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
571
587
  ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
572
- }
573
588
 
574
- // Reduce horizontal sums:
575
- simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
576
- simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
577
-
578
- // Handle the tail:
579
- for (; i + 2 <= n; i += 2) {
580
- simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
581
- ab_real += ar * br + ai * bi;
582
- ab_imag += ar * bi - ai * br;
589
+ n -= 8, a += 8, b += 8;
583
590
  }
584
- results[0] = ab_real;
585
- results[1] = ab_imag;
591
+
592
+ // Reduce horizontal sums and aggregate with the tail:
593
+ simsimd_vdot_bf16c_serial(a, b, n, results);
594
+ results[0] += vaddvq_f32(ab_real_vec);
595
+ results[1] += vaddvq_f32(ab_imag_vec);
586
596
  }
587
597
 
588
598
  #pragma clang attribute pop
@@ -592,8 +602,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
592
602
  #if SIMSIMD_TARGET_SVE
593
603
 
594
604
  #pragma GCC push_options
595
- #pragma GCC target("+sve")
596
- #pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
605
+ #pragma GCC target("arch=armv8.2-a+sve")
606
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
597
607
 
598
608
  SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
599
609
  simsimd_distance_t* result) {
@@ -719,8 +729,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t
719
729
  #pragma GCC pop_options
720
730
 
721
731
  #pragma GCC push_options
722
- #pragma GCC target("+sve+fp16")
723
- #pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
732
+ #pragma GCC target("arch=armv8.2-a+sve+fp16")
733
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
724
734
 
725
735
  SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
726
736
  simsimd_distance_t* result) {
@@ -796,44 +806,7 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t
796
806
  #pragma GCC target("avx2", "f16c", "fma")
797
807
  #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
798
808
 
799
- SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
800
- simsimd_distance_t* result) {
801
- __m256 ab_vec = _mm256_setzero_ps();
802
- simsimd_size_t i = 0;
803
- for (; i + 8 <= n; i += 8) {
804
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
805
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
806
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
807
- }
808
-
809
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
810
- // function will run. It is extremely slow, so even for the tail, let's combine serial
811
- // loads and stores with vectorized math.
812
- if (i < n) {
813
- union {
814
- __m128i f16_vec;
815
- simsimd_f16_t f16[8];
816
- } a_padded_tail, b_padded_tail;
817
- simsimd_size_t j = 0;
818
- for (; i < n; ++i, ++j)
819
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
820
- for (; j < 8; ++j)
821
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
822
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
823
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
824
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
825
- }
826
-
827
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
828
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
829
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
830
-
831
- simsimd_f32_t f32_result;
832
- _mm_store_ss(&f32_result, _mm256_castps256_ps128(ab_vec));
833
- *result = f32_result;
834
- }
835
-
836
- inline simsimd_f64_t _mm256_reduce_add_ps_dbl(__m256 vec) {
809
+ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x8_haswell(__m256 vec) {
837
810
  // Convert the lower and higher 128-bit lanes of the input vector to double precision
838
811
  __m128 low_f32 = _mm256_castps256_ps128(vec);
839
812
  __m128 high_f32 = _mm256_extractf128_ps(vec, 1);
@@ -868,7 +841,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
868
841
  __m256 b_vec = _mm256_loadu_ps(b + i);
869
842
  ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
870
843
  }
871
- simsimd_f64_t ab = _mm256_reduce_add_ps_dbl(ab_vec);
844
+ simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
872
845
  for (; i < n; ++i)
873
846
  ab += a[i] * b[i];
874
847
  *results = ab;
@@ -903,7 +876,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
903
876
  // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
904
877
  // This way we can avoid the shuffling and the need for separate real and imaginary parts.
905
878
  // For the imaginary part of the product, we would need to swap the real and imaginary parts of
906
- // one of the vectors.
879
+ // one of the vectors. Moreover, `XOR` can be placed after the primary loop.
907
880
  // Both operations are quite cheap, and the throughput doubles from 2.5 GB/s to 5 GB/s.
908
881
  __m256 ab_real_vec = _mm256_setzero_ps();
909
882
  __m256 ab_imag_vec = _mm256_setzero_ps();
@@ -922,15 +895,17 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
922
895
  for (; i + 8 <= n; i += 8) {
923
896
  __m256 a_vec = _mm256_loadu_ps(a + i);
924
897
  __m256 b_vec = _mm256_loadu_ps(b + i);
925
- __m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
926
898
  __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
927
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
899
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
928
900
  ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
929
901
  }
930
902
 
903
+ // Flip the sign bit in every second scalar before accumulation:
904
+ ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
905
+
931
906
  // Reduce horizontal sums:
932
- simsimd_distance_t ab_real = _mm256_reduce_add_ps_dbl(ab_real_vec);
933
- simsimd_distance_t ab_imag = _mm256_reduce_add_ps_dbl(ab_imag_vec);
907
+ simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
908
+ simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
934
909
 
935
910
  // Handle the tail:
936
911
  for (; i + 2 <= n; i += 2) {
@@ -963,14 +938,16 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
963
938
  __m256 a_vec = _mm256_loadu_ps(a + i);
964
939
  __m256 b_vec = _mm256_loadu_ps(b + i);
965
940
  ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
966
- a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
967
941
  b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
968
942
  ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
969
943
  }
970
944
 
945
+ // Flip the sign bit in every second scalar before accumulation:
946
+ ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
947
+
971
948
  // Reduce horizontal sums:
972
- simsimd_distance_t ab_real = _mm256_reduce_add_ps_dbl(ab_real_vec);
973
- simsimd_distance_t ab_imag = _mm256_reduce_add_ps_dbl(ab_imag_vec);
949
+ simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
950
+ simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
974
951
 
975
952
  // Handle the tail:
976
953
  for (; i + 2 <= n; i += 2) {
@@ -982,6 +959,44 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
982
959
  results[1] = ab_imag;
983
960
  }
984
961
 
962
+ SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const* a, simsimd_size_t n) {
963
+ // In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
964
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
965
+ // loads and stores with vectorized math.
966
+ union {
967
+ __m128i vec;
968
+ simsimd_f16_t scalars[8];
969
+ } result;
970
+ simsimd_size_t i = 0;
971
+ for (; i < n; ++i)
972
+ result.scalars[i] = a[i];
973
+ for (; i < 8; ++i)
974
+ result.scalars[i] = 0;
975
+ return _mm256_cvtph_ps(result.vec);
976
+ }
977
+
978
+ SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
979
+ simsimd_distance_t* result) {
980
+ __m256 a_vec, b_vec;
981
+ __m256 ab_vec = _mm256_setzero_ps();
982
+
983
+ simsimd_dot_f16_haswell_cycle:
984
+ if (n < 8) {
985
+ a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
986
+ b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
987
+ n = 0;
988
+ } else {
989
+ a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
990
+ b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
991
+ n -= 8, a += 8, b += 8;
992
+ }
993
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
994
+ if (n)
995
+ goto simsimd_dot_f16_haswell_cycle;
996
+
997
+ *result = _simsimd_reduce_f32x8_haswell(ab_vec);
998
+ }
999
+
985
1000
  SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
986
1001
  simsimd_distance_t* results) {
987
1002
  // Ideally the implementation would load 256 bits worth of vector data at a time,
@@ -1004,40 +1019,24 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16
1004
1019
  3, 2, 1, 0, // Points to the first f32 in 128-bit lane
1005
1020
  7, 6, 5, 4 // Points to the second f32 in 128-bit lane
1006
1021
  );
1007
- simsimd_size_t i = 0;
1008
- for (; i + 8 <= n; i += 8) {
1009
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
1010
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
1011
- __m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
1012
- __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
1013
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
1014
- ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
1015
- }
1016
1022
 
1017
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
1018
- // function will run. It is extremely slow, so even for the tail, let's combine serial
1019
- // loads and stores with vectorized math.
1020
- if (i < n) {
1021
- union {
1022
- __m128i f16_vec;
1023
- simsimd_f16_t f16[8];
1024
- } a_padded_tail, b_padded_tail;
1025
- simsimd_size_t j = 0;
1026
- for (; i < n; ++i, ++j)
1027
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
1028
- for (; j < 8; ++j)
1029
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
1030
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
1031
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
1032
- __m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
1023
+ while (n >= 8) {
1024
+ __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
1025
+ __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
1033
1026
  __m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
1034
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
1027
+ ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
1035
1028
  ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
1029
+
1030
+ n -= 8, a += 8, b += 8;
1036
1031
  }
1037
1032
 
1038
- // Reduce horizontal sums:
1039
- results[0] = _mm256_reduce_add_ps_dbl(ab_real_vec);
1040
- results[1] = _mm256_reduce_add_ps_dbl(ab_imag_vec);
1033
+ // Flip the sign bit in every second scalar before accumulation:
1034
+ ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
1035
+
1036
+ // Reduce horizontal sums and aggregate with the tail:
1037
+ simsimd_dot_f16c_serial(a, b, n, results);
1038
+ results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
1039
+ results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
1041
1040
  }
1042
1041
 
1043
1042
  SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
@@ -1056,40 +1055,24 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
1056
1055
  3, 2, 1, 0, // Points to the first f32 in 128-bit lane
1057
1056
  7, 6, 5, 4 // Points to the second f32 in 128-bit lane
1058
1057
  );
1059
- simsimd_size_t i = 0;
1060
- for (; i + 8 <= n; i += 8) {
1061
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
1062
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
1063
- ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
1064
- a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
1065
- b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
1066
- ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
1067
- }
1068
1058
 
1069
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
1070
- // function will run. It is extremely slow, so even for the tail, let's combine serial
1071
- // loads and stores with vectorized math.
1072
- if (i < n) {
1073
- union {
1074
- __m128i f16_vec;
1075
- simsimd_f16_t f16[8];
1076
- } a_padded_tail, b_padded_tail;
1077
- simsimd_size_t j = 0;
1078
- for (; i < n; ++i, ++j)
1079
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
1080
- for (; j < 8; ++j)
1081
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
1082
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
1083
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
1059
+ while (n >= 8) {
1060
+ __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
1061
+ __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
1084
1062
  ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
1085
- a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
1086
1063
  b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
1087
1064
  ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
1065
+
1066
+ n -= 8, a += 8, b += 8;
1088
1067
  }
1089
1068
 
1090
- // Reduce horizontal sums:
1091
- results[0] = _mm256_reduce_add_ps_dbl(ab_real_vec);
1092
- results[1] = _mm256_reduce_add_ps_dbl(ab_imag_vec);
1069
+ // Flip the sign bit in every second scalar before accumulation:
1070
+ ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
1071
+
1072
+ // Reduce horizontal sums and aggregate with the tail:
1073
+ simsimd_dot_f16c_serial(a, b, n, results);
1074
+ results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
1075
+ results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
1093
1076
  }
1094
1077
 
1095
1078
  SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
@@ -1123,49 +1106,51 @@ SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
1123
1106
  // Take care of the tail:
1124
1107
  int ab = _mm_extract_epi32(ab_sum, 0);
1125
1108
  for (; i < n; ++i)
1126
- ab += a[i] * b[i];
1109
+ ab += (int)(a[i]) * b[i];
1127
1110
  *result = ab;
1128
1111
  }
1129
1112
 
1113
+ SIMSIMD_INTERNAL __m256 _simsimd_bf16x8_to_f32x8_haswell(__m128i a) {
1114
+ // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
1115
+ return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a), 16));
1116
+ }
1117
+
1118
+ SIMSIMD_INTERNAL __m128i _simsimd_partial_load_bf16x8_haswell(simsimd_bf16_t const* a, simsimd_size_t n) {
1119
+ // In case the software emulation for `bf16` scalars is enabled, the `simsimd_bf16_to_f32`
1120
+ // function will run. It is extremely slow, so even for the tail, let's combine serial
1121
+ // loads and stores with vectorized math.
1122
+ union {
1123
+ __m128i vec;
1124
+ simsimd_bf16_t scalars[8];
1125
+ } result;
1126
+ simsimd_size_t i = 0;
1127
+ for (; i < n; ++i)
1128
+ result.scalars[i] = a[i];
1129
+ for (; i < 8; ++i)
1130
+ result.scalars[i] = 0;
1131
+ return result.vec;
1132
+ }
1133
+
1130
1134
  SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1131
1135
  simsimd_distance_t* result) {
1136
+ __m128i a_vec, b_vec;
1132
1137
  __m256 ab_vec = _mm256_setzero_ps();
1133
- simsimd_size_t i = 0;
1134
- for (; i + 8 <= n; i += 8) {
1135
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
1136
- // x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
1137
- __m256 a_vec =
1138
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
1139
- __m256 b_vec =
1140
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
1141
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1142
- }
1143
1138
 
1144
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
1145
- // function will run. It is extremely slow, so even for the tail, let's combine serial
1146
- // loads and stores with vectorized math.
1147
- if (i < n) {
1148
- union {
1149
- __m128i bf16_vec;
1150
- simsimd_bf16_t bf16[8];
1151
- } a_padded_tail, b_padded_tail;
1152
- simsimd_size_t j = 0;
1153
- for (; i < n; ++i, ++j)
1154
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
1155
- for (; j < 8; ++j)
1156
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
1157
- __m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
1158
- __m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
1159
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1139
+ simsimd_dot_bf16_haswell_cycle:
1140
+ if (n < 8) {
1141
+ a_vec = _simsimd_partial_load_bf16x8_haswell(a, n);
1142
+ b_vec = _simsimd_partial_load_bf16x8_haswell(b, n);
1143
+ n = 0;
1144
+ } else {
1145
+ a_vec = _mm_loadu_si128((__m128i const*)a);
1146
+ b_vec = _mm_loadu_si128((__m128i const*)b);
1147
+ a += 8, b += 8, n -= 8;
1160
1148
  }
1149
+ ab_vec = _mm256_fmadd_ps(_simsimd_bf16x8_to_f32x8_haswell(a_vec), _simsimd_bf16x8_to_f32x8_haswell(b_vec), ab_vec);
1150
+ if (n)
1151
+ goto simsimd_dot_bf16_haswell_cycle;
1161
1152
 
1162
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
1163
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
1164
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
1165
-
1166
- simsimd_f32_t f32_result;
1167
- _mm_store_ss(&f32_result, _mm256_castps256_ps128(ab_vec));
1168
- *result = f32_result;
1153
+ *result = _simsimd_reduce_f32x8_haswell(ab_vec);
1169
1154
  }
1170
1155
 
1171
1156
  #pragma clang attribute pop
@@ -1174,8 +1159,15 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf
1174
1159
 
1175
1160
  #if SIMSIMD_TARGET_SKYLAKE
1176
1161
  #pragma GCC push_options
1177
- #pragma GCC target("avx512f", "avx512vl", "avx512bw", "bmi2")
1178
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
1162
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
1163
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
1164
+
1165
+ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x16_skylake(__m512 a) {
1166
+ __m512 x = _mm512_add_ps(a, _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 0, 3, 2)));
1167
+ __m128 r = _mm512_castps512_ps128(_mm512_add_ps(x, _mm512_shuffle_f32x4(x, x, _MM_SHUFFLE(0, 0, 0, 1))));
1168
+ r = _mm_hadd_ps(r, r);
1169
+ return _mm_cvtss_f32(_mm_hadd_ps(r, r));
1170
+ }
1179
1171
 
1180
1172
  SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1181
1173
  simsimd_distance_t* result) {
@@ -1197,7 +1189,7 @@ simsimd_dot_f32_skylake_cycle:
1197
1189
  if (n)
1198
1190
  goto simsimd_dot_f32_skylake_cycle;
1199
1191
 
1200
- *result = _mm512_reduce_add_ps(ab_vec);
1192
+ *result = _simsimd_reduce_f32x16_skylake(ab_vec);
1201
1193
  }
1202
1194
 
1203
1195
  SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
@@ -1225,11 +1217,10 @@ simsimd_dot_f64_skylake_cycle:
1225
1217
 
1226
1218
  SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1227
1219
  simsimd_distance_t* results) {
1228
-
1229
1220
  __m512 ab_real_vec = _mm512_setzero();
1230
1221
  __m512 ab_imag_vec = _mm512_setzero();
1231
1222
  __m512 a_vec;
1232
- __m512i b_vec;
1223
+ __m512 b_vec;
1233
1224
 
1234
1225
  // We take into account, that FMS is the same as FMA with a negative multiplier.
1235
1226
  // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
@@ -1247,27 +1238,29 @@ simsimd_dot_f32c_skylake_cycle:
1247
1238
  if (n < 16) {
1248
1239
  __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1249
1240
  a_vec = _mm512_maskz_loadu_ps(mask, a);
1250
- b_vec = _mm512_castps_si512(_mm512_maskz_loadu_ps(mask, b));
1241
+ b_vec = _mm512_maskz_loadu_ps(mask, b);
1251
1242
  n = 0;
1252
1243
  } else {
1253
1244
  a_vec = _mm512_loadu_ps(a);
1254
- b_vec = _mm512_castps_si512(_mm512_loadu_ps(b));
1245
+ b_vec = _mm512_loadu_ps(b);
1255
1246
  a += 16, b += 16, n -= 16;
1256
1247
  }
1257
- ab_real_vec = _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_xor_si512(b_vec, sign_flip_vec)), a_vec, ab_real_vec);
1258
- ab_imag_vec =
1259
- _mm512_fmadd_ps(_mm512_castsi512_ps(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), a_vec, ab_imag_vec);
1248
+ ab_real_vec = _mm512_fmadd_ps(b_vec, a_vec, ab_real_vec);
1249
+ ab_imag_vec = _mm512_fmadd_ps(
1250
+ _mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
1260
1251
  if (n)
1261
1252
  goto simsimd_dot_f32c_skylake_cycle;
1262
1253
 
1254
+ // Flip the sign bit in every second scalar before accumulation:
1255
+ ab_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_real_vec), sign_flip_vec));
1256
+
1263
1257
  // Reduce horizontal sums:
1264
- results[0] = _mm512_reduce_add_ps(ab_real_vec);
1265
- results[1] = _mm512_reduce_add_ps(ab_imag_vec);
1258
+ results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1259
+ results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1266
1260
  }
1267
1261
 
1268
1262
  SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1269
1263
  simsimd_distance_t* results) {
1270
-
1271
1264
  __m512 ab_real_vec = _mm512_setzero();
1272
1265
  __m512 ab_imag_vec = _mm512_setzero();
1273
1266
  __m512 a_vec;
@@ -1297,24 +1290,25 @@ simsimd_vdot_f32c_skylake_cycle:
1297
1290
  a += 16, b += 16, n -= 16;
1298
1291
  }
1299
1292
  ab_real_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_real_vec);
1300
- a_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(a_vec), sign_flip_vec));
1301
1293
  b_vec = _mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec));
1302
1294
  ab_imag_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_imag_vec);
1303
1295
  if (n)
1304
1296
  goto simsimd_vdot_f32c_skylake_cycle;
1305
1297
 
1298
+ // Flip the sign bit in every second scalar before accumulation:
1299
+ ab_imag_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_imag_vec), sign_flip_vec));
1300
+
1306
1301
  // Reduce horizontal sums:
1307
- results[0] = _mm512_reduce_add_ps(ab_real_vec);
1308
- results[1] = _mm512_reduce_add_ps(ab_imag_vec);
1302
+ results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1303
+ results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1309
1304
  }
1310
1305
 
1311
1306
  SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1312
1307
  simsimd_distance_t* results) {
1313
-
1314
1308
  __m512d ab_real_vec = _mm512_setzero_pd();
1315
1309
  __m512d ab_imag_vec = _mm512_setzero_pd();
1316
1310
  __m512d a_vec;
1317
- __m512i b_vec;
1311
+ __m512d b_vec;
1318
1312
 
1319
1313
  // We take into account, that FMS is the same as FMA with a negative multiplier.
1320
1314
  // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
@@ -1335,19 +1329,22 @@ simsimd_dot_f64c_skylake_cycle:
1335
1329
  if (n < 8) {
1336
1330
  __mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1337
1331
  a_vec = _mm512_maskz_loadu_pd(mask, a);
1338
- b_vec = _mm512_castpd_si512(_mm512_maskz_loadu_pd(mask, b));
1332
+ b_vec = _mm512_maskz_loadu_pd(mask, b);
1339
1333
  n = 0;
1340
1334
  } else {
1341
1335
  a_vec = _mm512_loadu_pd(a);
1342
- b_vec = _mm512_castpd_si512(_mm512_loadu_pd(b));
1336
+ b_vec = _mm512_loadu_pd(b);
1343
1337
  a += 8, b += 8, n -= 8;
1344
1338
  }
1345
- ab_real_vec = _mm512_fmadd_pd(_mm512_castsi512_pd(_mm512_xor_si512(b_vec, sign_flip_vec)), a_vec, ab_real_vec);
1346
- ab_imag_vec =
1347
- _mm512_fmadd_pd(_mm512_castsi512_pd(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), a_vec, ab_imag_vec);
1339
+ ab_real_vec = _mm512_fmadd_pd(b_vec, a_vec, ab_real_vec);
1340
+ ab_imag_vec = _mm512_fmadd_pd(
1341
+ _mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
1348
1342
  if (n)
1349
1343
  goto simsimd_dot_f64c_skylake_cycle;
1350
1344
 
1345
+ // Flip the sign bit in every second scalar before accumulation:
1346
+ ab_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_real_vec), sign_flip_vec));
1347
+
1351
1348
  // Reduce horizontal sums:
1352
1349
  results[0] = _mm512_reduce_add_pd(ab_real_vec);
1353
1350
  results[1] = _mm512_reduce_add_pd(ab_imag_vec);
@@ -1355,7 +1352,6 @@ simsimd_dot_f64c_skylake_cycle:
1355
1352
 
1356
1353
  SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1357
1354
  simsimd_distance_t* results) {
1358
-
1359
1355
  __m512d ab_real_vec = _mm512_setzero_pd();
1360
1356
  __m512d ab_imag_vec = _mm512_setzero_pd();
1361
1357
  __m512d a_vec;
@@ -1388,12 +1384,14 @@ simsimd_vdot_f64c_skylake_cycle:
1388
1384
  a += 8, b += 8, n -= 8;
1389
1385
  }
1390
1386
  ab_real_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_real_vec);
1391
- a_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(a_vec), sign_flip_vec));
1392
1387
  b_vec = _mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec));
1393
1388
  ab_imag_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_imag_vec);
1394
1389
  if (n)
1395
1390
  goto simsimd_vdot_f64c_skylake_cycle;
1396
1391
 
1392
+ // Flip the sign bit in every second scalar before accumulation:
1393
+ ab_imag_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_imag_vec), sign_flip_vec));
1394
+
1397
1395
  // Reduce horizontal sums:
1398
1396
  results[0] = _mm512_reduce_add_pd(ab_real_vec);
1399
1397
  results[1] = _mm512_reduce_add_pd(ab_imag_vec);
@@ -1405,8 +1403,9 @@ simsimd_vdot_f64c_skylake_cycle:
1405
1403
 
1406
1404
  #if SIMSIMD_TARGET_GENOA
1407
1405
  #pragma GCC push_options
1408
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1409
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), apply_to = function)
1406
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1407
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1408
+ apply_to = function)
1410
1409
 
1411
1410
  SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1412
1411
  simsimd_distance_t* result) {
@@ -1428,12 +1427,11 @@ simsimd_dot_bf16_genoa_cycle:
1428
1427
  if (n)
1429
1428
  goto simsimd_dot_bf16_genoa_cycle;
1430
1429
 
1431
- *result = _mm512_reduce_add_ps(ab_vec);
1430
+ *result = _simsimd_reduce_f32x16_skylake(ab_vec);
1432
1431
  }
1433
1432
 
1434
1433
  SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1435
1434
  simsimd_distance_t* results) {
1436
-
1437
1435
  __m512 ab_real_vec = _mm512_setzero_ps();
1438
1436
  __m512 ab_imag_vec = _mm512_setzero_ps();
1439
1437
  __m512i a_vec;
@@ -1470,13 +1468,12 @@ simsimd_dot_bf16c_genoa_cycle:
1470
1468
  goto simsimd_dot_bf16c_genoa_cycle;
1471
1469
 
1472
1470
  // Reduce horizontal sums:
1473
- results[0] = _mm512_reduce_add_ps(ab_real_vec);
1474
- results[1] = _mm512_reduce_add_ps(ab_imag_vec);
1471
+ results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1472
+ results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1475
1473
  }
1476
1474
 
1477
1475
  SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1478
1476
  simsimd_distance_t* results) {
1479
-
1480
1477
  __m512 ab_real_vec = _mm512_setzero_ps();
1481
1478
  __m512 ab_imag_vec = _mm512_setzero_ps();
1482
1479
  __m512i a_vec;
@@ -1514,8 +1511,8 @@ simsimd_dot_bf16c_genoa_cycle:
1514
1511
  goto simsimd_dot_bf16c_genoa_cycle;
1515
1512
 
1516
1513
  // Reduce horizontal sums:
1517
- results[0] = _mm512_reduce_add_ps(ab_real_vec);
1518
- results[1] = _mm512_reduce_add_ps(ab_imag_vec);
1514
+ results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
1515
+ results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
1519
1516
  }
1520
1517
 
1521
1518
  #pragma clang attribute pop
@@ -1524,8 +1521,9 @@ simsimd_dot_bf16c_genoa_cycle:
1524
1521
 
1525
1522
  #if SIMSIMD_TARGET_SAPPHIRE
1526
1523
  #pragma GCC push_options
1527
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
1528
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), apply_to = function)
1524
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
1525
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
1526
+ apply_to = function)
1529
1527
 
1530
1528
  SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1531
1529
  simsimd_distance_t* result) {
@@ -1552,7 +1550,6 @@ simsimd_dot_f16_sapphire_cycle:
1552
1550
 
1553
1551
  SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1554
1552
  simsimd_distance_t* results) {
1555
-
1556
1553
  __m512h ab_real_vec = _mm512_setzero_ph();
1557
1554
  __m512h ab_imag_vec = _mm512_setzero_ph();
1558
1555
  __m512i a_vec;
@@ -1582,6 +1579,7 @@ simsimd_dot_f16c_sapphire_cycle:
1582
1579
  b_vec = _mm512_loadu_epi16(b);
1583
1580
  a += 32, b += 32, n -= 32;
1584
1581
  }
1582
+ // TODO: Consider using `_mm512_fmaddsub`
1585
1583
  ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_xor_si512(b_vec, sign_flip_vec)),
1586
1584
  _mm512_castsi512_ph(a_vec), ab_real_vec);
1587
1585
  ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)),
@@ -1590,13 +1588,13 @@ simsimd_dot_f16c_sapphire_cycle:
1590
1588
  goto simsimd_dot_f16c_sapphire_cycle;
1591
1589
 
1592
1590
  // Reduce horizontal sums:
1591
+ // TODO: Optimize this with tree-like reductions
1593
1592
  results[0] = _mm512_reduce_add_ph(ab_real_vec);
1594
1593
  results[1] = _mm512_reduce_add_ph(ab_imag_vec);
1595
1594
  }
1596
1595
 
1597
1596
  SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1598
1597
  simsimd_distance_t* results) {
1599
-
1600
1598
  __m512h ab_real_vec = _mm512_setzero_ph();
1601
1599
  __m512h ab_imag_vec = _mm512_setzero_ph();
1602
1600
  __m512i a_vec;
@@ -1644,8 +1642,9 @@ simsimd_dot_f16c_sapphire_cycle:
1644
1642
 
1645
1643
  #if SIMSIMD_TARGET_ICE
1646
1644
  #pragma GCC push_options
1647
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1648
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), apply_to = function)
1645
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1646
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1647
+ apply_to = function)
1649
1648
 
1650
1649
  SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1651
1650
  simsimd_distance_t* result) {