usearch 2.16.3 → 2.16.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,8 @@
5
5
  * @date October 16, 2024
6
6
  *
7
7
  * Contains following element-wise operations:
8
+ * - Sum (Add): R[i] = A[i] + B[i]
9
+ * - Scale (Multiply): R[i] = Alpha * A[i]
8
10
  * - WSum or Weighted-Sum: R[i] = Alpha * A[i] + Beta * B[i]
9
11
  * - FMA or Fused-Multiply-Add: R[i] = Alpha * A[i] * B[i] + Beta * C[i]
10
12
  *
@@ -23,8 +25,8 @@
23
25
  * - 8-bit signed integers
24
26
  *
25
27
  * For hardware architectures:
26
- * - Arm: NEON, SVE
27
- * - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
28
+ * - Arm: NEON
29
+ * - x86: Haswell, Skylake, Sapphire
28
30
  *
29
31
  * We use `f16` for `i8` and `u8` arithmetic. This is because Arm received `f16` support earlier than `bf16`.
30
32
  * For example, Apple M1 has `f16` support and `bf16` was only added in M2. On the other hand, on paper,
@@ -34,8 +36,8 @@
34
36
  * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
35
37
  * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
36
38
  */
37
- #ifndef SIMSIMD_FMA_H
38
- #define SIMSIMD_FMA_H
39
+ #ifndef SIMSIMD_ELEMENTWISE_H
40
+ #define SIMSIMD_ELEMENTWISE_H
39
41
 
40
42
  #include "types.h"
41
43
 
@@ -270,11 +272,60 @@ SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire(
270
272
  #pragma GCC target("avx2", "f16c", "fma")
271
273
  #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
272
274
 
275
+ SIMSIMD_PUBLIC void simsimd_sum_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
276
+ simsimd_f32_t *result) {
277
+ // The main loop:
278
+ simsimd_size_t i = 0;
279
+ for (; i + 8 <= n; i += 8) {
280
+ __m256 a_vec = _mm256_loadu_ps(a + i);
281
+ __m256 b_vec = _mm256_loadu_ps(b + i);
282
+ __m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
283
+ _mm256_storeu_ps(result + i, sum_vec);
284
+ }
285
+
286
+ // The tail:
287
+ for (; i < n; ++i) result[i] = a[i] + b[i];
288
+ }
289
+
290
+ SIMSIMD_PUBLIC void simsimd_scale_f32_haswell(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
291
+ simsimd_f32_t *result) {
292
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
293
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
294
+
295
+ // The main loop:
296
+ simsimd_size_t i = 0;
297
+ for (; i + 8 <= n; i += 8) {
298
+ __m256 a_vec = _mm256_loadu_ps(a + i);
299
+ __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
300
+ _mm256_storeu_ps(result + i, sum_vec);
301
+ }
302
+
303
+ // The tail:
304
+ for (; i < n; ++i) result[i] = alpha_f32 * a[i];
305
+ }
306
+
273
307
  SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
274
308
  simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
275
309
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
276
310
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
277
311
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
312
+
313
+ // There are are several special cases we may want to implement:
314
+ // 1. Simple addition, when both weights are equal to 1.0.
315
+ if (alpha == 1 && beta == 1) {
316
+ // In this case we can avoid expensive multiplications.
317
+ simsimd_sum_f32_haswell(a, b, n, result);
318
+ return;
319
+ }
320
+ // 2. Just scaling, when one of the weights is equal to zero.
321
+ else if (alpha == 0 || beta == 0) {
322
+ // In this case we can avoid half of the load instructions.
323
+ if (beta == 0) { simsimd_scale_f32_haswell(a, n, alpha, result); }
324
+ else { simsimd_scale_f32_haswell(b, n, beta, result); }
325
+ return;
326
+ }
327
+
328
+ // The general case.
278
329
  __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
279
330
  __m256 beta_vec = _mm256_set1_ps(beta_f32);
280
331
 
@@ -283,9 +334,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
283
334
  for (; i + 8 <= n; i += 8) {
284
335
  __m256 a_vec = _mm256_loadu_ps(a + i);
285
336
  __m256 b_vec = _mm256_loadu_ps(b + i);
286
- __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
287
- __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
288
- __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
337
+ __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
338
+ __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
339
+ __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
289
340
  _mm256_storeu_ps(result + i, sum_vec);
290
341
  }
291
342
 
@@ -293,9 +344,57 @@ SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
293
344
  for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
294
345
  }
295
346
 
347
+ SIMSIMD_PUBLIC void simsimd_sum_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
348
+ simsimd_f64_t *result) {
349
+ // The main loop:
350
+ simsimd_size_t i = 0;
351
+ for (; i + 4 <= n; i += 4) {
352
+ __m256d a_vec = _mm256_loadu_pd(a + i);
353
+ __m256d b_vec = _mm256_loadu_pd(b + i);
354
+ __m256d sum_vec = _mm256_add_pd(a_vec, b_vec);
355
+ _mm256_storeu_pd(result + i, sum_vec);
356
+ }
357
+
358
+ // The tail:
359
+ for (; i < n; ++i) result[i] = a[i] + b[i];
360
+ }
361
+
362
+ SIMSIMD_PUBLIC void simsimd_scale_f64_haswell(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
363
+ simsimd_f64_t *result) {
364
+ __m256d alpha_vec = _mm256_set1_pd(alpha);
365
+
366
+ // The main loop:
367
+ simsimd_size_t i = 0;
368
+ for (; i + 4 <= n; i += 4) {
369
+ __m256d a_vec = _mm256_loadu_pd(a + i);
370
+ __m256d sum_vec = _mm256_mul_pd(a_vec, alpha_vec);
371
+ _mm256_storeu_pd(result + i, sum_vec);
372
+ }
373
+
374
+ // The tail:
375
+ for (; i < n; ++i) result[i] = alpha * a[i];
376
+ }
377
+
296
378
  SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
297
379
  simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
298
380
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
381
+
382
+ // There are are several special cases we may want to implement:
383
+ // 1. Simple addition, when both weights are equal to 1.0.
384
+ if (alpha == 1 && beta == 1) {
385
+ // In this case we can avoid expensive multiplications.
386
+ simsimd_sum_f64_haswell(a, b, n, result);
387
+ return;
388
+ }
389
+ // 2. Just scaling, when one of the weights is equal to zero.
390
+ else if (alpha == 0 || beta == 0) {
391
+ // In this case we can avoid half of the load instructions.
392
+ if (beta == 0) { simsimd_scale_f64_haswell(a, n, alpha, result); }
393
+ else { simsimd_scale_f64_haswell(b, n, beta, result); }
394
+ return;
395
+ }
396
+
397
+ // The general case.
299
398
  __m256d alpha_vec = _mm256_set1_pd(alpha);
300
399
  __m256d beta_vec = _mm256_set1_pd(beta);
301
400
 
@@ -304,9 +403,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
304
403
  for (; i + 4 <= n; i += 4) {
305
404
  __m256d a_vec = _mm256_loadu_pd(a + i);
306
405
  __m256d b_vec = _mm256_loadu_pd(b + i);
307
- __m256d a_scaled = _mm256_mul_pd(a_vec, alpha_vec);
308
- __m256d b_scaled = _mm256_mul_pd(b_vec, beta_vec);
309
- __m256d sum_vec = _mm256_add_pd(a_scaled, b_scaled);
406
+ __m256d a_scaled_vec = _mm256_mul_pd(a_vec, alpha_vec);
407
+ __m256d b_scaled_vec = _mm256_mul_pd(b_vec, beta_vec);
408
+ __m256d sum_vec = _mm256_add_pd(a_scaled_vec, b_scaled_vec);
310
409
  _mm256_storeu_pd(result + i, sum_vec);
311
410
  }
312
411
 
@@ -314,9 +413,73 @@ SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
314
413
  for (; i < n; ++i) result[i] = alpha * a[i] + beta * b[i];
315
414
  }
316
415
 
416
+ SIMSIMD_PUBLIC void simsimd_sum_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
417
+ simsimd_f16_t *result) {
418
+
419
+ // The main loop:
420
+ simsimd_size_t i = 0;
421
+ for (; i + 8 <= n; i += 8) {
422
+ __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
423
+ __m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
424
+ __m256 a_vec = _mm256_cvtph_ps(a_f16);
425
+ __m256 b_vec = _mm256_cvtph_ps(b_f16);
426
+ __m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
427
+ __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
428
+ _mm_storeu_si128((__m128i *)(result + i), sum_f16);
429
+ }
430
+
431
+ // The tail:
432
+ for (; i < n; ++i) {
433
+ simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
434
+ simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
435
+ simsimd_f32_t sum = ai + bi;
436
+ SIMSIMD_F32_TO_F16(sum, result + i);
437
+ }
438
+ }
439
+
440
+ SIMSIMD_PUBLIC void simsimd_scale_f16_haswell(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
441
+ simsimd_f16_t *result) {
442
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
443
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
444
+
445
+ // The main loop:
446
+ simsimd_size_t i = 0;
447
+ for (; i + 8 <= n; i += 8) {
448
+ __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
449
+ __m256 a_vec = _mm256_cvtph_ps(a_f16);
450
+ __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
451
+ __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
452
+ _mm_storeu_si128((__m128i *)(result + i), sum_f16);
453
+ }
454
+
455
+ // The tail:
456
+ for (; i < n; ++i) {
457
+ simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
458
+ simsimd_f32_t sum = alpha_f32 * ai;
459
+ SIMSIMD_F32_TO_F16(sum, result + i);
460
+ }
461
+ }
462
+
317
463
  SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
318
464
  simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
319
465
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
466
+
467
+ // There are are several special cases we may want to implement:
468
+ // 1. Simple addition, when both weights are equal to 1.0.
469
+ if (alpha == 1 && beta == 1) {
470
+ // In this case we can avoid expensive multiplications.
471
+ simsimd_sum_f16_haswell(a, b, n, result);
472
+ return;
473
+ }
474
+ // 2. Just scaling, when one of the weights is equal to zero.
475
+ else if (alpha == 0 || beta == 0) {
476
+ // In this case we can avoid half of the load instructions.
477
+ if (beta == 0) { simsimd_scale_f16_haswell(a, n, alpha, result); }
478
+ else { simsimd_scale_f16_haswell(b, n, beta, result); }
479
+ return;
480
+ }
481
+
482
+ // The general case.
320
483
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
321
484
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
322
485
  __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
@@ -325,13 +488,13 @@ SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
325
488
  // The main loop:
326
489
  simsimd_size_t i = 0;
327
490
  for (; i + 8 <= n; i += 8) {
328
- __m128i a_f16 = _mm_loadu_si128((__m128i const *)(a + i));
329
- __m128i b_f16 = _mm_loadu_si128((__m128i const *)(b + i));
491
+ __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
492
+ __m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
330
493
  __m256 a_vec = _mm256_cvtph_ps(a_f16);
331
494
  __m256 b_vec = _mm256_cvtph_ps(b_f16);
332
- __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
333
- __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
334
- __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
495
+ __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
496
+ __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
497
+ __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
335
498
  __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
336
499
  _mm_storeu_si128((__m128i *)(result + i), sum_f16);
337
500
  }
@@ -345,9 +508,72 @@ SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
345
508
  }
346
509
  }
347
510
 
511
+ SIMSIMD_PUBLIC void simsimd_sum_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
512
+ simsimd_bf16_t *result) {
513
+ // The main loop:
514
+ simsimd_size_t i = 0;
515
+ for (; i + 8 <= n; i += 8) {
516
+ __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
517
+ __m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
518
+ __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
519
+ __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
520
+ __m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
521
+ __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
522
+ _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
523
+ }
524
+
525
+ // The tail:
526
+ for (; i < n; ++i) {
527
+ simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
528
+ simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
529
+ simsimd_f32_t sum = ai + bi;
530
+ SIMSIMD_F32_TO_BF16(sum, result + i);
531
+ }
532
+ }
533
+
534
+ SIMSIMD_PUBLIC void simsimd_scale_bf16_haswell(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
535
+ simsimd_bf16_t *result) {
536
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
537
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
538
+
539
+ // The main loop:
540
+ simsimd_size_t i = 0;
541
+ for (; i + 8 <= n; i += 8) {
542
+ __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
543
+ __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
544
+ __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
545
+ __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
546
+ _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
547
+ }
548
+
549
+ // The tail:
550
+ for (; i < n; ++i) {
551
+ simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
552
+ simsimd_f32_t sum = alpha_f32 * ai;
553
+ SIMSIMD_F32_TO_BF16(sum, result + i);
554
+ }
555
+ }
556
+
348
557
  SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
349
558
  simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
350
559
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
560
+
561
+ // There are are several special cases we may want to implement:
562
+ // 1. Simple addition, when both weights are equal to 1.0.
563
+ if (alpha == 1 && beta == 1) {
564
+ // In this case we can avoid expensive multiplications.
565
+ simsimd_sum_bf16_haswell(a, b, n, result);
566
+ return;
567
+ }
568
+ // 2. Just scaling, when one of the weights is equal to zero.
569
+ else if (alpha == 0 || beta == 0) {
570
+ // In this case we can avoid half of the load instructions.
571
+ if (beta == 0) { simsimd_scale_bf16_haswell(a, n, alpha, result); }
572
+ else { simsimd_scale_bf16_haswell(b, n, beta, result); }
573
+ return;
574
+ }
575
+
576
+ // The general case.
351
577
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
352
578
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
353
579
  __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
@@ -356,13 +582,13 @@ SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
356
582
  // The main loop:
357
583
  simsimd_size_t i = 0;
358
584
  for (; i + 8 <= n; i += 8) {
359
- __m128i a_bf16 = _mm_loadu_si128((__m128i const *)(a + i));
360
- __m128i b_bf16 = _mm_loadu_si128((__m128i const *)(b + i));
585
+ __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
586
+ __m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
361
587
  __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
362
588
  __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
363
- __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
364
- __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
365
- __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
589
+ __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
590
+ __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
591
+ __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
366
592
  __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
367
593
  _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
368
594
  }
@@ -435,9 +661,9 @@ SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
435
661
  // The main loop:
436
662
  simsimd_size_t i = 0;
437
663
  for (; i + 8 <= n; i += 8) {
438
- __m128i a_f16 = _mm_loadu_si128((__m128i const *)(a + i));
439
- __m128i b_f16 = _mm_loadu_si128((__m128i const *)(b + i));
440
- __m128i c_f16 = _mm_loadu_si128((__m128i const *)(c + i));
664
+ __m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
665
+ __m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
666
+ __m128i c_f16 = _mm_lddqu_si128((__m128i const *)(c + i));
441
667
  __m256 a_vec = _mm256_cvtph_ps(a_f16);
442
668
  __m256 b_vec = _mm256_cvtph_ps(b_f16);
443
669
  __m256 c_vec = _mm256_cvtph_ps(c_f16);
@@ -470,9 +696,9 @@ SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( /
470
696
  // The main loop:
471
697
  simsimd_size_t i = 0;
472
698
  for (; i + 8 <= n; i += 8) {
473
- __m128i a_bf16 = _mm_loadu_si128((__m128i const *)(a + i));
474
- __m128i b_bf16 = _mm_loadu_si128((__m128i const *)(b + i));
475
- __m128i c_bf16 = _mm_loadu_si128((__m128i const *)(c + i));
699
+ __m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
700
+ __m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
701
+ __m128i c_bf16 = _mm_lddqu_si128((__m128i const *)(c + i));
476
702
  __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
477
703
  __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
478
704
  __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(c_bf16);
@@ -494,10 +720,88 @@ SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( /
494
720
  }
495
721
  }
496
722
 
723
+ SIMSIMD_PUBLIC void simsimd_sum_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
724
+ simsimd_i8_t *result) {
725
+ // The main loop:
726
+ simsimd_size_t i = 0;
727
+ for (; i + 32 <= n; i += 32) {
728
+ __m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
729
+ __m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
730
+ __m256i sum_vec = _mm256_adds_epi8(a_vec, b_vec);
731
+ _mm256_storeu_si256((__m256i *)(result + i), sum_vec);
732
+ }
733
+
734
+ // The tail:
735
+ for (; i < n; ++i) {
736
+ simsimd_f32_t ai = a[i], bi = b[i];
737
+ simsimd_f32_t sum = ai + bi;
738
+ SIMSIMD_F32_TO_U8(sum, result + i);
739
+ }
740
+ }
741
+
742
+ SIMSIMD_PUBLIC void simsimd_scale_i8_haswell(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
743
+ simsimd_i8_t *result) {
744
+
745
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
746
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
747
+ int sum_i32s[8], a_i32s[8];
748
+
749
+ // The main loop:
750
+ simsimd_size_t i = 0;
751
+ for (; i + 8 <= n; i += 8) {
752
+ //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
753
+ //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
754
+ a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
755
+ a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
756
+ //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
757
+ //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
758
+ __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
759
+ // The normal part.
760
+ __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
761
+ // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
762
+ __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
763
+ sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
764
+ sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
765
+ // Export into a serial buffer.
766
+ _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
767
+ result[i + 0] = (simsimd_i8_t)sum_i32s[0];
768
+ result[i + 1] = (simsimd_i8_t)sum_i32s[1];
769
+ result[i + 2] = (simsimd_i8_t)sum_i32s[2];
770
+ result[i + 3] = (simsimd_i8_t)sum_i32s[3];
771
+ result[i + 4] = (simsimd_i8_t)sum_i32s[4];
772
+ result[i + 5] = (simsimd_i8_t)sum_i32s[5];
773
+ result[i + 6] = (simsimd_i8_t)sum_i32s[6];
774
+ result[i + 7] = (simsimd_i8_t)sum_i32s[7];
775
+ }
776
+
777
+ // The tail:
778
+ for (; i < n; ++i) {
779
+ simsimd_f32_t ai = a[i];
780
+ simsimd_f32_t sum = alpha_f32 * ai;
781
+ SIMSIMD_F32_TO_I8(sum, result + i);
782
+ }
783
+ }
784
+
497
785
  SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
498
786
  simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
499
787
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
500
788
 
789
+ // There are are several special cases we may want to implement:
790
+ // 1. Simple addition, when both weights are equal to 1.0.
791
+ if (alpha == 1 && beta == 1) {
792
+ // In this case we can avoid expensive multiplications.
793
+ simsimd_sum_i8_haswell(a, b, n, result);
794
+ return;
795
+ }
796
+ // 2. Just scaling, when one of the weights is equal to zero.
797
+ else if (alpha == 0 || beta == 0) {
798
+ // In this case we can avoid half of the load instructions.
799
+ if (beta == 0) { simsimd_scale_i8_haswell(a, n, alpha, result); }
800
+ else { simsimd_scale_i8_haswell(b, n, beta, result); }
801
+ return;
802
+ }
803
+
804
+ // The general case.
501
805
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
502
806
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
503
807
  __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
@@ -518,9 +822,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
518
822
  __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
519
823
  __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
520
824
  // The normal part.
521
- __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
522
- __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
523
- __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
825
+ __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
826
+ __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
827
+ __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
524
828
  // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
525
829
  __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
526
830
  sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
@@ -545,10 +849,88 @@ SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
545
849
  }
546
850
  }
547
851
 
852
+ SIMSIMD_PUBLIC void simsimd_sum_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
853
+ simsimd_u8_t *result) {
854
+ // The main loop:
855
+ simsimd_size_t i = 0;
856
+ for (; i + 32 <= n; i += 32) {
857
+ __m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
858
+ __m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
859
+ __m256i sum_vec = _mm256_adds_epu8(a_vec, b_vec);
860
+ _mm256_storeu_si256((__m256i *)(result + i), sum_vec);
861
+ }
862
+
863
+ // The tail:
864
+ for (; i < n; ++i) {
865
+ simsimd_f32_t ai = a[i], bi = b[i];
866
+ simsimd_f32_t sum = ai + bi;
867
+ SIMSIMD_F32_TO_U8(sum, result + i);
868
+ }
869
+ }
870
+
871
+ SIMSIMD_PUBLIC void simsimd_scale_u8_haswell(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
872
+ simsimd_u8_t *result) {
873
+
874
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
875
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
876
+ int sum_i32s[8], a_i32s[8];
877
+
878
+ // The main loop:
879
+ simsimd_size_t i = 0;
880
+ for (; i + 8 <= n; i += 8) {
881
+ //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
882
+ //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
883
+ a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
884
+ a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
885
+ //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
886
+ //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
887
+ __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
888
+ // The normal part.
889
+ __m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
890
+ // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
891
+ __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
892
+ sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
893
+ sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
894
+ // Export into a serial buffer.
895
+ _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
896
+ result[i + 0] = (simsimd_u8_t)sum_i32s[0];
897
+ result[i + 1] = (simsimd_u8_t)sum_i32s[1];
898
+ result[i + 2] = (simsimd_u8_t)sum_i32s[2];
899
+ result[i + 3] = (simsimd_u8_t)sum_i32s[3];
900
+ result[i + 4] = (simsimd_u8_t)sum_i32s[4];
901
+ result[i + 5] = (simsimd_u8_t)sum_i32s[5];
902
+ result[i + 6] = (simsimd_u8_t)sum_i32s[6];
903
+ result[i + 7] = (simsimd_u8_t)sum_i32s[7];
904
+ }
905
+
906
+ // The tail:
907
+ for (; i < n; ++i) {
908
+ simsimd_f32_t ai = a[i];
909
+ simsimd_f32_t sum = alpha_f32 * ai;
910
+ SIMSIMD_F32_TO_U8(sum, result + i);
911
+ }
912
+ }
913
+
548
914
  SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
549
915
  simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
550
916
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
551
917
 
918
+ // There are are several special cases we may want to implement:
919
+ // 1. Simple addition, when both weights are equal to 1.0.
920
+ if (alpha == 1 && beta == 1) {
921
+ // In this case we can avoid expensive multiplications.
922
+ simsimd_sum_u8_haswell(a, b, n, result);
923
+ return;
924
+ }
925
+ // 2. Just scaling, when one of the weights is equal to zero.
926
+ else if (alpha == 0 || beta == 0) {
927
+ // In this case we can avoid half of the load instructions.
928
+ if (beta == 0) { simsimd_scale_u8_haswell(a, n, alpha, result); }
929
+ else { simsimd_scale_u8_haswell(b, n, beta, result); }
930
+ return;
931
+ }
932
+
933
+ // The general case.
552
934
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
553
935
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
554
936
  __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
@@ -569,9 +951,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
569
951
  __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
570
952
  __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
571
953
  // The normal part.
572
- __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
573
- __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
574
- __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
954
+ __m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
955
+ __m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
956
+ __m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
575
957
  // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
576
958
  __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
577
959
  sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
@@ -715,14 +1097,73 @@ SIMSIMD_PUBLIC void simsimd_fma_u8_haswell(
715
1097
  #pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
716
1098
  #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
717
1099
 
1100
+ SIMSIMD_PUBLIC void simsimd_sum_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1101
+ simsimd_f64_t *result) {
1102
+ __m512d a_vec, b_vec, sum_vec;
1103
+ __mmask8 mask = 0xFF;
1104
+ simsimd_sum_f64_skylake_cycle:
1105
+ if (n < 8) {
1106
+ mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1107
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
1108
+ b_vec = _mm512_maskz_loadu_pd(mask, b);
1109
+ n = 0;
1110
+ }
1111
+ else {
1112
+ a_vec = _mm512_loadu_pd(a);
1113
+ b_vec = _mm512_loadu_pd(b);
1114
+ a += 8, b += 8, n -= 8;
1115
+ }
1116
+ sum_vec = _mm512_add_pd(a_vec, b_vec);
1117
+ _mm512_mask_storeu_pd(result, mask, sum_vec);
1118
+ result += 8;
1119
+ if (n) goto simsimd_sum_f64_skylake_cycle;
1120
+ }
1121
+
1122
+ SIMSIMD_PUBLIC void simsimd_scale_f64_skylake(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1123
+ simsimd_f64_t *result) {
1124
+ __m512d alpha_vec = _mm512_set1_pd(alpha);
1125
+ __m512d a_vec, b_vec, a_scaled_vec, sum_vec;
1126
+ __mmask8 mask = 0xFF;
1127
+ simsimd_scale_f64_skylake_cycle:
1128
+ if (n < 8) {
1129
+ mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1130
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
1131
+ n = 0;
1132
+ }
1133
+ else {
1134
+ a_vec = _mm512_loadu_pd(a);
1135
+ a += 8, n -= 8;
1136
+ }
1137
+ sum_vec = _mm512_mul_pd(a_vec, alpha_vec);
1138
+ _mm512_mask_storeu_pd(result, mask, sum_vec);
1139
+ result += 8;
1140
+ if (n) goto simsimd_scale_f64_skylake_cycle;
1141
+ }
1142
+
718
1143
  SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
719
1144
  simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
720
1145
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
1146
+
1147
+ // There are are several special cases we may want to implement:
1148
+ // 1. Simple addition, when both weights are equal to 1.0.
1149
+ if (alpha == 1 && beta == 1) {
1150
+ // In this case we can avoid expensive multiplications.
1151
+ simsimd_sum_f64_skylake(a, b, n, result);
1152
+ return;
1153
+ }
1154
+ // 2. Just scaling, when one of the weights is equal to zero.
1155
+ else if (alpha == 0 || beta == 0) {
1156
+ // In this case we can avoid half of the load instructions.
1157
+ if (beta == 0) { simsimd_scale_f64_skylake(a, n, alpha, result); }
1158
+ else { simsimd_scale_f64_skylake(b, n, beta, result); }
1159
+ return;
1160
+ }
1161
+
1162
+ // The general case.
721
1163
  __m512d alpha_vec = _mm512_set1_pd(alpha);
722
1164
  __m512d beta_vec = _mm512_set1_pd(beta);
723
1165
  __m512d a_vec, b_vec, a_scaled_vec, sum_vec;
724
1166
  __mmask8 mask = 0xFF;
725
-
726
1167
  simsimd_wsum_f64_skylake_cycle:
727
1168
  if (n < 8) {
728
1169
  mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
@@ -742,14 +1183,75 @@ simsimd_wsum_f64_skylake_cycle:
742
1183
  if (n) goto simsimd_wsum_f64_skylake_cycle;
743
1184
  }
744
1185
 
1186
+ SIMSIMD_PUBLIC void simsimd_sum_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1187
+ simsimd_f32_t *result) {
1188
+ __m512 a_vec, b_vec, sum_vec;
1189
+ __mmask16 mask = 0xFFFF;
1190
+
1191
+ simsimd_sum_f32_skylake_cycle:
1192
+ if (n < 16) {
1193
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1194
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
1195
+ b_vec = _mm512_maskz_loadu_ps(mask, b);
1196
+ n = 0;
1197
+ }
1198
+ else {
1199
+ a_vec = _mm512_loadu_ps(a);
1200
+ b_vec = _mm512_loadu_ps(b);
1201
+ a += 16, b += 16, n -= 16;
1202
+ }
1203
+ sum_vec = _mm512_add_ps(a_vec, b_vec);
1204
+ _mm512_mask_storeu_ps(result, mask, sum_vec);
1205
+ result += 16;
1206
+ if (n) goto simsimd_sum_f32_skylake_cycle;
1207
+ }
1208
+
1209
+ SIMSIMD_PUBLIC void simsimd_scale_f32_skylake(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1210
+ simsimd_f32_t *result) {
1211
+ __m512 alpha_vec = _mm512_set1_ps(alpha);
1212
+ __m512 a_vec, sum_vec;
1213
+ __mmask16 mask = 0xFFFF;
1214
+
1215
+ simsimd_scale_f32_skylake_cycle:
1216
+ if (n < 16) {
1217
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1218
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
1219
+ n = 0;
1220
+ }
1221
+ else {
1222
+ a_vec = _mm512_loadu_ps(a);
1223
+ a += 16, n -= 16;
1224
+ }
1225
+ sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
1226
+ _mm512_mask_storeu_ps(result, mask, sum_vec);
1227
+ result += 16;
1228
+ if (n) goto simsimd_scale_f32_skylake_cycle;
1229
+ }
1230
+
745
1231
  SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
746
1232
  simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
747
1233
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1234
+
1235
+ // There are are several special cases we may want to implement:
1236
+ // 1. Simple addition, when both weights are equal to 1.0.
1237
+ if (alpha == 1 && beta == 1) {
1238
+ // In this case we can avoid expensive multiplications.
1239
+ simsimd_sum_f32_skylake(a, b, n, result);
1240
+ return;
1241
+ }
1242
+ // 2. Just scaling, when one of the weights is equal to zero.
1243
+ else if (alpha == 0 || beta == 0) {
1244
+ // In this case we can avoid half of the load instructions.
1245
+ if (beta == 0) { simsimd_scale_f32_skylake(a, n, alpha, result); }
1246
+ else { simsimd_scale_f32_skylake(b, n, beta, result); }
1247
+ return;
1248
+ }
1249
+
1250
+ // The general case.
748
1251
  __m512 alpha_vec = _mm512_set1_ps(alpha);
749
1252
  __m512 beta_vec = _mm512_set1_ps(beta);
750
1253
  __m512 a_vec, b_vec, a_scaled_vec, sum_vec;
751
1254
  __mmask16 mask = 0xFFFF;
752
-
753
1255
  simsimd_wsum_f32_skylake_cycle:
754
1256
  if (n < 16) {
755
1257
  mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
@@ -769,15 +1271,81 @@ simsimd_wsum_f32_skylake_cycle:
769
1271
  if (n) goto simsimd_wsum_f32_skylake_cycle;
770
1272
  }
771
1273
 
1274
+ SIMSIMD_PUBLIC void simsimd_sum_bf16_skylake(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1275
+ simsimd_bf16_t *result) {
1276
+ __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
1277
+ __m512 a_vec, b_vec, sum_vec;
1278
+ __mmask16 mask = 0xFFFF;
1279
+ simsimd_sum_bf16_skylake_cycle:
1280
+ if (n < 16) {
1281
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1282
+ a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
1283
+ b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
1284
+ n = 0;
1285
+ }
1286
+ else {
1287
+ a_bf16_vec = _mm256_loadu_epi16(a);
1288
+ b_bf16_vec = _mm256_loadu_epi16(b);
1289
+ a += 16, b += 16, n -= 16;
1290
+ }
1291
+ a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
1292
+ b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
1293
+ sum_vec = _mm512_add_ps(a_vec, b_vec);
1294
+ sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
1295
+ _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
1296
+ result += 16;
1297
+ if (n) goto simsimd_sum_bf16_skylake_cycle;
1298
+ }
1299
+
1300
+ SIMSIMD_PUBLIC void simsimd_scale_bf16_skylake(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1301
+ simsimd_bf16_t *result) {
1302
+ __m512 alpha_vec = _mm512_set1_ps(alpha);
1303
+ __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
1304
+ __m512 a_vec, b_vec, sum_vec;
1305
+ __mmask16 mask = 0xFFFF;
1306
+ simsimd_wsum_bf16_skylake_cycle:
1307
+ if (n < 16) {
1308
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
1309
+ a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
1310
+ n = 0;
1311
+ }
1312
+ else {
1313
+ a_bf16_vec = _mm256_loadu_epi16(a);
1314
+ a += 16, n -= 16;
1315
+ }
1316
+ a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
1317
+ sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
1318
+ sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
1319
+ _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
1320
+ result += 16;
1321
+ if (n) goto simsimd_wsum_bf16_skylake_cycle;
1322
+ }
1323
+
772
1324
  SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
773
1325
  simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
774
1326
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
1327
+
1328
+ // There are are several special cases we may want to implement:
1329
+ // 1. Simple addition, when both weights are equal to 1.0.
1330
+ if (alpha == 1 && beta == 1) {
1331
+ // In this case we can avoid expensive multiplications.
1332
+ simsimd_sum_bf16_skylake(a, b, n, result);
1333
+ return;
1334
+ }
1335
+ // 2. Just scaling, when one of the weights is equal to zero.
1336
+ else if (alpha == 0 || beta == 0) {
1337
+ // In this case we can avoid half of the load instructions.
1338
+ if (beta == 0) { simsimd_scale_bf16_skylake(a, n, alpha, result); }
1339
+ else { simsimd_scale_bf16_skylake(b, n, beta, result); }
1340
+ return;
1341
+ }
1342
+
1343
+ // The general case.
775
1344
  __m512 alpha_vec = _mm512_set1_ps(alpha);
776
1345
  __m512 beta_vec = _mm512_set1_ps(beta);
777
1346
  __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
778
1347
  __m512 a_vec, b_vec, a_scaled_vec, sum_vec;
779
1348
  __mmask16 mask = 0xFFFF;
780
-
781
1349
  simsimd_wsum_bf16_skylake_cycle:
782
1350
  if (n < 16) {
783
1351
  mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
@@ -895,26 +1463,87 @@ simsimd_fma_bf16_skylake_cycle:
895
1463
  if (n) goto simsimd_fma_bf16_skylake_cycle;
896
1464
  }
897
1465
 
898
- #pragma clang attribute pop
899
- #pragma GCC pop_options
900
- #endif // SIMSIMD_TARGET_SKYLAKE
1466
+ #pragma clang attribute pop
1467
+ #pragma GCC pop_options
1468
+ #endif // SIMSIMD_TARGET_SKYLAKE
1469
+
1470
+ #if SIMSIMD_TARGET_SAPPHIRE
1471
+ #pragma GCC push_options
1472
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
1473
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
1474
+ apply_to = function)
1475
+
1476
+ SIMSIMD_PUBLIC void simsimd_sum_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1477
+ simsimd_f16_t *result) {
1478
+ __mmask32 mask = 0xFFFFFFFF;
1479
+ __m512h a_f16_vec, b_f16_vec;
1480
+ __m512h sum_f16_vec;
1481
+ simsimd_sum_f16_sapphire_cycle:
1482
+ if (n < 32) {
1483
+ mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1484
+ a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
1485
+ b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
1486
+ n = 0;
1487
+ }
1488
+ else {
1489
+ a_f16_vec = _mm512_loadu_ph(a);
1490
+ b_f16_vec = _mm512_loadu_ph(b);
1491
+ a += 32, b += 32, n -= 32;
1492
+ }
1493
+ sum_f16_vec = _mm512_add_ph(a_f16_vec, b_f16_vec);
1494
+ _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
1495
+ result += 32;
1496
+ if (n) goto simsimd_sum_f16_sapphire_cycle;
1497
+ }
1498
+
1499
+ SIMSIMD_PUBLIC void simsimd_scale_f16_sapphire(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1500
+ simsimd_f16_t *result) {
901
1501
 
902
- #if SIMSIMD_TARGET_SAPPHIRE
903
- #pragma GCC push_options
904
- #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
905
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
906
- apply_to = function)
1502
+ __mmask32 mask = 0xFFFFFFFF;
1503
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1504
+ __m512h a_f16_vec, b_f16_vec;
1505
+ __m512h sum_f16_vec;
1506
+ simsimd_scale_f16_sapphire_cycle:
1507
+ if (n < 32) {
1508
+ mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1509
+ a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
1510
+ n = 0;
1511
+ }
1512
+ else {
1513
+ a_f16_vec = _mm512_loadu_ph(a);
1514
+ a += 32, n -= 32;
1515
+ }
1516
+ sum_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
1517
+ _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
1518
+ result += 32;
1519
+ if (n) goto simsimd_scale_f16_sapphire_cycle;
1520
+ }
907
1521
 
908
1522
  SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
909
1523
  simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
910
1524
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
911
1525
 
1526
+ // There are are several special cases we may want to implement:
1527
+ // 1. Simple addition, when both weights are equal to 1.0.
1528
+ if (alpha == 1 && beta == 1) {
1529
+ // In this case we can avoid expensive multiplications.
1530
+ simsimd_sum_f16_sapphire(a, b, n, result);
1531
+ return;
1532
+ }
1533
+ // 2. Just scaling, when one of the weights is equal to zero.
1534
+ else if (alpha == 0 || beta == 0) {
1535
+ // In this case we can avoid half of the load instructions.
1536
+ if (beta == 0) { simsimd_scale_f16_sapphire(a, n, alpha, result); }
1537
+ else { simsimd_scale_f16_sapphire(b, n, beta, result); }
1538
+ return;
1539
+ }
1540
+
1541
+ // The general case.
912
1542
  __mmask32 mask = 0xFFFFFFFF;
913
1543
  __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
914
1544
  __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
915
- __m512h a_f16_vec, b_f16_vec, c_f16_vec;
1545
+ __m512h a_f16_vec, b_f16_vec;
916
1546
  __m512h a_scaled_f16_vec, sum_f16_vec;
917
-
918
1547
  simsimd_wsum_f16_sapphire_cycle:
919
1548
  if (n < 32) {
920
1549
  mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
@@ -943,7 +1572,6 @@ SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire(
943
1572
  __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
944
1573
  __m512h a_f16_vec, b_f16_vec, c_f16_vec;
945
1574
  __m512h ab_f16_vec, ab_scaled_f16_vec, sum_f16_vec;
946
-
947
1575
  simsimd_fma_f16_sapphire_cycle:
948
1576
  if (n < 32) {
949
1577
  mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
@@ -966,18 +1594,88 @@ simsimd_fma_f16_sapphire_cycle:
966
1594
  if (n) goto simsimd_fma_f16_sapphire_cycle;
967
1595
  }
968
1596
 
1597
+ SIMSIMD_PUBLIC void simsimd_sum_u8_sapphire(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1598
+ simsimd_u8_t *result) {
1599
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1600
+ __m512i a_u8_vec, b_u8_vec, sum_u8_vec;
1601
+ simsimd_sum_u8_sapphire_cycle:
1602
+ if (n < 64) {
1603
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1604
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1605
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1606
+ n = 0;
1607
+ }
1608
+ else {
1609
+ a_u8_vec = _mm512_loadu_epi8(a);
1610
+ b_u8_vec = _mm512_loadu_epi8(b);
1611
+ a += 64, b += 64, n -= 64;
1612
+ }
1613
+ sum_u8_vec = _mm512_adds_epu8(a_u8_vec, b_u8_vec);
1614
+ _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1615
+ result += 64;
1616
+ if (n) goto simsimd_sum_u8_sapphire_cycle;
1617
+ }
1618
+
1619
+ SIMSIMD_PUBLIC void simsimd_scale_u8_sapphire(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1620
+ simsimd_u8_t *result) {
1621
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1622
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1623
+ __m512i a_u8_vec, b_u8_vec, sum_u8_vec;
1624
+ __m512h a_f16_low_vec, a_f16_high_vec;
1625
+ __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1626
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
1627
+ simsimd_scale_u8_sapphire_cycle:
1628
+ if (n < 64) {
1629
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1630
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1631
+ n = 0;
1632
+ }
1633
+ else {
1634
+ a_u8_vec = _mm512_loadu_epi8(a);
1635
+ a += 64, n -= 64;
1636
+ }
1637
+ // Upcast:
1638
+ a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
1639
+ a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
1640
+ // Scale:
1641
+ sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1642
+ sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1643
+ // Downcast:
1644
+ sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1645
+ sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1646
+ sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
1647
+ _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1648
+ result += 64;
1649
+ if (n) goto simsimd_scale_u8_sapphire_cycle;
1650
+ }
1651
+
969
1652
  SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
970
1653
  simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
971
1654
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
972
1655
 
1656
+ // There are are several special cases we may want to implement:
1657
+ // 1. Simple addition, when both weights are equal to 1.0.
1658
+ if (alpha == 1 && beta == 1) {
1659
+ // In this case we can avoid expensive multiplications.
1660
+ simsimd_sum_u8_sapphire(a, b, n, result);
1661
+ return;
1662
+ }
1663
+ // 2. Just scaling, when one of the weights is equal to zero.
1664
+ else if (alpha == 0 || beta == 0) {
1665
+ // In this case we can avoid half of the load instructions.
1666
+ if (beta == 0) { simsimd_scale_u8_sapphire(a, n, alpha, result); }
1667
+ else { simsimd_scale_u8_sapphire(b, n, beta, result); }
1668
+ return;
1669
+ }
1670
+
1671
+ // The general case.
973
1672
  __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
974
1673
  __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
975
1674
  __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
976
- __m512i a_u8_vec, b_u8_vec, c_u8_vec, sum_u8_vec;
1675
+ __m512i a_u8_vec, b_u8_vec, sum_u8_vec;
977
1676
  __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
978
1677
  __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
979
1678
  __m512i sum_i16_low_vec, sum_i16_high_vec;
980
-
981
1679
  simsimd_wsum_u8_sapphire_cycle:
982
1680
  if (n < 64) {
983
1681
  mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
@@ -1010,14 +1708,92 @@ simsimd_wsum_u8_sapphire_cycle:
1010
1708
  if (n) goto simsimd_wsum_u8_sapphire_cycle;
1011
1709
  }
1012
1710
 
1711
+ SIMSIMD_PUBLIC void simsimd_sum_i8_sapphire(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1712
+ simsimd_i8_t *result) {
1713
+
1714
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1715
+ __m512i a_i8_vec, b_i8_vec, sum_i8_vec;
1716
+ __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1717
+ __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1718
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
1719
+
1720
+ simsimd_sum_i8_sapphire_cycle:
1721
+ if (n < 64) {
1722
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1723
+ a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1724
+ b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
1725
+ n = 0;
1726
+ }
1727
+ else {
1728
+ a_i8_vec = _mm512_loadu_epi8(a);
1729
+ b_i8_vec = _mm512_loadu_epi8(b);
1730
+ a += 64, b += 64, n -= 64;
1731
+ }
1732
+ sum_i8_vec = _mm512_adds_epi8(a_i8_vec, b_i8_vec);
1733
+ _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1734
+ result += 64;
1735
+ if (n) goto simsimd_sum_i8_sapphire_cycle;
1736
+ }
1737
+
1738
+ SIMSIMD_PUBLIC void simsimd_scale_i8_sapphire(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1739
+ simsimd_i8_t *result) {
1740
+
1741
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1742
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1743
+ __m512i a_i8_vec, sum_i8_vec;
1744
+ __m512h a_f16_low_vec, a_f16_high_vec;
1745
+ __m512h sum_f16_low_vec, sum_f16_high_vec;
1746
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
1747
+ simsimd_wsum_i8_sapphire_cycle:
1748
+ if (n < 64) {
1749
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1750
+ a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1751
+ n = 0;
1752
+ }
1753
+ else {
1754
+ a_i8_vec = _mm512_loadu_epi8(a);
1755
+ a += 64, n -= 64;
1756
+ }
1757
+ // Upcast:
1758
+ a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
1759
+ a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
1760
+ // Scale:
1761
+ sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1762
+ sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1763
+ // Downcast:
1764
+ sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1765
+ sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1766
+ sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
1767
+ _mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
1768
+ _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1769
+ result += 64;
1770
+ if (n) goto simsimd_wsum_i8_sapphire_cycle;
1771
+ }
1772
+
1013
1773
  SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
1014
1774
  simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
1015
1775
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1016
1776
 
1777
+ // There are are several special cases we may want to implement:
1778
+ // 1. Simple addition, when both weights are equal to 1.0.
1779
+ if (alpha == 1 && beta == 1) {
1780
+ // In this case we can avoid expensive multiplications.
1781
+ simsimd_sum_i8_sapphire(a, b, n, result);
1782
+ return;
1783
+ }
1784
+ // 2. Just scaling, when one of the weights is equal to zero.
1785
+ else if (alpha == 0 || beta == 0) {
1786
+ // In this case we can avoid half of the load instructions.
1787
+ if (beta == 0) { simsimd_scale_i8_sapphire(a, n, alpha, result); }
1788
+ else { simsimd_scale_i8_sapphire(b, n, beta, result); }
1789
+ return;
1790
+ }
1791
+
1792
+ // The general case.
1017
1793
  __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1018
1794
  __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1019
1795
  __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
1020
- __m512i a_i8_vec, b_i8_vec, c_i8_vec, sum_i8_vec;
1796
+ __m512i a_i8_vec, b_i8_vec, sum_i8_vec;
1021
1797
  __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1022
1798
  __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1023
1799
  __m512i sum_i16_low_vec, sum_i16_high_vec;
@@ -1171,9 +1947,57 @@ simsimd_fma_u8_sapphire_cycle:
1171
1947
  #pragma GCC target("arch=armv8.2-a+simd")
1172
1948
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
1173
1949
 
1950
+ SIMSIMD_PUBLIC void simsimd_sum_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1951
+ simsimd_f32_t *result) {
1952
+ // The main loop:
1953
+ simsimd_size_t i = 0;
1954
+ for (; i + 4 <= n; i += 4) {
1955
+ float32x4_t a_vec = vld1q_f32(a + i);
1956
+ float32x4_t b_vec = vld1q_f32(b + i);
1957
+ float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
1958
+ vst1q_f32(result + i, sum_vec);
1959
+ }
1960
+
1961
+ // The tail:
1962
+ for (; i < n; ++i) result[i] = a[i] + b[i];
1963
+ }
1964
+
1965
+ SIMSIMD_PUBLIC void simsimd_scale_f32_neon(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
1966
+ simsimd_f32_t *result) {
1967
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1968
+
1969
+ // The main loop:
1970
+ simsimd_size_t i = 0;
1971
+ for (; i + 4 <= n; i += 4) {
1972
+ float32x4_t a_vec = vld1q_f32(a + i);
1973
+ float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
1974
+ vst1q_f32(result + i, sum_vec);
1975
+ }
1976
+
1977
+ // The tail:
1978
+ for (; i < n; ++i) result[i] = alpha_f32 * a[i];
1979
+ }
1980
+
1174
1981
  SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
1175
1982
  simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
1176
1983
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1984
+
1985
+ // There are are several special cases we may want to implement:
1986
+ // 1. Simple addition, when both weights are equal to 1.0.
1987
+ if (alpha == 1 && beta == 1) {
1988
+ // In this case we can avoid expensive multiplications.
1989
+ simsimd_sum_f32_neon(a, b, n, result);
1990
+ return;
1991
+ }
1992
+ // 2. Just scaling, when one of the weights is equal to zero.
1993
+ else if (alpha == 0 || beta == 0) {
1994
+ // In this case we can avoid half of the load instructions.
1995
+ if (beta == 0) { simsimd_scale_f32_neon(a, n, alpha, result); }
1996
+ else { simsimd_scale_f32_neon(b, n, beta, result); }
1997
+ return;
1998
+ }
1999
+
2000
+ // The general case.
1177
2001
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1178
2002
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1179
2003
 
@@ -1223,9 +2047,57 @@ SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
1223
2047
  #pragma GCC target("arch=armv8.6-a+simd+bf16")
1224
2048
  #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
1225
2049
 
2050
+ SIMSIMD_PUBLIC void simsimd_sum_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
2051
+ simsimd_bf16_t *result) {
2052
+ // The main loop:
2053
+ simsimd_size_t i = 0;
2054
+ for (; i + 4 <= n; i += 4) {
2055
+ float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
2056
+ float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
2057
+ float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
2058
+ vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
2059
+ }
2060
+
2061
+ // The tail:
2062
+ for (; i < n; ++i) simsimd_f32_to_bf16(simsimd_bf16_to_f32(a + i) + simsimd_bf16_to_f32(b + i), result + i);
2063
+ }
2064
+
2065
+ SIMSIMD_PUBLIC void simsimd_scale_bf16_neon(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2066
+ simsimd_bf16_t *result) {
2067
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
2068
+
2069
+ // The main loop:
2070
+ simsimd_size_t i = 0;
2071
+ for (; i + 4 <= n; i += 4) {
2072
+ float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
2073
+ float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
2074
+ vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
2075
+ }
2076
+
2077
+ // The tail:
2078
+ for (; i < n; ++i) simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i), result + i);
2079
+ }
2080
+
1226
2081
  SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
1227
2082
  simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
1228
2083
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
2084
+
2085
+ // There are are several special cases we may want to implement:
2086
+ // 1. Simple addition, when both weights are equal to 1.0.
2087
+ if (alpha == 1 && beta == 1) {
2088
+ // In this case we can avoid expensive multiplications.
2089
+ simsimd_sum_bf16_neon(a, b, n, result);
2090
+ return;
2091
+ }
2092
+ // 2. Just scaling, when one of the weights is equal to zero.
2093
+ else if (alpha == 0 || beta == 0) {
2094
+ // In this case we can avoid half of the load instructions.
2095
+ if (beta == 0) { simsimd_scale_bf16_neon(a, n, alpha, result); }
2096
+ else { simsimd_scale_bf16_neon(b, n, beta, result); }
2097
+ return;
2098
+ }
2099
+
2100
+ // The general case.
1229
2101
  simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1230
2102
  simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1231
2103
 
@@ -1279,9 +2151,56 @@ SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( /
1279
2151
  #pragma GCC target("arch=armv8.2-a+simd+fp16")
1280
2152
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
1281
2153
 
2154
+ SIMSIMD_PUBLIC void simsimd_sum_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
2155
+ simsimd_f16_t *result) {
2156
+ // The main loop:
2157
+ simsimd_size_t i = 0;
2158
+ for (; i + 8 <= n; i += 8) {
2159
+ float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
2160
+ float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
2161
+ float16x8_t sum_vec = vaddq_f16(a_vec, b_vec);
2162
+ vst1q_f16((float16_t *)result + i, sum_vec);
2163
+ }
2164
+
2165
+ // The tail:
2166
+ for (; i < n; ++i) ((float16_t *)result)[i] = ((float16_t const *)a)[i] + ((float16_t const *)b)[i];
2167
+ }
2168
+
2169
+ SIMSIMD_PUBLIC void simsimd_scale_f16_neon(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2170
+ simsimd_f16_t *result) {
2171
+ float16_t alpha_f16 = (float16_t)alpha;
2172
+
2173
+ // The main loop:
2174
+ simsimd_size_t i = 0;
2175
+ for (; i + 8 <= n; i += 8) {
2176
+ float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
2177
+ float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
2178
+ vst1q_f16((float16_t *)result + i, sum_vec);
2179
+ }
2180
+
2181
+ // The tail:
2182
+ for (; i < n; ++i) ((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i];
2183
+ }
2184
+
1282
2185
  SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
1283
2186
  simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
1284
2187
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
2188
+ // There are are several special cases we may want to implement:
2189
+ // 1. Simple addition, when both weights are equal to 1.0.
2190
+ if (alpha == 1 && beta == 1) {
2191
+ // In this case we can avoid expensive multiplications.
2192
+ simsimd_sum_f16_neon(a, b, n, result);
2193
+ return;
2194
+ }
2195
+ // 2. Just scaling, when one of the weights is equal to zero.
2196
+ else if (alpha == 0 || beta == 0) {
2197
+ // In this case we can avoid half of the load instructions.
2198
+ if (beta == 0) { simsimd_scale_f16_neon(a, n, alpha, result); }
2199
+ else { simsimd_scale_f16_neon(b, n, beta, result); }
2200
+ return;
2201
+ }
2202
+
2203
+ // The general case.
1285
2204
  float16_t alpha_f16 = (float16_t)alpha;
1286
2205
  float16_t beta_f16 = (float16_t)beta;
1287
2206
 
@@ -1325,9 +2244,59 @@ SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
1325
2244
  alpha_f16 * ((float16_t const *)a)[i] * ((float16_t const *)b)[i] + beta_f16 * ((float16_t const *)c)[i];
1326
2245
  }
1327
2246
 
2247
+ SIMSIMD_PUBLIC void simsimd_sum_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
2248
+ simsimd_u8_t *result) {
2249
+ // The main loop:
2250
+ simsimd_size_t i = 0;
2251
+ for (; i + 16 <= n; i += 16) {
2252
+ uint8x16_t a_vec = vld1q_u8(a + i);
2253
+ uint8x16_t b_vec = vld1q_u8(b + i);
2254
+ uint8x16_t sum_vec = vqaddq_u8(a_vec, b_vec);
2255
+ vst1q_u8(result + i, sum_vec);
2256
+ }
2257
+
2258
+ // The tail:
2259
+ for (; i < n; ++i) { SIMSIMD_F32_TO_U8(a[i] + b[i], result + i); }
2260
+ }
2261
+
2262
+ SIMSIMD_PUBLIC void simsimd_scale_u8_neon(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2263
+ simsimd_u8_t *result) {
2264
+ float16_t alpha_f16 = (float16_t)alpha;
2265
+
2266
+ // The main loop:
2267
+ simsimd_size_t i = 0;
2268
+ for (; i + 8 <= n; i += 8) {
2269
+ uint8x8_t a_u8_vec = vld1_u8(a + i);
2270
+ float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
2271
+ float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
2272
+ uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
2273
+ vst1_u8(result + i, sum_u8_vec);
2274
+ }
2275
+
2276
+ // The tail:
2277
+ for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i], result + i); }
2278
+ }
2279
+
1328
2280
  SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
1329
2281
  simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
1330
2282
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
2283
+
2284
+ // There are are several special cases we may want to implement:
2285
+ // 1. Simple addition, when both weights are equal to 1.0.
2286
+ if (alpha == 1 && beta == 1) {
2287
+ // In this case we can avoid expensive multiplications.
2288
+ simsimd_sum_u8_neon(a, b, n, result);
2289
+ return;
2290
+ }
2291
+ // 2. Just scaling, when one of the weights is equal to zero.
2292
+ else if (alpha == 0 || beta == 0) {
2293
+ // In this case we can avoid half of the load instructions.
2294
+ if (beta == 0) { simsimd_scale_u8_neon(a, n, alpha, result); }
2295
+ else { simsimd_scale_u8_neon(b, n, beta, result); }
2296
+ return;
2297
+ }
2298
+
2299
+ // The general case.
1331
2300
  float16_t alpha_f16 = (float16_t)alpha;
1332
2301
  float16_t beta_f16 = (float16_t)beta;
1333
2302
 
@@ -1375,9 +2344,59 @@ SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
1375
2344
  for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
1376
2345
  }
1377
2346
 
2347
+ SIMSIMD_PUBLIC void simsimd_sum_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
2348
+ simsimd_i8_t *result) {
2349
+ // The main loop:
2350
+ simsimd_size_t i = 0;
2351
+ for (; i + 16 <= n; i += 16) {
2352
+ int8x16_t a_vec = vld1q_s8(a + i);
2353
+ int8x16_t b_vec = vld1q_s8(b + i);
2354
+ int8x16_t sum_vec = vqaddq_s8(a_vec, b_vec);
2355
+ vst1q_s8(result + i, sum_vec);
2356
+ }
2357
+
2358
+ // The tail:
2359
+ for (; i < n; ++i) { SIMSIMD_F32_TO_I8(a[i] + b[i], result + i); }
2360
+ }
2361
+
2362
+ SIMSIMD_PUBLIC void simsimd_scale_i8_neon(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
2363
+ simsimd_i8_t *result) {
2364
+ float16_t alpha_f16 = (float16_t)alpha;
2365
+
2366
+ // The main loop:
2367
+ simsimd_size_t i = 0;
2368
+ for (; i + 8 <= n; i += 8) {
2369
+ int8x8_t a_i8_vec = vld1_s8(a + i);
2370
+ float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
2371
+ float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
2372
+ int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
2373
+ vst1_s8(result + i, sum_i8_vec);
2374
+ }
2375
+
2376
+ // The tail:
2377
+ for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i], result + i); }
2378
+ }
2379
+
1378
2380
  SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
1379
2381
  simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
1380
2382
  simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
2383
+
2384
+ // There are are several special cases we may want to implement:
2385
+ // 1. Simple addition, when both weights are equal to 1.0.
2386
+ if (alpha == 1 && beta == 1) {
2387
+ // In this case we can avoid expensive multiplications.
2388
+ simsimd_sum_i8_neon(a, b, n, result);
2389
+ return;
2390
+ }
2391
+ // 2. Just scaling, when one of the weights is equal to zero.
2392
+ else if (alpha == 0 || beta == 0) {
2393
+ // In this case we can avoid half of the load instructions.
2394
+ if (beta == 0) { simsimd_scale_i8_neon(a, n, alpha, result); }
2395
+ else { simsimd_scale_i8_neon(b, n, beta, result); }
2396
+ return;
2397
+ }
2398
+
2399
+ // The general case.
1381
2400
  float16_t alpha_f16 = (float16_t)alpha;
1382
2401
  float16_t beta_f16 = (float16_t)beta;
1383
2402