usearch 2.16.3 → 2.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/include/usearch/index.hpp +1 -1
- package/include/usearch/index_dense.hpp +1 -0
- package/include/usearch/index_plugins.hpp +5 -4
- package/javascript/dist/cjs/usearch.d.ts +2 -2
- package/javascript/dist/esm/usearch.d.ts +2 -2
- package/javascript/usearch.ts +2 -2
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/CONTRIBUTING.md +31 -2
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +1 -1
- package/simsimd/README.md +93 -39
- package/simsimd/c/lib.c +32 -25
- package/simsimd/include/simsimd/elementwise.h +1068 -49
- package/simsimd/include/simsimd/simsimd.h +78 -73
- package/simsimd/include/simsimd/types.h +34 -17
- package/simsimd/javascript/fallback.ts +27 -4
- package/simsimd/javascript/lib.c +49 -29
- package/simsimd/javascript/simsimd.ts +34 -14
- package/simsimd/package-lock.json +13 -11
- package/simsimd/package.json +3 -3
- package/simsimd/scripts/bench.cxx +2 -5
- package/simsimd/scripts/bench_vectors.py +44 -10
- package/simsimd/scripts/test.c +39 -35
- package/simsimd/scripts/test.mjs +28 -0
- package/simsimd/scripts/test.py +19 -19
- package/simsimd/swift/SimSIMD.swift +12 -7
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* @date October 16, 2024
|
|
6
6
|
*
|
|
7
7
|
* Contains following element-wise operations:
|
|
8
|
+
* - Sum (Add): R[i] = A[i] + B[i]
|
|
9
|
+
* - Scale (Multiply): R[i] = Alpha * A[i]
|
|
8
10
|
* - WSum or Weighted-Sum: R[i] = Alpha * A[i] + Beta * B[i]
|
|
9
11
|
* - FMA or Fused-Multiply-Add: R[i] = Alpha * A[i] * B[i] + Beta * C[i]
|
|
10
12
|
*
|
|
@@ -23,8 +25,8 @@
|
|
|
23
25
|
* - 8-bit signed integers
|
|
24
26
|
*
|
|
25
27
|
* For hardware architectures:
|
|
26
|
-
* - Arm: NEON
|
|
27
|
-
* - x86: Haswell,
|
|
28
|
+
* - Arm: NEON
|
|
29
|
+
* - x86: Haswell, Skylake, Sapphire
|
|
28
30
|
*
|
|
29
31
|
* We use `f16` for `i8` and `u8` arithmetic. This is because Arm received `f16` support earlier than `bf16`.
|
|
30
32
|
* For example, Apple M1 has `f16` support and `bf16` was only added in M2. On the other hand, on paper,
|
|
@@ -34,8 +36,8 @@
|
|
|
34
36
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
35
37
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
36
38
|
*/
|
|
37
|
-
#ifndef
|
|
38
|
-
#define
|
|
39
|
+
#ifndef SIMSIMD_ELEMENTWISE_H
|
|
40
|
+
#define SIMSIMD_ELEMENTWISE_H
|
|
39
41
|
|
|
40
42
|
#include "types.h"
|
|
41
43
|
|
|
@@ -270,11 +272,60 @@ SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire(
|
|
|
270
272
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
271
273
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
272
274
|
|
|
275
|
+
SIMSIMD_PUBLIC void simsimd_sum_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
276
|
+
simsimd_f32_t *result) {
|
|
277
|
+
// The main loop:
|
|
278
|
+
simsimd_size_t i = 0;
|
|
279
|
+
for (; i + 8 <= n; i += 8) {
|
|
280
|
+
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
281
|
+
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
282
|
+
__m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
|
|
283
|
+
_mm256_storeu_ps(result + i, sum_vec);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// The tail:
|
|
287
|
+
for (; i < n; ++i) result[i] = a[i] + b[i];
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
SIMSIMD_PUBLIC void simsimd_scale_f32_haswell(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
291
|
+
simsimd_f32_t *result) {
|
|
292
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
293
|
+
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
294
|
+
|
|
295
|
+
// The main loop:
|
|
296
|
+
simsimd_size_t i = 0;
|
|
297
|
+
for (; i + 8 <= n; i += 8) {
|
|
298
|
+
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
299
|
+
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
300
|
+
_mm256_storeu_ps(result + i, sum_vec);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// The tail:
|
|
304
|
+
for (; i < n; ++i) result[i] = alpha_f32 * a[i];
|
|
305
|
+
}
|
|
306
|
+
|
|
273
307
|
SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
|
|
274
308
|
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
275
309
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
276
310
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
277
311
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
312
|
+
|
|
313
|
+
// There are are several special cases we may want to implement:
|
|
314
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
315
|
+
if (alpha == 1 && beta == 1) {
|
|
316
|
+
// In this case we can avoid expensive multiplications.
|
|
317
|
+
simsimd_sum_f32_haswell(a, b, n, result);
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
321
|
+
else if (alpha == 0 || beta == 0) {
|
|
322
|
+
// In this case we can avoid half of the load instructions.
|
|
323
|
+
if (beta == 0) { simsimd_scale_f32_haswell(a, n, alpha, result); }
|
|
324
|
+
else { simsimd_scale_f32_haswell(b, n, beta, result); }
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// The general case.
|
|
278
329
|
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
279
330
|
__m256 beta_vec = _mm256_set1_ps(beta_f32);
|
|
280
331
|
|
|
@@ -283,9 +334,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
|
|
|
283
334
|
for (; i + 8 <= n; i += 8) {
|
|
284
335
|
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
285
336
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
286
|
-
__m256
|
|
287
|
-
__m256
|
|
288
|
-
__m256 sum_vec = _mm256_add_ps(
|
|
337
|
+
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
338
|
+
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
339
|
+
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
289
340
|
_mm256_storeu_ps(result + i, sum_vec);
|
|
290
341
|
}
|
|
291
342
|
|
|
@@ -293,9 +344,57 @@ SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
|
|
|
293
344
|
for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
|
|
294
345
|
}
|
|
295
346
|
|
|
347
|
+
SIMSIMD_PUBLIC void simsimd_sum_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
348
|
+
simsimd_f64_t *result) {
|
|
349
|
+
// The main loop:
|
|
350
|
+
simsimd_size_t i = 0;
|
|
351
|
+
for (; i + 4 <= n; i += 4) {
|
|
352
|
+
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
353
|
+
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
354
|
+
__m256d sum_vec = _mm256_add_pd(a_vec, b_vec);
|
|
355
|
+
_mm256_storeu_pd(result + i, sum_vec);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// The tail:
|
|
359
|
+
for (; i < n; ++i) result[i] = a[i] + b[i];
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
SIMSIMD_PUBLIC void simsimd_scale_f64_haswell(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
363
|
+
simsimd_f64_t *result) {
|
|
364
|
+
__m256d alpha_vec = _mm256_set1_pd(alpha);
|
|
365
|
+
|
|
366
|
+
// The main loop:
|
|
367
|
+
simsimd_size_t i = 0;
|
|
368
|
+
for (; i + 4 <= n; i += 4) {
|
|
369
|
+
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
370
|
+
__m256d sum_vec = _mm256_mul_pd(a_vec, alpha_vec);
|
|
371
|
+
_mm256_storeu_pd(result + i, sum_vec);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// The tail:
|
|
375
|
+
for (; i < n; ++i) result[i] = alpha * a[i];
|
|
376
|
+
}
|
|
377
|
+
|
|
296
378
|
SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
|
|
297
379
|
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
298
380
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
|
|
381
|
+
|
|
382
|
+
// There are are several special cases we may want to implement:
|
|
383
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
384
|
+
if (alpha == 1 && beta == 1) {
|
|
385
|
+
// In this case we can avoid expensive multiplications.
|
|
386
|
+
simsimd_sum_f64_haswell(a, b, n, result);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
390
|
+
else if (alpha == 0 || beta == 0) {
|
|
391
|
+
// In this case we can avoid half of the load instructions.
|
|
392
|
+
if (beta == 0) { simsimd_scale_f64_haswell(a, n, alpha, result); }
|
|
393
|
+
else { simsimd_scale_f64_haswell(b, n, beta, result); }
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// The general case.
|
|
299
398
|
__m256d alpha_vec = _mm256_set1_pd(alpha);
|
|
300
399
|
__m256d beta_vec = _mm256_set1_pd(beta);
|
|
301
400
|
|
|
@@ -304,9 +403,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
|
|
|
304
403
|
for (; i + 4 <= n; i += 4) {
|
|
305
404
|
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
306
405
|
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
307
|
-
__m256d
|
|
308
|
-
__m256d
|
|
309
|
-
__m256d sum_vec = _mm256_add_pd(
|
|
406
|
+
__m256d a_scaled_vec = _mm256_mul_pd(a_vec, alpha_vec);
|
|
407
|
+
__m256d b_scaled_vec = _mm256_mul_pd(b_vec, beta_vec);
|
|
408
|
+
__m256d sum_vec = _mm256_add_pd(a_scaled_vec, b_scaled_vec);
|
|
310
409
|
_mm256_storeu_pd(result + i, sum_vec);
|
|
311
410
|
}
|
|
312
411
|
|
|
@@ -314,9 +413,73 @@ SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
|
|
|
314
413
|
for (; i < n; ++i) result[i] = alpha * a[i] + beta * b[i];
|
|
315
414
|
}
|
|
316
415
|
|
|
416
|
+
SIMSIMD_PUBLIC void simsimd_sum_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
417
|
+
simsimd_f16_t *result) {
|
|
418
|
+
|
|
419
|
+
// The main loop:
|
|
420
|
+
simsimd_size_t i = 0;
|
|
421
|
+
for (; i + 8 <= n; i += 8) {
|
|
422
|
+
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
423
|
+
__m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
424
|
+
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
425
|
+
__m256 b_vec = _mm256_cvtph_ps(b_f16);
|
|
426
|
+
__m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
|
|
427
|
+
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
428
|
+
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// The tail:
|
|
432
|
+
for (; i < n; ++i) {
|
|
433
|
+
simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
|
|
434
|
+
simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
|
|
435
|
+
simsimd_f32_t sum = ai + bi;
|
|
436
|
+
SIMSIMD_F32_TO_F16(sum, result + i);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
SIMSIMD_PUBLIC void simsimd_scale_f16_haswell(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
441
|
+
simsimd_f16_t *result) {
|
|
442
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
443
|
+
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
444
|
+
|
|
445
|
+
// The main loop:
|
|
446
|
+
simsimd_size_t i = 0;
|
|
447
|
+
for (; i + 8 <= n; i += 8) {
|
|
448
|
+
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
449
|
+
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
450
|
+
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
451
|
+
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
452
|
+
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// The tail:
|
|
456
|
+
for (; i < n; ++i) {
|
|
457
|
+
simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
|
|
458
|
+
simsimd_f32_t sum = alpha_f32 * ai;
|
|
459
|
+
SIMSIMD_F32_TO_F16(sum, result + i);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
317
463
|
SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
|
|
318
464
|
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
319
465
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
466
|
+
|
|
467
|
+
// There are are several special cases we may want to implement:
|
|
468
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
469
|
+
if (alpha == 1 && beta == 1) {
|
|
470
|
+
// In this case we can avoid expensive multiplications.
|
|
471
|
+
simsimd_sum_f16_haswell(a, b, n, result);
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
475
|
+
else if (alpha == 0 || beta == 0) {
|
|
476
|
+
// In this case we can avoid half of the load instructions.
|
|
477
|
+
if (beta == 0) { simsimd_scale_f16_haswell(a, n, alpha, result); }
|
|
478
|
+
else { simsimd_scale_f16_haswell(b, n, beta, result); }
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// The general case.
|
|
320
483
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
321
484
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
322
485
|
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
@@ -325,13 +488,13 @@ SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
|
|
|
325
488
|
// The main loop:
|
|
326
489
|
simsimd_size_t i = 0;
|
|
327
490
|
for (; i + 8 <= n; i += 8) {
|
|
328
|
-
__m128i a_f16 =
|
|
329
|
-
__m128i b_f16 =
|
|
491
|
+
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
492
|
+
__m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
330
493
|
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
331
494
|
__m256 b_vec = _mm256_cvtph_ps(b_f16);
|
|
332
|
-
__m256
|
|
333
|
-
__m256
|
|
334
|
-
__m256 sum_vec = _mm256_add_ps(
|
|
495
|
+
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
496
|
+
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
497
|
+
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
335
498
|
__m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
|
|
336
499
|
_mm_storeu_si128((__m128i *)(result + i), sum_f16);
|
|
337
500
|
}
|
|
@@ -345,9 +508,72 @@ SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
|
|
|
345
508
|
}
|
|
346
509
|
}
|
|
347
510
|
|
|
511
|
+
SIMSIMD_PUBLIC void simsimd_sum_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
512
|
+
simsimd_bf16_t *result) {
|
|
513
|
+
// The main loop:
|
|
514
|
+
simsimd_size_t i = 0;
|
|
515
|
+
for (; i + 8 <= n; i += 8) {
|
|
516
|
+
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
517
|
+
__m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
518
|
+
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
519
|
+
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
|
|
520
|
+
__m256 sum_vec = _mm256_add_ps(a_vec, b_vec);
|
|
521
|
+
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
522
|
+
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// The tail:
|
|
526
|
+
for (; i < n; ++i) {
|
|
527
|
+
simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
|
|
528
|
+
simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
|
|
529
|
+
simsimd_f32_t sum = ai + bi;
|
|
530
|
+
SIMSIMD_F32_TO_BF16(sum, result + i);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
SIMSIMD_PUBLIC void simsimd_scale_bf16_haswell(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
535
|
+
simsimd_bf16_t *result) {
|
|
536
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
537
|
+
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
538
|
+
|
|
539
|
+
// The main loop:
|
|
540
|
+
simsimd_size_t i = 0;
|
|
541
|
+
for (; i + 8 <= n; i += 8) {
|
|
542
|
+
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
543
|
+
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
544
|
+
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
545
|
+
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
546
|
+
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// The tail:
|
|
550
|
+
for (; i < n; ++i) {
|
|
551
|
+
simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
|
|
552
|
+
simsimd_f32_t sum = alpha_f32 * ai;
|
|
553
|
+
SIMSIMD_F32_TO_BF16(sum, result + i);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
348
557
|
SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
|
|
349
558
|
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
350
559
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
560
|
+
|
|
561
|
+
// There are are several special cases we may want to implement:
|
|
562
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
563
|
+
if (alpha == 1 && beta == 1) {
|
|
564
|
+
// In this case we can avoid expensive multiplications.
|
|
565
|
+
simsimd_sum_bf16_haswell(a, b, n, result);
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
569
|
+
else if (alpha == 0 || beta == 0) {
|
|
570
|
+
// In this case we can avoid half of the load instructions.
|
|
571
|
+
if (beta == 0) { simsimd_scale_bf16_haswell(a, n, alpha, result); }
|
|
572
|
+
else { simsimd_scale_bf16_haswell(b, n, beta, result); }
|
|
573
|
+
return;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
// The general case.
|
|
351
577
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
352
578
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
353
579
|
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
@@ -356,13 +582,13 @@ SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
|
|
|
356
582
|
// The main loop:
|
|
357
583
|
simsimd_size_t i = 0;
|
|
358
584
|
for (; i + 8 <= n; i += 8) {
|
|
359
|
-
__m128i a_bf16 =
|
|
360
|
-
__m128i b_bf16 =
|
|
585
|
+
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
586
|
+
__m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
361
587
|
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
362
588
|
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
|
|
363
|
-
__m256
|
|
364
|
-
__m256
|
|
365
|
-
__m256 sum_vec = _mm256_add_ps(
|
|
589
|
+
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
590
|
+
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
591
|
+
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
366
592
|
__m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
|
|
367
593
|
_mm_storeu_si128((__m128i *)(result + i), sum_bf16);
|
|
368
594
|
}
|
|
@@ -435,9 +661,9 @@ SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
|
|
|
435
661
|
// The main loop:
|
|
436
662
|
simsimd_size_t i = 0;
|
|
437
663
|
for (; i + 8 <= n; i += 8) {
|
|
438
|
-
__m128i a_f16 =
|
|
439
|
-
__m128i b_f16 =
|
|
440
|
-
__m128i c_f16 =
|
|
664
|
+
__m128i a_f16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
665
|
+
__m128i b_f16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
666
|
+
__m128i c_f16 = _mm_lddqu_si128((__m128i const *)(c + i));
|
|
441
667
|
__m256 a_vec = _mm256_cvtph_ps(a_f16);
|
|
442
668
|
__m256 b_vec = _mm256_cvtph_ps(b_f16);
|
|
443
669
|
__m256 c_vec = _mm256_cvtph_ps(c_f16);
|
|
@@ -470,9 +696,9 @@ SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( /
|
|
|
470
696
|
// The main loop:
|
|
471
697
|
simsimd_size_t i = 0;
|
|
472
698
|
for (; i + 8 <= n; i += 8) {
|
|
473
|
-
__m128i a_bf16 =
|
|
474
|
-
__m128i b_bf16 =
|
|
475
|
-
__m128i c_bf16 =
|
|
699
|
+
__m128i a_bf16 = _mm_lddqu_si128((__m128i const *)(a + i));
|
|
700
|
+
__m128i b_bf16 = _mm_lddqu_si128((__m128i const *)(b + i));
|
|
701
|
+
__m128i c_bf16 = _mm_lddqu_si128((__m128i const *)(c + i));
|
|
476
702
|
__m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
|
|
477
703
|
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
|
|
478
704
|
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(c_bf16);
|
|
@@ -494,10 +720,88 @@ SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( /
|
|
|
494
720
|
}
|
|
495
721
|
}
|
|
496
722
|
|
|
723
|
+
SIMSIMD_PUBLIC void simsimd_sum_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
724
|
+
simsimd_i8_t *result) {
|
|
725
|
+
// The main loop:
|
|
726
|
+
simsimd_size_t i = 0;
|
|
727
|
+
for (; i + 32 <= n; i += 32) {
|
|
728
|
+
__m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
|
|
729
|
+
__m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
|
|
730
|
+
__m256i sum_vec = _mm256_adds_epi8(a_vec, b_vec);
|
|
731
|
+
_mm256_storeu_si256((__m256i *)(result + i), sum_vec);
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// The tail:
|
|
735
|
+
for (; i < n; ++i) {
|
|
736
|
+
simsimd_f32_t ai = a[i], bi = b[i];
|
|
737
|
+
simsimd_f32_t sum = ai + bi;
|
|
738
|
+
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
SIMSIMD_PUBLIC void simsimd_scale_i8_haswell(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
743
|
+
simsimd_i8_t *result) {
|
|
744
|
+
|
|
745
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
746
|
+
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
747
|
+
int sum_i32s[8], a_i32s[8];
|
|
748
|
+
|
|
749
|
+
// The main loop:
|
|
750
|
+
simsimd_size_t i = 0;
|
|
751
|
+
for (; i + 8 <= n; i += 8) {
|
|
752
|
+
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
753
|
+
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
754
|
+
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
755
|
+
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
756
|
+
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
757
|
+
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
758
|
+
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
759
|
+
// The normal part.
|
|
760
|
+
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
761
|
+
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
762
|
+
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
763
|
+
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
|
|
764
|
+
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
|
|
765
|
+
// Export into a serial buffer.
|
|
766
|
+
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
767
|
+
result[i + 0] = (simsimd_i8_t)sum_i32s[0];
|
|
768
|
+
result[i + 1] = (simsimd_i8_t)sum_i32s[1];
|
|
769
|
+
result[i + 2] = (simsimd_i8_t)sum_i32s[2];
|
|
770
|
+
result[i + 3] = (simsimd_i8_t)sum_i32s[3];
|
|
771
|
+
result[i + 4] = (simsimd_i8_t)sum_i32s[4];
|
|
772
|
+
result[i + 5] = (simsimd_i8_t)sum_i32s[5];
|
|
773
|
+
result[i + 6] = (simsimd_i8_t)sum_i32s[6];
|
|
774
|
+
result[i + 7] = (simsimd_i8_t)sum_i32s[7];
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// The tail:
|
|
778
|
+
for (; i < n; ++i) {
|
|
779
|
+
simsimd_f32_t ai = a[i];
|
|
780
|
+
simsimd_f32_t sum = alpha_f32 * ai;
|
|
781
|
+
SIMSIMD_F32_TO_I8(sum, result + i);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
497
785
|
SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
|
|
498
786
|
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
499
787
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
500
788
|
|
|
789
|
+
// There are are several special cases we may want to implement:
|
|
790
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
791
|
+
if (alpha == 1 && beta == 1) {
|
|
792
|
+
// In this case we can avoid expensive multiplications.
|
|
793
|
+
simsimd_sum_i8_haswell(a, b, n, result);
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
797
|
+
else if (alpha == 0 || beta == 0) {
|
|
798
|
+
// In this case we can avoid half of the load instructions.
|
|
799
|
+
if (beta == 0) { simsimd_scale_i8_haswell(a, n, alpha, result); }
|
|
800
|
+
else { simsimd_scale_i8_haswell(b, n, beta, result); }
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
// The general case.
|
|
501
805
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
502
806
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
503
807
|
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
@@ -518,9 +822,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
|
|
|
518
822
|
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
519
823
|
__m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
|
|
520
824
|
// The normal part.
|
|
521
|
-
__m256
|
|
522
|
-
__m256
|
|
523
|
-
__m256 sum_vec = _mm256_add_ps(
|
|
825
|
+
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
826
|
+
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
827
|
+
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
524
828
|
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
525
829
|
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
526
830
|
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
|
|
@@ -545,10 +849,88 @@ SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
|
|
|
545
849
|
}
|
|
546
850
|
}
|
|
547
851
|
|
|
852
|
+
SIMSIMD_PUBLIC void simsimd_sum_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
853
|
+
simsimd_u8_t *result) {
|
|
854
|
+
// The main loop:
|
|
855
|
+
simsimd_size_t i = 0;
|
|
856
|
+
for (; i + 32 <= n; i += 32) {
|
|
857
|
+
__m256i a_vec = _mm256_lddqu_si256((__m256i *)(a + i));
|
|
858
|
+
__m256i b_vec = _mm256_lddqu_si256((__m256i *)(b + i));
|
|
859
|
+
__m256i sum_vec = _mm256_adds_epu8(a_vec, b_vec);
|
|
860
|
+
_mm256_storeu_si256((__m256i *)(result + i), sum_vec);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// The tail:
|
|
864
|
+
for (; i < n; ++i) {
|
|
865
|
+
simsimd_f32_t ai = a[i], bi = b[i];
|
|
866
|
+
simsimd_f32_t sum = ai + bi;
|
|
867
|
+
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
SIMSIMD_PUBLIC void simsimd_scale_u8_haswell(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
872
|
+
simsimd_u8_t *result) {
|
|
873
|
+
|
|
874
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
875
|
+
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
876
|
+
int sum_i32s[8], a_i32s[8];
|
|
877
|
+
|
|
878
|
+
// The main loop:
|
|
879
|
+
simsimd_size_t i = 0;
|
|
880
|
+
for (; i + 8 <= n; i += 8) {
|
|
881
|
+
//? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
|
|
882
|
+
//? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
|
|
883
|
+
a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
|
|
884
|
+
a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
|
|
885
|
+
//! This can be done at least 50% faster if we convert 8-bit integers to floats instead
|
|
886
|
+
//! of relying on the slow `_mm256_cvtepi32_ps` instruction.
|
|
887
|
+
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
888
|
+
// The normal part.
|
|
889
|
+
__m256 sum_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
890
|
+
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
891
|
+
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
892
|
+
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
|
|
893
|
+
sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
|
|
894
|
+
// Export into a serial buffer.
|
|
895
|
+
_mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
|
|
896
|
+
result[i + 0] = (simsimd_u8_t)sum_i32s[0];
|
|
897
|
+
result[i + 1] = (simsimd_u8_t)sum_i32s[1];
|
|
898
|
+
result[i + 2] = (simsimd_u8_t)sum_i32s[2];
|
|
899
|
+
result[i + 3] = (simsimd_u8_t)sum_i32s[3];
|
|
900
|
+
result[i + 4] = (simsimd_u8_t)sum_i32s[4];
|
|
901
|
+
result[i + 5] = (simsimd_u8_t)sum_i32s[5];
|
|
902
|
+
result[i + 6] = (simsimd_u8_t)sum_i32s[6];
|
|
903
|
+
result[i + 7] = (simsimd_u8_t)sum_i32s[7];
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
// The tail:
|
|
907
|
+
for (; i < n; ++i) {
|
|
908
|
+
simsimd_f32_t ai = a[i];
|
|
909
|
+
simsimd_f32_t sum = alpha_f32 * ai;
|
|
910
|
+
SIMSIMD_F32_TO_U8(sum, result + i);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
|
|
548
914
|
SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
|
|
549
915
|
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
550
916
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
551
917
|
|
|
918
|
+
// There are are several special cases we may want to implement:
|
|
919
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
920
|
+
if (alpha == 1 && beta == 1) {
|
|
921
|
+
// In this case we can avoid expensive multiplications.
|
|
922
|
+
simsimd_sum_u8_haswell(a, b, n, result);
|
|
923
|
+
return;
|
|
924
|
+
}
|
|
925
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
926
|
+
else if (alpha == 0 || beta == 0) {
|
|
927
|
+
// In this case we can avoid half of the load instructions.
|
|
928
|
+
if (beta == 0) { simsimd_scale_u8_haswell(a, n, alpha, result); }
|
|
929
|
+
else { simsimd_scale_u8_haswell(b, n, beta, result); }
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
// The general case.
|
|
552
934
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
553
935
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
554
936
|
__m256 alpha_vec = _mm256_set1_ps(alpha_f32);
|
|
@@ -569,9 +951,9 @@ SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
|
|
|
569
951
|
__m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
|
|
570
952
|
__m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
|
|
571
953
|
// The normal part.
|
|
572
|
-
__m256
|
|
573
|
-
__m256
|
|
574
|
-
__m256 sum_vec = _mm256_add_ps(
|
|
954
|
+
__m256 a_scaled_vec = _mm256_mul_ps(a_vec, alpha_vec);
|
|
955
|
+
__m256 b_scaled_vec = _mm256_mul_ps(b_vec, beta_vec);
|
|
956
|
+
__m256 sum_vec = _mm256_add_ps(a_scaled_vec, b_scaled_vec);
|
|
575
957
|
// Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
|
|
576
958
|
__m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
|
|
577
959
|
sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
|
|
@@ -715,14 +1097,73 @@ SIMSIMD_PUBLIC void simsimd_fma_u8_haswell(
|
|
|
715
1097
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
|
|
716
1098
|
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
|
|
717
1099
|
|
|
1100
|
+
SIMSIMD_PUBLIC void simsimd_sum_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1101
|
+
simsimd_f64_t *result) {
|
|
1102
|
+
__m512d a_vec, b_vec, sum_vec;
|
|
1103
|
+
__mmask8 mask = 0xFF;
|
|
1104
|
+
simsimd_sum_f64_skylake_cycle:
|
|
1105
|
+
if (n < 8) {
|
|
1106
|
+
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1107
|
+
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1108
|
+
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1109
|
+
n = 0;
|
|
1110
|
+
}
|
|
1111
|
+
else {
|
|
1112
|
+
a_vec = _mm512_loadu_pd(a);
|
|
1113
|
+
b_vec = _mm512_loadu_pd(b);
|
|
1114
|
+
a += 8, b += 8, n -= 8;
|
|
1115
|
+
}
|
|
1116
|
+
sum_vec = _mm512_add_pd(a_vec, b_vec);
|
|
1117
|
+
_mm512_mask_storeu_pd(result, mask, sum_vec);
|
|
1118
|
+
result += 8;
|
|
1119
|
+
if (n) goto simsimd_sum_f64_skylake_cycle;
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
SIMSIMD_PUBLIC void simsimd_scale_f64_skylake(simsimd_f64_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1123
|
+
simsimd_f64_t *result) {
|
|
1124
|
+
__m512d alpha_vec = _mm512_set1_pd(alpha);
|
|
1125
|
+
__m512d a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
1126
|
+
__mmask8 mask = 0xFF;
|
|
1127
|
+
simsimd_scale_f64_skylake_cycle:
|
|
1128
|
+
if (n < 8) {
|
|
1129
|
+
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1130
|
+
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1131
|
+
n = 0;
|
|
1132
|
+
}
|
|
1133
|
+
else {
|
|
1134
|
+
a_vec = _mm512_loadu_pd(a);
|
|
1135
|
+
a += 8, n -= 8;
|
|
1136
|
+
}
|
|
1137
|
+
sum_vec = _mm512_mul_pd(a_vec, alpha_vec);
|
|
1138
|
+
_mm512_mask_storeu_pd(result, mask, sum_vec);
|
|
1139
|
+
result += 8;
|
|
1140
|
+
if (n) goto simsimd_scale_f64_skylake_cycle;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
718
1143
|
SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
|
|
719
1144
|
simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
|
|
720
1145
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
|
|
1146
|
+
|
|
1147
|
+
// There are are several special cases we may want to implement:
|
|
1148
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1149
|
+
if (alpha == 1 && beta == 1) {
|
|
1150
|
+
// In this case we can avoid expensive multiplications.
|
|
1151
|
+
simsimd_sum_f64_skylake(a, b, n, result);
|
|
1152
|
+
return;
|
|
1153
|
+
}
|
|
1154
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1155
|
+
else if (alpha == 0 || beta == 0) {
|
|
1156
|
+
// In this case we can avoid half of the load instructions.
|
|
1157
|
+
if (beta == 0) { simsimd_scale_f64_skylake(a, n, alpha, result); }
|
|
1158
|
+
else { simsimd_scale_f64_skylake(b, n, beta, result); }
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
// The general case.
|
|
721
1163
|
__m512d alpha_vec = _mm512_set1_pd(alpha);
|
|
722
1164
|
__m512d beta_vec = _mm512_set1_pd(beta);
|
|
723
1165
|
__m512d a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
724
1166
|
__mmask8 mask = 0xFF;
|
|
725
|
-
|
|
726
1167
|
simsimd_wsum_f64_skylake_cycle:
|
|
727
1168
|
if (n < 8) {
|
|
728
1169
|
mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
@@ -742,14 +1183,75 @@ simsimd_wsum_f64_skylake_cycle:
|
|
|
742
1183
|
if (n) goto simsimd_wsum_f64_skylake_cycle;
|
|
743
1184
|
}
|
|
744
1185
|
|
|
1186
|
+
SIMSIMD_PUBLIC void simsimd_sum_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1187
|
+
simsimd_f32_t *result) {
|
|
1188
|
+
__m512 a_vec, b_vec, sum_vec;
|
|
1189
|
+
__mmask16 mask = 0xFFFF;
|
|
1190
|
+
|
|
1191
|
+
simsimd_sum_f32_skylake_cycle:
|
|
1192
|
+
if (n < 16) {
|
|
1193
|
+
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1194
|
+
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1195
|
+
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1196
|
+
n = 0;
|
|
1197
|
+
}
|
|
1198
|
+
else {
|
|
1199
|
+
a_vec = _mm512_loadu_ps(a);
|
|
1200
|
+
b_vec = _mm512_loadu_ps(b);
|
|
1201
|
+
a += 16, b += 16, n -= 16;
|
|
1202
|
+
}
|
|
1203
|
+
sum_vec = _mm512_add_ps(a_vec, b_vec);
|
|
1204
|
+
_mm512_mask_storeu_ps(result, mask, sum_vec);
|
|
1205
|
+
result += 16;
|
|
1206
|
+
if (n) goto simsimd_sum_f32_skylake_cycle;
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
SIMSIMD_PUBLIC void simsimd_scale_f32_skylake(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1210
|
+
simsimd_f32_t *result) {
|
|
1211
|
+
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1212
|
+
__m512 a_vec, sum_vec;
|
|
1213
|
+
__mmask16 mask = 0xFFFF;
|
|
1214
|
+
|
|
1215
|
+
simsimd_scale_f32_skylake_cycle:
|
|
1216
|
+
if (n < 16) {
|
|
1217
|
+
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1218
|
+
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1219
|
+
n = 0;
|
|
1220
|
+
}
|
|
1221
|
+
else {
|
|
1222
|
+
a_vec = _mm512_loadu_ps(a);
|
|
1223
|
+
a += 16, n -= 16;
|
|
1224
|
+
}
|
|
1225
|
+
sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
|
|
1226
|
+
_mm512_mask_storeu_ps(result, mask, sum_vec);
|
|
1227
|
+
result += 16;
|
|
1228
|
+
if (n) goto simsimd_scale_f32_skylake_cycle;
|
|
1229
|
+
}
|
|
1230
|
+
|
|
745
1231
|
SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
|
|
746
1232
|
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
747
1233
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
1234
|
+
|
|
1235
|
+
// There are are several special cases we may want to implement:
|
|
1236
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1237
|
+
if (alpha == 1 && beta == 1) {
|
|
1238
|
+
// In this case we can avoid expensive multiplications.
|
|
1239
|
+
simsimd_sum_f32_skylake(a, b, n, result);
|
|
1240
|
+
return;
|
|
1241
|
+
}
|
|
1242
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1243
|
+
else if (alpha == 0 || beta == 0) {
|
|
1244
|
+
// In this case we can avoid half of the load instructions.
|
|
1245
|
+
if (beta == 0) { simsimd_scale_f32_skylake(a, n, alpha, result); }
|
|
1246
|
+
else { simsimd_scale_f32_skylake(b, n, beta, result); }
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
// The general case.
|
|
748
1251
|
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
749
1252
|
__m512 beta_vec = _mm512_set1_ps(beta);
|
|
750
1253
|
__m512 a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
751
1254
|
__mmask16 mask = 0xFFFF;
|
|
752
|
-
|
|
753
1255
|
simsimd_wsum_f32_skylake_cycle:
|
|
754
1256
|
if (n < 16) {
|
|
755
1257
|
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
@@ -769,15 +1271,81 @@ simsimd_wsum_f32_skylake_cycle:
|
|
|
769
1271
|
if (n) goto simsimd_wsum_f32_skylake_cycle;
|
|
770
1272
|
}
|
|
771
1273
|
|
|
1274
|
+
SIMSIMD_PUBLIC void simsimd_sum_bf16_skylake(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1275
|
+
simsimd_bf16_t *result) {
|
|
1276
|
+
__m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
|
|
1277
|
+
__m512 a_vec, b_vec, sum_vec;
|
|
1278
|
+
__mmask16 mask = 0xFFFF;
|
|
1279
|
+
simsimd_sum_bf16_skylake_cycle:
|
|
1280
|
+
if (n < 16) {
|
|
1281
|
+
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1282
|
+
a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
|
|
1283
|
+
b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
|
|
1284
|
+
n = 0;
|
|
1285
|
+
}
|
|
1286
|
+
else {
|
|
1287
|
+
a_bf16_vec = _mm256_loadu_epi16(a);
|
|
1288
|
+
b_bf16_vec = _mm256_loadu_epi16(b);
|
|
1289
|
+
a += 16, b += 16, n -= 16;
|
|
1290
|
+
}
|
|
1291
|
+
a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
|
|
1292
|
+
b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
|
|
1293
|
+
sum_vec = _mm512_add_ps(a_vec, b_vec);
|
|
1294
|
+
sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
|
|
1295
|
+
_mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
|
|
1296
|
+
result += 16;
|
|
1297
|
+
if (n) goto simsimd_sum_bf16_skylake_cycle;
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
SIMSIMD_PUBLIC void simsimd_scale_bf16_skylake(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1301
|
+
simsimd_bf16_t *result) {
|
|
1302
|
+
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
1303
|
+
__m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
|
|
1304
|
+
__m512 a_vec, b_vec, sum_vec;
|
|
1305
|
+
__mmask16 mask = 0xFFFF;
|
|
1306
|
+
simsimd_wsum_bf16_skylake_cycle:
|
|
1307
|
+
if (n < 16) {
|
|
1308
|
+
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1309
|
+
a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
|
|
1310
|
+
n = 0;
|
|
1311
|
+
}
|
|
1312
|
+
else {
|
|
1313
|
+
a_bf16_vec = _mm256_loadu_epi16(a);
|
|
1314
|
+
a += 16, n -= 16;
|
|
1315
|
+
}
|
|
1316
|
+
a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
|
|
1317
|
+
sum_vec = _mm512_mul_ps(a_vec, alpha_vec);
|
|
1318
|
+
sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
|
|
1319
|
+
_mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
|
|
1320
|
+
result += 16;
|
|
1321
|
+
if (n) goto simsimd_wsum_bf16_skylake_cycle;
|
|
1322
|
+
}
|
|
1323
|
+
|
|
772
1324
|
SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
|
|
773
1325
|
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
774
1326
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
1327
|
+
|
|
1328
|
+
// There are are several special cases we may want to implement:
|
|
1329
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1330
|
+
if (alpha == 1 && beta == 1) {
|
|
1331
|
+
// In this case we can avoid expensive multiplications.
|
|
1332
|
+
simsimd_sum_bf16_skylake(a, b, n, result);
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1336
|
+
else if (alpha == 0 || beta == 0) {
|
|
1337
|
+
// In this case we can avoid half of the load instructions.
|
|
1338
|
+
if (beta == 0) { simsimd_scale_bf16_skylake(a, n, alpha, result); }
|
|
1339
|
+
else { simsimd_scale_bf16_skylake(b, n, beta, result); }
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
// The general case.
|
|
775
1344
|
__m512 alpha_vec = _mm512_set1_ps(alpha);
|
|
776
1345
|
__m512 beta_vec = _mm512_set1_ps(beta);
|
|
777
1346
|
__m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
|
|
778
1347
|
__m512 a_vec, b_vec, a_scaled_vec, sum_vec;
|
|
779
1348
|
__mmask16 mask = 0xFFFF;
|
|
780
|
-
|
|
781
1349
|
simsimd_wsum_bf16_skylake_cycle:
|
|
782
1350
|
if (n < 16) {
|
|
783
1351
|
mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
@@ -895,26 +1463,87 @@ simsimd_fma_bf16_skylake_cycle:
|
|
|
895
1463
|
if (n) goto simsimd_fma_bf16_skylake_cycle;
|
|
896
1464
|
}
|
|
897
1465
|
|
|
898
|
-
#pragma clang attribute pop
|
|
899
|
-
#pragma GCC pop_options
|
|
900
|
-
#endif // SIMSIMD_TARGET_SKYLAKE
|
|
1466
|
+
#pragma clang attribute pop
|
|
1467
|
+
#pragma GCC pop_options
|
|
1468
|
+
#endif // SIMSIMD_TARGET_SKYLAKE
|
|
1469
|
+
|
|
1470
|
+
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1471
|
+
#pragma GCC push_options
|
|
1472
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1473
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
|
|
1474
|
+
apply_to = function)
|
|
1475
|
+
|
|
1476
|
+
SIMSIMD_PUBLIC void simsimd_sum_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1477
|
+
simsimd_f16_t *result) {
|
|
1478
|
+
__mmask32 mask = 0xFFFFFFFF;
|
|
1479
|
+
__m512h a_f16_vec, b_f16_vec;
|
|
1480
|
+
__m512h sum_f16_vec;
|
|
1481
|
+
simsimd_sum_f16_sapphire_cycle:
|
|
1482
|
+
if (n < 32) {
|
|
1483
|
+
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1484
|
+
a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
1485
|
+
b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
1486
|
+
n = 0;
|
|
1487
|
+
}
|
|
1488
|
+
else {
|
|
1489
|
+
a_f16_vec = _mm512_loadu_ph(a);
|
|
1490
|
+
b_f16_vec = _mm512_loadu_ph(b);
|
|
1491
|
+
a += 32, b += 32, n -= 32;
|
|
1492
|
+
}
|
|
1493
|
+
sum_f16_vec = _mm512_add_ph(a_f16_vec, b_f16_vec);
|
|
1494
|
+
_mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
|
|
1495
|
+
result += 32;
|
|
1496
|
+
if (n) goto simsimd_sum_f16_sapphire_cycle;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
SIMSIMD_PUBLIC void simsimd_scale_f16_sapphire(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1500
|
+
simsimd_f16_t *result) {
|
|
901
1501
|
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
1502
|
+
__mmask32 mask = 0xFFFFFFFF;
|
|
1503
|
+
__m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
|
|
1504
|
+
__m512h a_f16_vec, b_f16_vec;
|
|
1505
|
+
__m512h sum_f16_vec;
|
|
1506
|
+
simsimd_scale_f16_sapphire_cycle:
|
|
1507
|
+
if (n < 32) {
|
|
1508
|
+
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1509
|
+
a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
|
|
1510
|
+
n = 0;
|
|
1511
|
+
}
|
|
1512
|
+
else {
|
|
1513
|
+
a_f16_vec = _mm512_loadu_ph(a);
|
|
1514
|
+
a += 32, n -= 32;
|
|
1515
|
+
}
|
|
1516
|
+
sum_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
|
|
1517
|
+
_mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
|
|
1518
|
+
result += 32;
|
|
1519
|
+
if (n) goto simsimd_scale_f16_sapphire_cycle;
|
|
1520
|
+
}
|
|
907
1521
|
|
|
908
1522
|
SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
|
|
909
1523
|
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
910
1524
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
911
1525
|
|
|
1526
|
+
// There are are several special cases we may want to implement:
|
|
1527
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1528
|
+
if (alpha == 1 && beta == 1) {
|
|
1529
|
+
// In this case we can avoid expensive multiplications.
|
|
1530
|
+
simsimd_sum_f16_sapphire(a, b, n, result);
|
|
1531
|
+
return;
|
|
1532
|
+
}
|
|
1533
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1534
|
+
else if (alpha == 0 || beta == 0) {
|
|
1535
|
+
// In this case we can avoid half of the load instructions.
|
|
1536
|
+
if (beta == 0) { simsimd_scale_f16_sapphire(a, n, alpha, result); }
|
|
1537
|
+
else { simsimd_scale_f16_sapphire(b, n, beta, result); }
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
// The general case.
|
|
912
1542
|
__mmask32 mask = 0xFFFFFFFF;
|
|
913
1543
|
__m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
|
|
914
1544
|
__m512h beta_vec = _mm512_set1_ph((_Float16)beta);
|
|
915
|
-
__m512h a_f16_vec, b_f16_vec
|
|
1545
|
+
__m512h a_f16_vec, b_f16_vec;
|
|
916
1546
|
__m512h a_scaled_f16_vec, sum_f16_vec;
|
|
917
|
-
|
|
918
1547
|
simsimd_wsum_f16_sapphire_cycle:
|
|
919
1548
|
if (n < 32) {
|
|
920
1549
|
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
@@ -943,7 +1572,6 @@ SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire(
|
|
|
943
1572
|
__m512h beta_vec = _mm512_set1_ph((_Float16)beta);
|
|
944
1573
|
__m512h a_f16_vec, b_f16_vec, c_f16_vec;
|
|
945
1574
|
__m512h ab_f16_vec, ab_scaled_f16_vec, sum_f16_vec;
|
|
946
|
-
|
|
947
1575
|
simsimd_fma_f16_sapphire_cycle:
|
|
948
1576
|
if (n < 32) {
|
|
949
1577
|
mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
@@ -966,18 +1594,88 @@ simsimd_fma_f16_sapphire_cycle:
|
|
|
966
1594
|
if (n) goto simsimd_fma_f16_sapphire_cycle;
|
|
967
1595
|
}
|
|
968
1596
|
|
|
1597
|
+
SIMSIMD_PUBLIC void simsimd_sum_u8_sapphire(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1598
|
+
simsimd_u8_t *result) {
|
|
1599
|
+
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1600
|
+
__m512i a_u8_vec, b_u8_vec, sum_u8_vec;
|
|
1601
|
+
simsimd_sum_u8_sapphire_cycle:
|
|
1602
|
+
if (n < 64) {
|
|
1603
|
+
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1604
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1605
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1606
|
+
n = 0;
|
|
1607
|
+
}
|
|
1608
|
+
else {
|
|
1609
|
+
a_u8_vec = _mm512_loadu_epi8(a);
|
|
1610
|
+
b_u8_vec = _mm512_loadu_epi8(b);
|
|
1611
|
+
a += 64, b += 64, n -= 64;
|
|
1612
|
+
}
|
|
1613
|
+
sum_u8_vec = _mm512_adds_epu8(a_u8_vec, b_u8_vec);
|
|
1614
|
+
_mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
|
|
1615
|
+
result += 64;
|
|
1616
|
+
if (n) goto simsimd_sum_u8_sapphire_cycle;
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
SIMSIMD_PUBLIC void simsimd_scale_u8_sapphire(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1620
|
+
simsimd_u8_t *result) {
|
|
1621
|
+
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1622
|
+
__m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
|
|
1623
|
+
__m512i a_u8_vec, b_u8_vec, sum_u8_vec;
|
|
1624
|
+
__m512h a_f16_low_vec, a_f16_high_vec;
|
|
1625
|
+
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1626
|
+
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1627
|
+
simsimd_scale_u8_sapphire_cycle:
|
|
1628
|
+
if (n < 64) {
|
|
1629
|
+
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1630
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1631
|
+
n = 0;
|
|
1632
|
+
}
|
|
1633
|
+
else {
|
|
1634
|
+
a_u8_vec = _mm512_loadu_epi8(a);
|
|
1635
|
+
a += 64, n -= 64;
|
|
1636
|
+
}
|
|
1637
|
+
// Upcast:
|
|
1638
|
+
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1639
|
+
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
|
|
1640
|
+
// Scale:
|
|
1641
|
+
sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
|
|
1642
|
+
sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
|
|
1643
|
+
// Downcast:
|
|
1644
|
+
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1645
|
+
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1646
|
+
sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
|
|
1647
|
+
_mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
|
|
1648
|
+
result += 64;
|
|
1649
|
+
if (n) goto simsimd_scale_u8_sapphire_cycle;
|
|
1650
|
+
}
|
|
1651
|
+
|
|
969
1652
|
SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
|
|
970
1653
|
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
971
1654
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
972
1655
|
|
|
1656
|
+
// There are are several special cases we may want to implement:
|
|
1657
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1658
|
+
if (alpha == 1 && beta == 1) {
|
|
1659
|
+
// In this case we can avoid expensive multiplications.
|
|
1660
|
+
simsimd_sum_u8_sapphire(a, b, n, result);
|
|
1661
|
+
return;
|
|
1662
|
+
}
|
|
1663
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1664
|
+
else if (alpha == 0 || beta == 0) {
|
|
1665
|
+
// In this case we can avoid half of the load instructions.
|
|
1666
|
+
if (beta == 0) { simsimd_scale_u8_sapphire(a, n, alpha, result); }
|
|
1667
|
+
else { simsimd_scale_u8_sapphire(b, n, beta, result); }
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
// The general case.
|
|
973
1672
|
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
974
1673
|
__m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
|
|
975
1674
|
__m512h beta_vec = _mm512_set1_ph((_Float16)beta);
|
|
976
|
-
__m512i a_u8_vec, b_u8_vec,
|
|
1675
|
+
__m512i a_u8_vec, b_u8_vec, sum_u8_vec;
|
|
977
1676
|
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
978
1677
|
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
979
1678
|
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
980
|
-
|
|
981
1679
|
simsimd_wsum_u8_sapphire_cycle:
|
|
982
1680
|
if (n < 64) {
|
|
983
1681
|
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
@@ -1010,14 +1708,92 @@ simsimd_wsum_u8_sapphire_cycle:
|
|
|
1010
1708
|
if (n) goto simsimd_wsum_u8_sapphire_cycle;
|
|
1011
1709
|
}
|
|
1012
1710
|
|
|
1711
|
+
SIMSIMD_PUBLIC void simsimd_sum_i8_sapphire(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1712
|
+
simsimd_i8_t *result) {
|
|
1713
|
+
|
|
1714
|
+
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1715
|
+
__m512i a_i8_vec, b_i8_vec, sum_i8_vec;
|
|
1716
|
+
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1717
|
+
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1718
|
+
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1719
|
+
|
|
1720
|
+
simsimd_sum_i8_sapphire_cycle:
|
|
1721
|
+
if (n < 64) {
|
|
1722
|
+
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1723
|
+
a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1724
|
+
b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1725
|
+
n = 0;
|
|
1726
|
+
}
|
|
1727
|
+
else {
|
|
1728
|
+
a_i8_vec = _mm512_loadu_epi8(a);
|
|
1729
|
+
b_i8_vec = _mm512_loadu_epi8(b);
|
|
1730
|
+
a += 64, b += 64, n -= 64;
|
|
1731
|
+
}
|
|
1732
|
+
sum_i8_vec = _mm512_adds_epi8(a_i8_vec, b_i8_vec);
|
|
1733
|
+
_mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
|
|
1734
|
+
result += 64;
|
|
1735
|
+
if (n) goto simsimd_sum_i8_sapphire_cycle;
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
SIMSIMD_PUBLIC void simsimd_scale_i8_sapphire(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1739
|
+
simsimd_i8_t *result) {
|
|
1740
|
+
|
|
1741
|
+
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1742
|
+
__m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
|
|
1743
|
+
__m512i a_i8_vec, sum_i8_vec;
|
|
1744
|
+
__m512h a_f16_low_vec, a_f16_high_vec;
|
|
1745
|
+
__m512h sum_f16_low_vec, sum_f16_high_vec;
|
|
1746
|
+
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
1747
|
+
simsimd_wsum_i8_sapphire_cycle:
|
|
1748
|
+
if (n < 64) {
|
|
1749
|
+
mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
|
|
1750
|
+
a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1751
|
+
n = 0;
|
|
1752
|
+
}
|
|
1753
|
+
else {
|
|
1754
|
+
a_i8_vec = _mm512_loadu_epi8(a);
|
|
1755
|
+
a += 64, n -= 64;
|
|
1756
|
+
}
|
|
1757
|
+
// Upcast:
|
|
1758
|
+
a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
|
|
1759
|
+
a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
|
|
1760
|
+
// Scale:
|
|
1761
|
+
sum_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
|
|
1762
|
+
sum_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
|
|
1763
|
+
// Downcast:
|
|
1764
|
+
sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
|
|
1765
|
+
sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
|
|
1766
|
+
sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
|
|
1767
|
+
_mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
|
|
1768
|
+
_mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
|
|
1769
|
+
result += 64;
|
|
1770
|
+
if (n) goto simsimd_wsum_i8_sapphire_cycle;
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1013
1773
|
SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
|
|
1014
1774
|
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
1015
1775
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
1016
1776
|
|
|
1777
|
+
// There are are several special cases we may want to implement:
|
|
1778
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1779
|
+
if (alpha == 1 && beta == 1) {
|
|
1780
|
+
// In this case we can avoid expensive multiplications.
|
|
1781
|
+
simsimd_sum_i8_sapphire(a, b, n, result);
|
|
1782
|
+
return;
|
|
1783
|
+
}
|
|
1784
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1785
|
+
else if (alpha == 0 || beta == 0) {
|
|
1786
|
+
// In this case we can avoid half of the load instructions.
|
|
1787
|
+
if (beta == 0) { simsimd_scale_i8_sapphire(a, n, alpha, result); }
|
|
1788
|
+
else { simsimd_scale_i8_sapphire(b, n, beta, result); }
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
// The general case.
|
|
1017
1793
|
__mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
|
|
1018
1794
|
__m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
|
|
1019
1795
|
__m512h beta_vec = _mm512_set1_ph((_Float16)beta);
|
|
1020
|
-
__m512i a_i8_vec, b_i8_vec,
|
|
1796
|
+
__m512i a_i8_vec, b_i8_vec, sum_i8_vec;
|
|
1021
1797
|
__m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
|
|
1022
1798
|
__m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
|
|
1023
1799
|
__m512i sum_i16_low_vec, sum_i16_high_vec;
|
|
@@ -1171,9 +1947,57 @@ simsimd_fma_u8_sapphire_cycle:
|
|
|
1171
1947
|
#pragma GCC target("arch=armv8.2-a+simd")
|
|
1172
1948
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
1173
1949
|
|
|
1950
|
+
SIMSIMD_PUBLIC void simsimd_sum_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1951
|
+
simsimd_f32_t *result) {
|
|
1952
|
+
// The main loop:
|
|
1953
|
+
simsimd_size_t i = 0;
|
|
1954
|
+
for (; i + 4 <= n; i += 4) {
|
|
1955
|
+
float32x4_t a_vec = vld1q_f32(a + i);
|
|
1956
|
+
float32x4_t b_vec = vld1q_f32(b + i);
|
|
1957
|
+
float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
|
|
1958
|
+
vst1q_f32(result + i, sum_vec);
|
|
1959
|
+
}
|
|
1960
|
+
|
|
1961
|
+
// The tail:
|
|
1962
|
+
for (; i < n; ++i) result[i] = a[i] + b[i];
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1965
|
+
SIMSIMD_PUBLIC void simsimd_scale_f32_neon(simsimd_f32_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
1966
|
+
simsimd_f32_t *result) {
|
|
1967
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
1968
|
+
|
|
1969
|
+
// The main loop:
|
|
1970
|
+
simsimd_size_t i = 0;
|
|
1971
|
+
for (; i + 4 <= n; i += 4) {
|
|
1972
|
+
float32x4_t a_vec = vld1q_f32(a + i);
|
|
1973
|
+
float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
|
|
1974
|
+
vst1q_f32(result + i, sum_vec);
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
// The tail:
|
|
1978
|
+
for (; i < n; ++i) result[i] = alpha_f32 * a[i];
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1174
1981
|
SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
|
|
1175
1982
|
simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
1176
1983
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
|
|
1984
|
+
|
|
1985
|
+
// There are are several special cases we may want to implement:
|
|
1986
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
1987
|
+
if (alpha == 1 && beta == 1) {
|
|
1988
|
+
// In this case we can avoid expensive multiplications.
|
|
1989
|
+
simsimd_sum_f32_neon(a, b, n, result);
|
|
1990
|
+
return;
|
|
1991
|
+
}
|
|
1992
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
1993
|
+
else if (alpha == 0 || beta == 0) {
|
|
1994
|
+
// In this case we can avoid half of the load instructions.
|
|
1995
|
+
if (beta == 0) { simsimd_scale_f32_neon(a, n, alpha, result); }
|
|
1996
|
+
else { simsimd_scale_f32_neon(b, n, beta, result); }
|
|
1997
|
+
return;
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// The general case.
|
|
1177
2001
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
1178
2002
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
1179
2003
|
|
|
@@ -1223,9 +2047,57 @@ SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
|
|
|
1223
2047
|
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
1224
2048
|
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
1225
2049
|
|
|
2050
|
+
SIMSIMD_PUBLIC void simsimd_sum_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
2051
|
+
simsimd_bf16_t *result) {
|
|
2052
|
+
// The main loop:
|
|
2053
|
+
simsimd_size_t i = 0;
|
|
2054
|
+
for (; i + 4 <= n; i += 4) {
|
|
2055
|
+
float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
|
|
2056
|
+
float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
|
|
2057
|
+
float32x4_t sum_vec = vaddq_f32(a_vec, b_vec);
|
|
2058
|
+
vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
|
|
2059
|
+
}
|
|
2060
|
+
|
|
2061
|
+
// The tail:
|
|
2062
|
+
for (; i < n; ++i) simsimd_f32_to_bf16(simsimd_bf16_to_f32(a + i) + simsimd_bf16_to_f32(b + i), result + i);
|
|
2063
|
+
}
|
|
2064
|
+
|
|
2065
|
+
SIMSIMD_PUBLIC void simsimd_scale_bf16_neon(simsimd_bf16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2066
|
+
simsimd_bf16_t *result) {
|
|
2067
|
+
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
2068
|
+
|
|
2069
|
+
// The main loop:
|
|
2070
|
+
simsimd_size_t i = 0;
|
|
2071
|
+
for (; i + 4 <= n; i += 4) {
|
|
2072
|
+
float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
|
|
2073
|
+
float32x4_t sum_vec = vmulq_n_f32(a_vec, alpha_f32);
|
|
2074
|
+
vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
// The tail:
|
|
2078
|
+
for (; i < n; ++i) simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i), result + i);
|
|
2079
|
+
}
|
|
2080
|
+
|
|
1226
2081
|
SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
|
|
1227
2082
|
simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
1228
2083
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
|
|
2084
|
+
|
|
2085
|
+
// There are are several special cases we may want to implement:
|
|
2086
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2087
|
+
if (alpha == 1 && beta == 1) {
|
|
2088
|
+
// In this case we can avoid expensive multiplications.
|
|
2089
|
+
simsimd_sum_bf16_neon(a, b, n, result);
|
|
2090
|
+
return;
|
|
2091
|
+
}
|
|
2092
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2093
|
+
else if (alpha == 0 || beta == 0) {
|
|
2094
|
+
// In this case we can avoid half of the load instructions.
|
|
2095
|
+
if (beta == 0) { simsimd_scale_bf16_neon(a, n, alpha, result); }
|
|
2096
|
+
else { simsimd_scale_bf16_neon(b, n, beta, result); }
|
|
2097
|
+
return;
|
|
2098
|
+
}
|
|
2099
|
+
|
|
2100
|
+
// The general case.
|
|
1229
2101
|
simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
|
|
1230
2102
|
simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
|
|
1231
2103
|
|
|
@@ -1279,9 +2151,56 @@ SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( /
|
|
|
1279
2151
|
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
1280
2152
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
1281
2153
|
|
|
2154
|
+
SIMSIMD_PUBLIC void simsimd_sum_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
2155
|
+
simsimd_f16_t *result) {
|
|
2156
|
+
// The main loop:
|
|
2157
|
+
simsimd_size_t i = 0;
|
|
2158
|
+
for (; i + 8 <= n; i += 8) {
|
|
2159
|
+
float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
|
|
2160
|
+
float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
|
|
2161
|
+
float16x8_t sum_vec = vaddq_f16(a_vec, b_vec);
|
|
2162
|
+
vst1q_f16((float16_t *)result + i, sum_vec);
|
|
2163
|
+
}
|
|
2164
|
+
|
|
2165
|
+
// The tail:
|
|
2166
|
+
for (; i < n; ++i) ((float16_t *)result)[i] = ((float16_t const *)a)[i] + ((float16_t const *)b)[i];
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
SIMSIMD_PUBLIC void simsimd_scale_f16_neon(simsimd_f16_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2170
|
+
simsimd_f16_t *result) {
|
|
2171
|
+
float16_t alpha_f16 = (float16_t)alpha;
|
|
2172
|
+
|
|
2173
|
+
// The main loop:
|
|
2174
|
+
simsimd_size_t i = 0;
|
|
2175
|
+
for (; i + 8 <= n; i += 8) {
|
|
2176
|
+
float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
|
|
2177
|
+
float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2178
|
+
vst1q_f16((float16_t *)result + i, sum_vec);
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
// The tail:
|
|
2182
|
+
for (; i < n; ++i) ((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i];
|
|
2183
|
+
}
|
|
2184
|
+
|
|
1282
2185
|
SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
|
|
1283
2186
|
simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
1284
2187
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
|
|
2188
|
+
// There are are several special cases we may want to implement:
|
|
2189
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2190
|
+
if (alpha == 1 && beta == 1) {
|
|
2191
|
+
// In this case we can avoid expensive multiplications.
|
|
2192
|
+
simsimd_sum_f16_neon(a, b, n, result);
|
|
2193
|
+
return;
|
|
2194
|
+
}
|
|
2195
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2196
|
+
else if (alpha == 0 || beta == 0) {
|
|
2197
|
+
// In this case we can avoid half of the load instructions.
|
|
2198
|
+
if (beta == 0) { simsimd_scale_f16_neon(a, n, alpha, result); }
|
|
2199
|
+
else { simsimd_scale_f16_neon(b, n, beta, result); }
|
|
2200
|
+
return;
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
// The general case.
|
|
1285
2204
|
float16_t alpha_f16 = (float16_t)alpha;
|
|
1286
2205
|
float16_t beta_f16 = (float16_t)beta;
|
|
1287
2206
|
|
|
@@ -1325,9 +2244,59 @@ SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
|
|
|
1325
2244
|
alpha_f16 * ((float16_t const *)a)[i] * ((float16_t const *)b)[i] + beta_f16 * ((float16_t const *)c)[i];
|
|
1326
2245
|
}
|
|
1327
2246
|
|
|
2247
|
+
SIMSIMD_PUBLIC void simsimd_sum_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
2248
|
+
simsimd_u8_t *result) {
|
|
2249
|
+
// The main loop:
|
|
2250
|
+
simsimd_size_t i = 0;
|
|
2251
|
+
for (; i + 16 <= n; i += 16) {
|
|
2252
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
2253
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
2254
|
+
uint8x16_t sum_vec = vqaddq_u8(a_vec, b_vec);
|
|
2255
|
+
vst1q_u8(result + i, sum_vec);
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
// The tail:
|
|
2259
|
+
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(a[i] + b[i], result + i); }
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
SIMSIMD_PUBLIC void simsimd_scale_u8_neon(simsimd_u8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2263
|
+
simsimd_u8_t *result) {
|
|
2264
|
+
float16_t alpha_f16 = (float16_t)alpha;
|
|
2265
|
+
|
|
2266
|
+
// The main loop:
|
|
2267
|
+
simsimd_size_t i = 0;
|
|
2268
|
+
for (; i + 8 <= n; i += 8) {
|
|
2269
|
+
uint8x8_t a_u8_vec = vld1_u8(a + i);
|
|
2270
|
+
float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
|
|
2271
|
+
float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2272
|
+
uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
|
|
2273
|
+
vst1_u8(result + i, sum_u8_vec);
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
// The tail:
|
|
2277
|
+
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i], result + i); }
|
|
2278
|
+
}
|
|
2279
|
+
|
|
1328
2280
|
SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
|
|
1329
2281
|
simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
|
|
1330
2282
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
|
|
2283
|
+
|
|
2284
|
+
// There are are several special cases we may want to implement:
|
|
2285
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2286
|
+
if (alpha == 1 && beta == 1) {
|
|
2287
|
+
// In this case we can avoid expensive multiplications.
|
|
2288
|
+
simsimd_sum_u8_neon(a, b, n, result);
|
|
2289
|
+
return;
|
|
2290
|
+
}
|
|
2291
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2292
|
+
else if (alpha == 0 || beta == 0) {
|
|
2293
|
+
// In this case we can avoid half of the load instructions.
|
|
2294
|
+
if (beta == 0) { simsimd_scale_u8_neon(a, n, alpha, result); }
|
|
2295
|
+
else { simsimd_scale_u8_neon(b, n, beta, result); }
|
|
2296
|
+
return;
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
// The general case.
|
|
1331
2300
|
float16_t alpha_f16 = (float16_t)alpha;
|
|
1332
2301
|
float16_t beta_f16 = (float16_t)beta;
|
|
1333
2302
|
|
|
@@ -1375,9 +2344,59 @@ SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
|
|
|
1375
2344
|
for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
|
|
1376
2345
|
}
|
|
1377
2346
|
|
|
2347
|
+
SIMSIMD_PUBLIC void simsimd_sum_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
2348
|
+
simsimd_i8_t *result) {
|
|
2349
|
+
// The main loop:
|
|
2350
|
+
simsimd_size_t i = 0;
|
|
2351
|
+
for (; i + 16 <= n; i += 16) {
|
|
2352
|
+
int8x16_t a_vec = vld1q_s8(a + i);
|
|
2353
|
+
int8x16_t b_vec = vld1q_s8(b + i);
|
|
2354
|
+
int8x16_t sum_vec = vqaddq_s8(a_vec, b_vec);
|
|
2355
|
+
vst1q_s8(result + i, sum_vec);
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
// The tail:
|
|
2359
|
+
for (; i < n; ++i) { SIMSIMD_F32_TO_I8(a[i] + b[i], result + i); }
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
SIMSIMD_PUBLIC void simsimd_scale_i8_neon(simsimd_i8_t const *a, simsimd_size_t n, simsimd_distance_t alpha,
|
|
2363
|
+
simsimd_i8_t *result) {
|
|
2364
|
+
float16_t alpha_f16 = (float16_t)alpha;
|
|
2365
|
+
|
|
2366
|
+
// The main loop:
|
|
2367
|
+
simsimd_size_t i = 0;
|
|
2368
|
+
for (; i + 8 <= n; i += 8) {
|
|
2369
|
+
int8x8_t a_i8_vec = vld1_s8(a + i);
|
|
2370
|
+
float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
|
|
2371
|
+
float16x8_t sum_vec = vmulq_n_f16(a_vec, alpha_f16);
|
|
2372
|
+
int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
|
|
2373
|
+
vst1_s8(result + i, sum_i8_vec);
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
// The tail:
|
|
2377
|
+
for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i], result + i); }
|
|
2378
|
+
}
|
|
2379
|
+
|
|
1378
2380
|
SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
|
|
1379
2381
|
simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
|
|
1380
2382
|
simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
|
|
2383
|
+
|
|
2384
|
+
// There are are several special cases we may want to implement:
|
|
2385
|
+
// 1. Simple addition, when both weights are equal to 1.0.
|
|
2386
|
+
if (alpha == 1 && beta == 1) {
|
|
2387
|
+
// In this case we can avoid expensive multiplications.
|
|
2388
|
+
simsimd_sum_i8_neon(a, b, n, result);
|
|
2389
|
+
return;
|
|
2390
|
+
}
|
|
2391
|
+
// 2. Just scaling, when one of the weights is equal to zero.
|
|
2392
|
+
else if (alpha == 0 || beta == 0) {
|
|
2393
|
+
// In this case we can avoid half of the load instructions.
|
|
2394
|
+
if (beta == 0) { simsimd_scale_i8_neon(a, n, alpha, result); }
|
|
2395
|
+
else { simsimd_scale_i8_neon(b, n, beta, result); }
|
|
2396
|
+
return;
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
// The general case.
|
|
1381
2400
|
float16_t alpha_f16 = (float16_t)alpha;
|
|
1382
2401
|
float16_t beta_f16 = (float16_t)beta;
|
|
1383
2402
|
|