usearch 2.15.2 → 2.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +1 -1
  2. package/binding.gyp +1 -1
  3. package/c/CMakeLists.txt +10 -3
  4. package/cpp/README.md +1 -1
  5. package/cpp/bench.cpp +2 -2
  6. package/cpp/test.cpp +16 -0
  7. package/include/usearch/index.hpp +44 -17
  8. package/include/usearch/index_dense.hpp +51 -89
  9. package/include/usearch/index_plugins.hpp +468 -43
  10. package/javascript/lib.cpp +17 -5
  11. package/javascript/usearch.test.js +205 -34
  12. package/package.json +1 -1
  13. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  14. package/prebuilds/linux-arm64/usearch.node +0 -0
  15. package/prebuilds/linux-x64/usearch.node +0 -0
  16. package/simsimd/.git-blame-ignore-revs +2 -0
  17. package/simsimd/CODE_OF_CONDUCT.md +128 -0
  18. package/simsimd/CONTRIBUTING.md +84 -26
  19. package/simsimd/Cargo.lock +1 -1
  20. package/simsimd/Cargo.toml +4 -4
  21. package/simsimd/MANIFEST.in +3 -0
  22. package/simsimd/Package.swift +15 -3
  23. package/simsimd/README.md +534 -177
  24. package/simsimd/binding.gyp +6 -2
  25. package/simsimd/build.rs +13 -1
  26. package/simsimd/c/lib.c +266 -53
  27. package/simsimd/include/simsimd/binary.h +321 -112
  28. package/simsimd/include/simsimd/curved.h +803 -0
  29. package/simsimd/include/simsimd/dot.h +694 -521
  30. package/simsimd/include/simsimd/elementwise.h +1437 -0
  31. package/simsimd/include/simsimd/geospatial.h +9 -3
  32. package/simsimd/include/simsimd/mesh.h +69 -0
  33. package/simsimd/include/simsimd/probability.h +234 -206
  34. package/simsimd/include/simsimd/simsimd.h +1476 -617
  35. package/simsimd/include/simsimd/sparse.h +1428 -0
  36. package/simsimd/include/simsimd/spatial.h +1556 -742
  37. package/simsimd/include/simsimd/types.h +276 -132
  38. package/simsimd/javascript/lib.c +1 -2
  39. package/simsimd/javascript/simsimd.ts +4 -4
  40. package/simsimd/package.json +4 -4
  41. package/simsimd/rust/lib.rs +179 -1
  42. package/simsimd/scripts/bench.cxx +1070 -0
  43. package/simsimd/scripts/bench.ipynb +187 -0
  44. package/simsimd/{javascript → scripts}/bench.js +1 -1
  45. package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
  46. package/simsimd/scripts/bench_vectors.py +700 -0
  47. package/simsimd/scripts/bench_vectors_live.py +213 -0
  48. package/simsimd/scripts/test.c +155 -0
  49. package/simsimd/{javascript → scripts}/test.mjs +2 -2
  50. package/simsimd/scripts/test.py +1410 -0
  51. package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
  52. package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
  53. /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
  54. /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
@@ -5,19 +5,21 @@
5
5
  * @date March 14, 2023
6
6
  *
7
7
  * Contains:
8
- * - L2 (Euclidean) squared distance
9
- * - Cosine (Angular) similarity
8
+ * - L2 (Euclidean) regular and squared distance
9
+ * - Cosine (Angular) distance - @b not similarity!
10
10
  *
11
11
  * For datatypes:
12
12
  * - 64-bit IEEE floating point numbers
13
13
  * - 32-bit IEEE floating point numbers
14
14
  * - 16-bit IEEE floating point numbers
15
15
  * - 16-bit brain floating point numbers
16
+ * - 8-bit unsigned integral numbers
16
17
  * - 8-bit signed integral numbers
18
+ * - 4-bit signed integral numbers
17
19
  *
18
20
  * For hardware architectures:
19
- * - Arm (NEON, SVE)
20
- * - x86 (AVX2, AVX512)
21
+ * - Arm: NEON, SVE
22
+ * - x86: Haswell, Skylake, Ice Lake, Genoa, Sapphire
21
23
  *
22
24
  * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
23
25
  * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
@@ -27,7 +29,7 @@
27
29
 
28
30
  #include "types.h"
29
31
 
30
- #include "dot.h" // `_mm256_reduce_add_ps_dbl`
32
+ #include "dot.h" // `_simsimd_reduce_f32x8_haswell`
31
33
 
32
34
  #ifdef __cplusplus
33
35
  extern "C" {
@@ -39,49 +41,75 @@ extern "C" {
39
41
  * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
40
42
  * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
41
43
  */
44
+ SIMSIMD_PUBLIC void simsimd_l2_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
42
45
  SIMSIMD_PUBLIC void simsimd_l2sq_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
43
46
  SIMSIMD_PUBLIC void simsimd_cos_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
47
+ SIMSIMD_PUBLIC void simsimd_l2_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
44
48
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
45
49
  SIMSIMD_PUBLIC void simsimd_cos_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
50
+ SIMSIMD_PUBLIC void simsimd_l2_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
46
51
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
47
52
  SIMSIMD_PUBLIC void simsimd_cos_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
53
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
48
54
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
49
55
  SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
56
+ SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
50
57
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
51
58
  SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
59
+ SIMSIMD_PUBLIC void simsimd_l2_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
60
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
61
+ SIMSIMD_PUBLIC void simsimd_cos_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
52
62
 
53
63
  /* Double-precision serial backends for all numeric types.
54
64
  * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
55
65
  */
66
+ SIMSIMD_PUBLIC void simsimd_l2_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
56
67
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
57
68
  SIMSIMD_PUBLIC void simsimd_cos_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
69
+ SIMSIMD_PUBLIC void simsimd_l2_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
58
70
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
59
71
  SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
72
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
60
73
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
61
- SIMSIMD_PUBLIC void simsimd_cos_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
62
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
63
- SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
64
74
 
65
75
  /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
66
76
  * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
67
77
  * server CPUs produced before 2023.
68
78
  */
79
+ SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
80
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
81
+ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
82
+ SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
69
83
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
70
84
  SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
85
+ SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
71
86
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
72
87
  SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
88
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
89
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
90
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
91
+ SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
73
92
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
74
93
  SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
94
+ SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
95
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
96
+ SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
75
97
 
76
98
  /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
77
99
  * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
78
100
  */
79
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
80
- SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
81
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
82
- SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
83
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
84
- SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
101
+ SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
102
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
103
+ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
104
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
105
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
106
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
107
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
108
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
109
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
110
+ SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
111
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
112
+ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
85
113
 
86
114
  /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
87
115
  * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
@@ -89,100 +117,201 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
89
117
  * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
90
118
  * properly vectorized by recent compilers.
91
119
  */
92
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
93
- SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
94
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
95
- SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
96
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
97
- SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
98
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
99
- SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
120
+ SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
121
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
122
+ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
123
+ SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
124
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
125
+ SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
126
+ SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
127
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
128
+ SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
129
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
130
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
131
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
132
+ SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
133
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
134
+ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
135
+ SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
136
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
137
+ SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
100
138
 
101
139
  /* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
102
140
  * Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
103
141
  * as well as masked operations. This is enough to supersede auto-vectorization on `f32` and `f64` types.
104
142
  */
105
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
106
- SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
107
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
108
- SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
143
+ SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
144
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
145
+ SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
146
+ SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
147
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
148
+ SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
109
149
 
110
150
  /* SIMD-powered backends for AVX512 CPUs of Ice Lake generation and newer, using mixed arithmetic over 512-bit words.
111
151
  * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
112
152
  * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
113
153
  */
114
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
115
- SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
116
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
117
- SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
118
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
119
- SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
120
-
154
+ SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
155
+ SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
156
+ SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
157
+ SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
158
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
159
+ SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
160
+ SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
161
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
162
+ SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
163
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
164
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
165
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
166
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
167
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
168
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
169
+
170
+ /* SIMD-powered backends for AVX-INT8-VNNI extensions on Xeon 6 CPUs, including Sierra Forest and Granite Rapids.
171
+ * The packs many "efficiency" cores into a single socket, avoiding heavy 512-bit operations, and focusing on 256-bit ones.
172
+ */
173
+ SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
121
174
  // clang-format on
122
175
 
123
- #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, converter) \
124
- SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const* a, \
125
- simsimd_##input_type##_t const* b, simsimd_size_t n, \
126
- simsimd_distance_t* result) { \
127
- simsimd_##accumulator_type##_t d2 = 0; \
128
- for (simsimd_size_t i = 0; i != n; ++i) { \
129
- simsimd_##accumulator_type##_t ai = converter(a[i]); \
130
- simsimd_##accumulator_type##_t bi = converter(b[i]); \
131
- d2 += (ai - bi) * (ai - bi); \
132
- } \
133
- *result = d2; \
176
+ #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
177
+ SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const *a, \
178
+ simsimd_##input_type##_t const *b, simsimd_size_t n, \
179
+ simsimd_distance_t *result) { \
180
+ simsimd_##accumulator_type##_t d2 = 0; \
181
+ for (simsimd_size_t i = 0; i != n; ++i) { \
182
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
183
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
184
+ d2 += (ai - bi) * (ai - bi); \
185
+ } \
186
+ *result = d2; \
187
+ }
188
+
189
+ #define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
190
+ SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const *a, \
191
+ simsimd_##input_type##_t const *b, simsimd_size_t n, \
192
+ simsimd_distance_t *result) { \
193
+ simsimd_l2sq_##input_type##_##name(a, b, n, result); \
194
+ *result = SIMSIMD_SQRT(*result); \
134
195
  }
135
196
 
136
- #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, converter) \
137
- SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const* a, \
138
- simsimd_##input_type##_t const* b, simsimd_size_t n, \
139
- simsimd_distance_t* result) { \
140
- simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
141
- for (simsimd_size_t i = 0; i != n; ++i) { \
142
- simsimd_##accumulator_type##_t ai = converter(a[i]); \
143
- simsimd_##accumulator_type##_t bi = converter(b[i]); \
144
- ab += ai * bi; \
145
- a2 += ai * ai; \
146
- b2 += bi * bi; \
147
- } \
148
- *result = ab != 0 ? (1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2)) : 1; \
197
+ #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
198
+ SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const *a, \
199
+ simsimd_##input_type##_t const *b, simsimd_size_t n, \
200
+ simsimd_distance_t *result) { \
201
+ simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
202
+ for (simsimd_size_t i = 0; i != n; ++i) { \
203
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
204
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
205
+ ab += ai * bi; \
206
+ a2 += ai * ai; \
207
+ b2 += bi * bi; \
208
+ } \
209
+ if (a2 == 0 && b2 == 0) { *result = 0; } \
210
+ else if (ab == 0) { *result = 1; } \
211
+ else { \
212
+ simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
213
+ *result = unclipped_result > 0 ? unclipped_result : 0; \
214
+ } \
149
215
  }
150
216
 
151
- SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_l2sq_f64_serial
152
- SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_cos_f64_serial
217
+ SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
218
+ SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f64_serial
219
+ SIMSIMD_MAKE_L2(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f64_serial
153
220
 
154
- SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_l2sq_f32_serial
155
- SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_cos_f32_serial
221
+ SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_serial
222
+ SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_serial
223
+ SIMSIMD_MAKE_L2(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_serial
156
224
 
157
- SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_l2sq_f16_serial
158
- SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_cos_f16_serial
225
+ SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_serial
226
+ SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_serial
227
+ SIMSIMD_MAKE_L2(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_serial
159
228
 
160
- SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_l2sq_bf16_serial
161
- SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_cos_bf16_serial
229
+ SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_serial
230
+ SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_serial
231
+ SIMSIMD_MAKE_L2(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_serial
162
232
 
163
- SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_IDENTIFY) // simsimd_l2sq_i8_serial
164
- SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_IDENTIFY) // simsimd_cos_i8_serial
233
+ SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
234
+ SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
235
+ SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
165
236
 
166
- SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_l2sq_f32_accurate
167
- SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_cos_f32_accurate
237
+ SIMSIMD_MAKE_COS(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_u8_serial
238
+ SIMSIMD_MAKE_L2SQ(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_u8_serial
239
+ SIMSIMD_MAKE_L2(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_u8_serial
168
240
 
169
- SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_l2sq_f16_accurate
170
- SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_cos_f16_accurate
241
+ SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
242
+ SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
243
+ SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
171
244
 
172
- SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_l2sq_bf16_accurate
173
- SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_cos_bf16_accurate
245
+ SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_accurate
246
+ SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_accurate
247
+ SIMSIMD_MAKE_L2(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_accurate
174
248
 
175
- SIMSIMD_MAKE_L2SQ(accurate, i8, i32, SIMSIMD_IDENTIFY) // simsimd_l2sq_i8_accurate
176
- SIMSIMD_MAKE_COS(accurate, i8, i32, SIMSIMD_IDENTIFY) // simsimd_cos_i8_accurate
249
+ SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_accurate
250
+ SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
251
+ SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
177
252
 
178
- #if SIMSIMD_TARGET_ARM
253
+ #if _SIMSIMD_TARGET_ARM
179
254
  #if SIMSIMD_TARGET_NEON
180
255
  #pragma GCC push_options
181
- #pragma GCC target("+simd")
182
- #pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
256
+ #pragma GCC target("arch=armv8.2-a+simd")
257
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
183
258
 
184
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
185
- simsimd_distance_t* result) {
259
+ SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_neon(simsimd_f32_t x) {
260
+ return vget_lane_f32(vsqrt_f32(vdup_n_f32(x)), 0);
261
+ }
262
+ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
263
+ return vget_lane_f64(vsqrt_f64(vdup_n_f64(x)), 0);
264
+ }
265
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
266
+ simsimd_f32_t b2) {
267
+ if (a2 == 0 && b2 == 0) return 0;
268
+ if (ab == 0) return 1;
269
+ simsimd_f32_t squares_arr[2] = {a2, b2};
270
+ float32x2_t squares = vld1_f32(squares_arr);
271
+ // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
272
+ // Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
273
+ // One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
274
+ // https://github.com/lighttransport/embree-aarch64/issues/24
275
+ // https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
276
+ float32x2_t rsqrts = vrsqrte_f32(squares);
277
+ // Perform two rounds of Newton-Raphson refinement:
278
+ // https://en.wikipedia.org/wiki/Newton%27s_method
279
+ rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
280
+ rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
281
+ vst1_f32(squares_arr, rsqrts);
282
+ simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
283
+ return result > 0 ? result : 0;
284
+ }
285
+
286
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
287
+ simsimd_f64_t b2) {
288
+ if (a2 == 0 && b2 == 0) return 0;
289
+ if (ab == 0) return 1;
290
+ simsimd_f64_t squares_arr[2] = {a2, b2};
291
+ float64x2_t squares = vld1q_f64(squares_arr);
292
+
293
+ // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
294
+ // Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
295
+ // One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
296
+ // https://github.com/lighttransport/embree-aarch64/issues/24
297
+ // https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
298
+ float64x2_t rsqrts = vrsqrteq_f64(squares);
299
+ // Perform two rounds of Newton-Raphson refinement:
300
+ // https://en.wikipedia.org/wiki/Newton%27s_method
301
+ rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
302
+ rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
303
+ vst1q_f64(squares_arr, rsqrts);
304
+ simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
305
+ return result > 0 ? result : 0;
306
+ }
307
+
308
+ SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
309
+ simsimd_distance_t *result) {
310
+ simsimd_l2sq_f32_neon(a, b, n, result);
311
+ *result = _simsimd_sqrt_f64_neon(*result);
312
+ }
313
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
314
+ simsimd_distance_t *result) {
186
315
  float32x4_t sum_vec = vdupq_n_f32(0);
187
316
  simsimd_size_t i = 0;
188
317
  for (; i + 4 <= n; i += 4) {
@@ -199,8 +328,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t
199
328
  *result = sum;
200
329
  }
201
330
 
202
- SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
203
- simsimd_distance_t* result) {
331
+ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
332
+ simsimd_distance_t *result) {
204
333
  float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
205
334
  simsimd_size_t i = 0;
206
335
  for (; i + 4 <= n; i += 4) {
@@ -216,103 +345,125 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
216
345
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
217
346
  }
218
347
 
219
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
220
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
221
- vst1_f32(a2_b2_arr, vrsqrte_f32(vld1_f32(a2_b2_arr)));
222
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
348
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
349
+ }
350
+
351
+ SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
352
+ simsimd_distance_t *result) {
353
+ simsimd_l2sq_f64_neon(a, b, n, result);
354
+ *result = _simsimd_sqrt_f64_neon(*result);
355
+ }
356
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
357
+ simsimd_distance_t *result) {
358
+ float64x2_t sum_vec = vdupq_n_f64(0);
359
+ simsimd_size_t i = 0;
360
+ for (; i + 2 <= n; i += 2) {
361
+ float64x2_t a_vec = vld1q_f64(a + i);
362
+ float64x2_t b_vec = vld1q_f64(b + i);
363
+ float64x2_t diff_vec = vsubq_f64(a_vec, b_vec);
364
+ sum_vec = vfmaq_f64(sum_vec, diff_vec, diff_vec);
365
+ }
366
+ simsimd_f64_t sum = vaddvq_f64(sum_vec);
367
+ for (; i < n; ++i) {
368
+ simsimd_f64_t diff = a[i] - b[i];
369
+ sum += diff * diff;
370
+ }
371
+ *result = sum;
372
+ }
373
+
374
+ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
375
+ simsimd_distance_t *result) {
376
+ float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
377
+ simsimd_size_t i = 0;
378
+ for (; i + 2 <= n; i += 2) {
379
+ float64x2_t a_vec = vld1q_f64(a + i);
380
+ float64x2_t b_vec = vld1q_f64(b + i);
381
+ ab_vec = vfmaq_f64(ab_vec, a_vec, b_vec);
382
+ a2_vec = vfmaq_f64(a2_vec, a_vec, a_vec);
383
+ b2_vec = vfmaq_f64(b2_vec, b_vec, b_vec);
384
+ }
385
+ simsimd_f64_t ab = vaddvq_f64(ab_vec), a2 = vaddvq_f64(a2_vec), b2 = vaddvq_f64(b2_vec);
386
+ for (; i < n; ++i) {
387
+ simsimd_f64_t ai = a[i], bi = b[i];
388
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
389
+ }
390
+
391
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
223
392
  }
224
393
 
225
394
  #pragma clang attribute pop
226
395
  #pragma GCC pop_options
396
+ #endif // SIMSIMD_TARGET_NEON
227
397
 
398
+ #if SIMSIMD_TARGET_NEON_F16
228
399
  #pragma GCC push_options
229
- #pragma GCC target("+simd+fp16")
230
- #pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
400
+ #pragma GCC target("arch=armv8.2-a+simd+fp16")
401
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
231
402
 
232
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
233
- simsimd_distance_t* result) {
403
+ SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
404
+ simsimd_distance_t *result) {
405
+ simsimd_l2sq_f16_neon(a, b, n, result);
406
+ *result = _simsimd_sqrt_f32_neon(*result);
407
+ }
408
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
409
+ simsimd_distance_t *result) {
410
+ float32x4_t a_vec, b_vec;
234
411
  float32x4_t sum_vec = vdupq_n_f32(0);
235
- simsimd_size_t i = 0;
236
- for (; i + 4 <= n; i += 4) {
237
- float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
238
- float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
239
- float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
240
- sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
241
- }
242
412
 
243
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
244
- // function will run. It is extremely slow, so even for the tail, let's combine serial
245
- // loads and stores with vectorized math.
246
- if (i < n) {
247
- union {
248
- float16x4_t f16_vec;
249
- simsimd_f16_t f16[4];
250
- } a_padded_tail, b_padded_tail;
251
- simsimd_size_t j = 0;
252
- for (; i < n; ++i, ++j)
253
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
254
- for (; j < 4; ++j)
255
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
256
- float32x4_t diff_vec = vsubq_f32(vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
257
- sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
413
+ simsimd_l2sq_f16_neon_cycle:
414
+ if (n < 4) {
415
+ a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
416
+ b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
417
+ n = 0;
418
+ }
419
+ else {
420
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
421
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
422
+ n -= 4, a += 4, b += 4;
258
423
  }
424
+ float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
425
+ sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
426
+ if (n) goto simsimd_l2sq_f16_neon_cycle;
259
427
 
260
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
261
- *result = sum;
428
+ *result = vaddvq_f32(sum_vec);
262
429
  }
263
430
 
264
- SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
265
- simsimd_distance_t* result) {
431
+ SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
432
+ simsimd_distance_t *result) {
266
433
  float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
267
- simsimd_size_t i = 0;
268
- for (; i + 4 <= n; i += 4) {
269
- float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
270
- float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
271
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
272
- a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
273
- b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
274
- }
434
+ float32x4_t a_vec, b_vec;
275
435
 
276
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
277
- // function will run. It is extremely slow, so even for the tail, let's combine serial
278
- // loads and stores with vectorized math.
279
- if (i < n) {
280
- union {
281
- float16x4_t f16_vec;
282
- simsimd_f16_t f16[4];
283
- } a_padded_tail, b_padded_tail;
284
- simsimd_size_t j = 0;
285
- for (; i < n; ++i, ++j)
286
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
287
- for (; j < 4; ++j)
288
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
289
- float32x4_t a_vec = vcvt_f32_f16(a_padded_tail.f16_vec);
290
- float32x4_t b_vec = vcvt_f32_f16(b_padded_tail.f16_vec);
291
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
292
- a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
293
- b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
436
+ simsimd_cos_f16_neon_cycle:
437
+ if (n < 4) {
438
+ a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
439
+ b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
440
+ n = 0;
294
441
  }
442
+ else {
443
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
444
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
445
+ n -= 4, a += 4, b += 4;
446
+ }
447
+ ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
448
+ a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
449
+ b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
450
+ if (n) goto simsimd_cos_f16_neon_cycle;
295
451
 
296
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
297
452
  simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
298
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
299
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
300
- a2_b2 = vrsqrte_f32(a2_b2);
301
- vst1_f32(a2_b2_arr, a2_b2);
302
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
453
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
303
454
  }
304
455
 
305
456
  #pragma clang attribute pop
306
457
  #pragma GCC pop_options
307
- #endif // SIMSIMD_TARGET_NEON
458
+ #endif // SIMSIMD_TARGET_NEON_F16
308
459
 
309
460
  #if SIMSIMD_TARGET_NEON_BF16
310
461
  #pragma GCC push_options
311
462
  #pragma GCC target("arch=armv8.6-a+simd+bf16")
312
463
  #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
313
464
 
314
- SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
315
- simsimd_distance_t* result) {
465
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
466
+ simsimd_distance_t *result) {
316
467
 
317
468
  // Similar to `simsimd_cos_i8_neon`, we can use the `BFMMLA` instruction through
318
469
  // the `vbfmmlaq_f32` intrinsic to compute matrix products and later drop 1/4 of values.
@@ -341,95 +492,76 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_
341
492
  // }
342
493
  // float32x4_t products_vec = vaddq_f32(products_high_vec, products_low_vec);
343
494
  // simsimd_f32_t a2 = products_vec[0], ab = products_vec[1], b2 = products_vec[3];
495
+ //
496
+ // Another way of accomplishing the same thing is to process the odd and even elements separately,
497
+ // using special `vbfmlaltq_f32` and `vbfmlalbq_f32` intrinsics:
498
+ //
499
+ // ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
500
+ // ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
501
+ // a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
502
+ // a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
503
+ // b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
504
+ // b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
505
+ //
344
506
 
345
- float32x4_t ab_high_vec = vdupq_n_f32(0), ab_low_vec = vdupq_n_f32(0);
346
- float32x4_t a2_high_vec = vdupq_n_f32(0), a2_low_vec = vdupq_n_f32(0);
347
- float32x4_t b2_high_vec = vdupq_n_f32(0), b2_low_vec = vdupq_n_f32(0);
348
- simsimd_size_t i = 0;
349
- for (; i + 8 <= n; i += 8) {
350
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
351
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
352
- ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
353
- ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
354
- a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
355
- a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
356
- b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
357
- b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
358
- }
359
-
360
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
361
- // function will run. It is extremely slow, so even for the tail, let's combine serial
362
- // loads and stores with vectorized math.
363
- if (i < n) {
364
- union {
365
- bfloat16x8_t bf16_vec;
366
- simsimd_bf16_t bf16[8];
367
- } a_padded_tail, b_padded_tail;
368
- simsimd_size_t j = 0;
369
- for (; i < n; ++i, ++j)
370
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
371
- for (; j < 8; ++j)
372
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
373
- ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
374
- ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
375
- a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
376
- a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
377
- b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
378
- b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
507
+ float32x4_t ab_vec = vdupq_n_f32(0);
508
+ float32x4_t a2_vec = vdupq_n_f32(0);
509
+ float32x4_t b2_vec = vdupq_n_f32(0);
510
+ bfloat16x8_t a_vec, b_vec;
511
+
512
+ simsimd_cos_bf16_neon_cycle:
513
+ if (n < 8) {
514
+ a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
515
+ b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
516
+ n = 0;
379
517
  }
518
+ else {
519
+ a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
520
+ b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
521
+ n -= 8, a += 8, b += 8;
522
+ }
523
+ ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
524
+ a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
525
+ b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
526
+ if (n) goto simsimd_cos_bf16_neon_cycle;
380
527
 
381
528
  // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
382
- simsimd_f32_t ab = vaddvq_f32(vaddq_f32(ab_high_vec, ab_low_vec)),
383
- a2 = vaddvq_f32(vaddq_f32(a2_high_vec, a2_low_vec)),
384
- b2 = vaddvq_f32(vaddq_f32(b2_high_vec, b2_low_vec));
385
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
386
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
387
- a2_b2 = vrsqrte_f32(a2_b2);
388
- vst1_f32(a2_b2_arr, a2_b2);
389
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
390
- }
391
-
392
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
393
- simsimd_distance_t* result) {
394
- float32x4_t diff_high_vec = vdupq_n_f32(0), diff_low_vec = vdupq_n_f32(0);
529
+ simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
530
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
531
+ }
532
+
533
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
534
+ simsimd_distance_t *result) {
535
+ simsimd_l2sq_bf16_neon(a, b, n, result);
536
+ *result = _simsimd_sqrt_f64_neon(*result);
537
+ }
538
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
539
+ simsimd_distance_t *result) {
540
+ float32x4_t diff_high_vec, diff_low_vec;
395
541
  float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
396
- simsimd_size_t i = 0;
397
- for (; i + 8 <= n; i += 8) {
398
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
399
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
400
- // We can't perform subtraction in `bf16`. One option would be to upcast to `f32`
401
- // and then subtract, converting back to `bf16` for computing the squared difference.
542
+
543
+ simsimd_l2sq_bf16_neon_cycle:
544
+ if (n < 8) {
545
+ bfloat16x8_t a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
546
+ bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
402
547
  diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
403
548
  diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
404
- sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
405
- sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
406
- }
407
-
408
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
409
- // function will run. It is extremely slow, so even for the tail, let's combine serial
410
- // loads and stores with vectorized math.
411
- if (i < n) {
412
- union {
413
- bfloat16x8_t bf16_vec;
414
- simsimd_bf16_t bf16[8];
415
- } a_padded_tail, b_padded_tail;
416
- simsimd_size_t j = 0;
417
- for (; i < n; ++i, ++j)
418
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
419
- for (; j < 8; ++j)
420
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
421
- diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_padded_tail.bf16_vec)),
422
- vcvt_f32_bf16(vget_high_bf16(b_padded_tail.bf16_vec)));
423
- diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_padded_tail.bf16_vec)),
424
- vcvt_f32_bf16(vget_low_bf16(b_padded_tail.bf16_vec)));
425
- sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
426
- sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
549
+ n = 0;
427
550
  }
551
+ else {
552
+ bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
553
+ bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
554
+ diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
555
+ diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
556
+ n -= 8, a += 8, b += 8;
557
+ }
558
+ sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
559
+ sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
560
+ if (n) goto simsimd_l2sq_bf16_neon_cycle;
428
561
 
429
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
430
- simsimd_f32_t sum = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
431
- *result = sum;
562
+ *result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
432
563
  }
564
+
433
565
  #pragma clang attribute pop
434
566
  #pragma GCC pop_options
435
567
  #endif // SIMSIMD_TARGET_NEON_BF16
@@ -439,30 +571,40 @@ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16
439
571
  #pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
440
572
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
441
573
 
442
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
443
- simsimd_distance_t* result) {
444
- int32x4_t d2_vec = vdupq_n_s32(0);
574
+ SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
575
+ simsimd_distance_t *result) {
576
+ simsimd_l2sq_i8_neon(a, b, n, result);
577
+ *result = _simsimd_sqrt_f32_neon(*result);
578
+ }
579
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
580
+ simsimd_distance_t *result) {
581
+
582
+ // The naive approach is to upcast 8-bit signed integers into 16-bit signed integers
583
+ // for subtraction, then multiply within 16-bit integers and accumulate the results
584
+ // into 32-bit integers. This approach is slow on modern Arm CPUs. On Graviton 4,
585
+ // that approach results in 17 GB/s of throughput, compared to 39 GB/s for `i8`
586
+ // dot-products.
587
+ //
588
+ // Luckily we can use the `vabdq_s8` which technically returns `i8` values, but it's a
589
+ // matter of reinterpret-casting! That approach boosts us to 33 GB/s of throughput.
590
+ uint32x4_t d2_vec = vdupq_n_u32(0);
445
591
  simsimd_size_t i = 0;
446
- for (; i + 8 <= n; i += 8) {
447
- int8x8_t a_vec = vld1_s8(a + i);
448
- int8x8_t b_vec = vld1_s8(b + i);
449
- int16x8_t a_vec16 = vmovl_s8(a_vec);
450
- int16x8_t b_vec16 = vmovl_s8(b_vec);
451
- int16x8_t d_vec = vsubq_s16(a_vec16, b_vec16);
452
- int32x4_t d_low = vmull_s16(vget_low_s16(d_vec), vget_low_s16(d_vec));
453
- int32x4_t d_high = vmull_s16(vget_high_s16(d_vec), vget_high_s16(d_vec));
454
- d2_vec = vaddq_s32(d2_vec, vaddq_s32(d_low, d_high));
455
- }
456
- int32_t d2 = vaddvq_s32(d2_vec);
592
+ for (; i + 16 <= n; i += 16) {
593
+ int8x16_t a_vec = vld1q_s8(a + i);
594
+ int8x16_t b_vec = vld1q_s8(b + i);
595
+ uint8x16_t d_vec = vreinterpretq_u8_s8(vabdq_s8(a_vec, b_vec));
596
+ d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
597
+ }
598
+ uint32_t d2 = vaddvq_u32(d2_vec);
457
599
  for (; i < n; ++i) {
458
- int32_t n = a[i] - b[i];
459
- d2 += n * n;
600
+ int32_t n = (int32_t)a[i] - b[i];
601
+ d2 += (uint32_t)(n * n);
460
602
  }
461
603
  *result = d2;
462
604
  }
463
605
 
464
- SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
465
- simsimd_distance_t* result) {
606
+ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
607
+ simsimd_distance_t *result) {
466
608
 
467
609
  simsimd_size_t i = 0;
468
610
 
@@ -577,12 +719,57 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
577
719
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
578
720
  }
579
721
 
580
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
581
- simsimd_f32_t a2_b2_arr[2] = {(simsimd_f32_t)a2, (simsimd_f32_t)b2};
582
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
583
- a2_b2 = vrsqrte_f32(a2_b2);
584
- vst1_f32(a2_b2_arr, a2_b2);
585
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
722
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
723
+ }
724
+
725
+ SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
726
+ simsimd_distance_t *result) {
727
+ simsimd_l2sq_u8_neon(a, b, n, result);
728
+ *result = _simsimd_sqrt_f32_neon(*result);
729
+ }
730
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
731
+ simsimd_distance_t *result) {
732
+ uint32x4_t d2_vec = vdupq_n_u32(0);
733
+ simsimd_size_t i = 0;
734
+ for (; i + 16 <= n; i += 16) {
735
+ uint8x16_t a_vec = vld1q_u8(a + i);
736
+ uint8x16_t b_vec = vld1q_u8(b + i);
737
+ uint8x16_t d_vec = vabdq_u8(a_vec, b_vec);
738
+ d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
739
+ }
740
+ uint32_t d2 = vaddvq_u32(d2_vec);
741
+ for (; i < n; ++i) {
742
+ int32_t n = (int32_t)a[i] - b[i];
743
+ d2 += (uint32_t)(n * n);
744
+ }
745
+ *result = d2;
746
+ }
747
+
748
+ SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
749
+ simsimd_distance_t *result) {
750
+
751
+ simsimd_size_t i = 0;
752
+ uint32x4_t ab_vec = vdupq_n_u32(0);
753
+ uint32x4_t a2_vec = vdupq_n_u32(0);
754
+ uint32x4_t b2_vec = vdupq_n_u32(0);
755
+ for (; i + 16 <= n; i += 16) {
756
+ uint8x16_t a_vec = vld1q_u8(a + i);
757
+ uint8x16_t b_vec = vld1q_u8(b + i);
758
+ ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
759
+ a2_vec = vdotq_u32(a2_vec, a_vec, a_vec);
760
+ b2_vec = vdotq_u32(b2_vec, b_vec, b_vec);
761
+ }
762
+ uint32_t ab = vaddvq_u32(ab_vec);
763
+ uint32_t a2 = vaddvq_u32(a2_vec);
764
+ uint32_t b2 = vaddvq_u32(b2_vec);
765
+
766
+ // Take care of the tail:
767
+ for (; i < n; ++i) {
768
+ uint32_t ai = a[i], bi = b[i];
769
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
770
+ }
771
+
772
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
586
773
  }
587
774
 
588
775
  #pragma clang attribute pop
@@ -591,11 +778,16 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
591
778
 
592
779
  #if SIMSIMD_TARGET_SVE
593
780
  #pragma GCC push_options
594
- #pragma GCC target("+sve")
595
- #pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
781
+ #pragma GCC target("arch=armv8.2-a+sve")
782
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
596
783
 
597
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
598
- simsimd_distance_t* result) {
784
+ SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
785
+ simsimd_distance_t *result) {
786
+ simsimd_l2sq_f32_sve(a, b, n, result);
787
+ *result = _simsimd_sqrt_f64_neon(*result);
788
+ }
789
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
790
+ simsimd_distance_t *result) {
599
791
  simsimd_size_t i = 0;
600
792
  svfloat32_t d2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
601
793
  do {
@@ -610,8 +802,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t c
610
802
  *result = d2;
611
803
  }
612
804
 
613
- SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
614
- simsimd_distance_t* result) {
805
+ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
806
+ simsimd_distance_t *result) {
615
807
  simsimd_size_t i = 0;
616
808
  svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
617
809
  svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
@@ -629,17 +821,16 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
629
821
  simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
630
822
  simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
631
823
  simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
632
-
633
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
634
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
635
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
636
- a2_b2 = vrsqrte_f32(a2_b2);
637
- vst1_f32(a2_b2_arr, a2_b2);
638
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
824
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
639
825
  }
640
826
 
641
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
642
- simsimd_distance_t* result) {
827
+ SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
828
+ simsimd_distance_t *result) {
829
+ simsimd_l2sq_f64_sve(a, b, n, result);
830
+ *result = _simsimd_sqrt_f64_neon(*result);
831
+ }
832
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
833
+ simsimd_distance_t *result) {
643
834
  simsimd_size_t i = 0;
644
835
  svfloat64_t d2_vec = svdupq_n_f64(0.0, 0.0);
645
836
  do {
@@ -654,8 +845,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t c
654
845
  *result = d2;
655
846
  }
656
847
 
657
- SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
658
- simsimd_distance_t* result) {
848
+ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
849
+ simsimd_distance_t *result) {
659
850
  simsimd_size_t i = 0;
660
851
  svfloat64_t ab_vec = svdupq_n_f64(0.0, 0.0);
661
852
  svfloat64_t a2_vec = svdupq_n_f64(0.0, 0.0);
@@ -673,28 +864,29 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
673
864
  simsimd_f64_t ab = svaddv_f64(svptrue_b32(), ab_vec);
674
865
  simsimd_f64_t a2 = svaddv_f64(svptrue_b32(), a2_vec);
675
866
  simsimd_f64_t b2 = svaddv_f64(svptrue_b32(), b2_vec);
676
-
677
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
678
- simsimd_f64_t a2_b2_arr[2] = {a2, b2};
679
- float64x2_t a2_b2 = vld1q_f64(a2_b2_arr);
680
- a2_b2 = vrsqrteq_f64(a2_b2);
681
- vst1q_f64(a2_b2_arr, a2_b2);
682
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
867
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
683
868
  }
684
869
 
685
870
  #pragma clang attribute pop
686
871
  #pragma GCC pop_options
872
+ #endif // SIMSIMD_TARGET_SVE
687
873
 
874
+ #if SIMSIMD_TARGET_SVE_F16
688
875
  #pragma GCC push_options
689
- #pragma GCC target("+sve+fp16")
690
- #pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
876
+ #pragma GCC target("arch=armv8.2-a+sve+fp16")
877
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
691
878
 
692
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
693
- simsimd_distance_t* result) {
879
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
880
+ simsimd_distance_t *result) {
881
+ simsimd_l2sq_f16_sve(a, b, n, result);
882
+ *result = _simsimd_sqrt_f32_neon(*result);
883
+ }
884
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
885
+ simsimd_distance_t *result) {
694
886
  simsimd_size_t i = 0;
695
887
  svfloat16_t d2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
696
- simsimd_f16_for_arm_simd_t const* a = (simsimd_f16_for_arm_simd_t const*)(a_enum);
697
- simsimd_f16_for_arm_simd_t const* b = (simsimd_f16_for_arm_simd_t const*)(b_enum);
888
+ simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
889
+ simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
698
890
  do {
699
891
  svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
700
892
  svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
@@ -707,14 +899,14 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f1
707
899
  *result = d2_f16;
708
900
  }
709
901
 
710
- SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
711
- simsimd_distance_t* result) {
902
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
903
+ simsimd_distance_t *result) {
712
904
  simsimd_size_t i = 0;
713
905
  svfloat16_t ab_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
714
906
  svfloat16_t a2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
715
907
  svfloat16_t b2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
716
- simsimd_f16_for_arm_simd_t const* a = (simsimd_f16_for_arm_simd_t const*)(a_enum);
717
- simsimd_f16_for_arm_simd_t const* b = (simsimd_f16_for_arm_simd_t const*)(b_enum);
908
+ simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
909
+ simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
718
910
  do {
719
911
  svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
720
912
  svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
@@ -728,324 +920,467 @@ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
728
920
  simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
729
921
  simsimd_f16_for_arm_simd_t a2 = svaddv_f16(svptrue_b16(), a2_vec);
730
922
  simsimd_f16_for_arm_simd_t b2 = svaddv_f16(svptrue_b16(), b2_vec);
923
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
924
+ }
731
925
 
732
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
733
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
734
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
735
- a2_b2 = vrsqrte_f32(a2_b2);
736
- vst1_f32(a2_b2_arr, a2_b2);
737
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
926
+ #pragma clang attribute pop
927
+ #pragma GCC pop_options
928
+ #endif // SIMSIMD_TARGET_SVE_F16
929
+
930
+ #if SIMSIMD_TARGET_SVE_BF16
931
+ #pragma GCC push_options
932
+ #pragma GCC target("arch=armv8.2-a+sve+bf16")
933
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
934
+
935
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
936
+ simsimd_distance_t *result) {
937
+ simsimd_l2sq_bf16_sve(a, b, n, result);
938
+ *result = _simsimd_sqrt_f32_neon(*result);
939
+ }
940
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
941
+ simsimd_distance_t *result) {
942
+ simsimd_size_t i = 0;
943
+ svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
944
+ svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
945
+ simsimd_u16_t const *a = (simsimd_u16_t const *)(a_enum);
946
+ simsimd_u16_t const *b = (simsimd_u16_t const *)(b_enum);
947
+ do {
948
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
949
+ svuint16_t a_vec = svld1_u16(pg_vec, a + i);
950
+ svuint16_t b_vec = svld1_u16(pg_vec, b + i);
951
+
952
+ // There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
953
+ svbool_t pg_low_vec = svwhilelt_b32((unsigned int)(i), (unsigned int)n);
954
+ svbool_t pg_high_vec = svwhilelt_b32((unsigned int)(i + svcnth() / 2), (unsigned int)n);
955
+ svfloat32_t a_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(a_vec), 16));
956
+ svfloat32_t a_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(a_vec), 16));
957
+ svfloat32_t b_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(b_vec), 16));
958
+ svfloat32_t b_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(b_vec), 16));
959
+
960
+ svfloat32_t a_minus_b_low_vec = svsub_f32_x(pg_low_vec, a_low_vec, b_low_vec);
961
+ svfloat32_t a_minus_b_high_vec = svsub_f32_x(pg_high_vec, a_high_vec, b_high_vec);
962
+ d2_low_vec = svmla_f32_x(pg_vec, d2_low_vec, a_minus_b_low_vec, a_minus_b_low_vec);
963
+ d2_high_vec = svmla_f32_x(pg_vec, d2_high_vec, a_minus_b_low_vec, a_minus_b_low_vec);
964
+ i += svcnth();
965
+ } while (i < n);
966
+ simsimd_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_vec) + svaddv_f32(svptrue_b32(), d2_high_vec);
967
+ *result = d2;
968
+ }
969
+
970
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
971
+ simsimd_distance_t *result) {
972
+ simsimd_size_t i = 0;
973
+ svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
974
+ svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
975
+ svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
976
+ simsimd_bf16_for_arm_simd_t const *a = (simsimd_bf16_for_arm_simd_t const *)(a_enum);
977
+ simsimd_bf16_for_arm_simd_t const *b = (simsimd_bf16_for_arm_simd_t const *)(b_enum);
978
+ do {
979
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
980
+ svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
981
+ svbfloat16_t b_vec = svld1_bf16(pg_vec, b + i);
982
+ ab_vec = svbfdot_f32(ab_vec, a_vec, b_vec);
983
+ a2_vec = svbfdot_f32(a2_vec, a_vec, a_vec);
984
+ b2_vec = svbfdot_f32(b2_vec, b_vec, b_vec);
985
+ i += svcnth();
986
+ } while (i < n);
987
+
988
+ simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
989
+ simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
990
+ simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
991
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
738
992
  }
739
993
 
740
994
  #pragma clang attribute pop
741
995
  #pragma GCC pop_options
742
- #endif // SIMSIMD_TARGET_SVE
743
- #endif // SIMSIMD_TARGET_ARM
996
+ #endif // SIMSIMD_TARGET_SVE_BF16
997
+ #endif // _SIMSIMD_TARGET_ARM
998
+
999
+ #if _SIMSIMD_TARGET_X86
1000
+ #if SIMSIMD_TARGET_HASWELL
1001
+ #pragma GCC push_options
1002
+ #pragma GCC target("avx2")
1003
+ #pragma clang attribute push(__attribute__((target("avx2"))), apply_to = function)
1004
+
1005
+ SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_haswell(simsimd_f32_t x) {
1006
+ return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set_ss(x)));
1007
+ }
1008
+ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_haswell(simsimd_f64_t x) {
1009
+ return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set_sd(x)));
1010
+ }
1011
+
1012
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f64_t ab, simsimd_f64_t a2,
1013
+ simsimd_f64_t b2) {
1014
+
1015
+ // If both vectors have magnitude 0, the distance is 0.
1016
+ if (a2 == 0 && b2 == 0) return 0;
1017
+ // If any one of the vectors is 0, the square root of the product is 0,
1018
+ // the division is illformed, and the result is 1.
1019
+ else if (ab == 0)
1020
+ return 1;
1021
+ // We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
1022
+ // https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
1023
+ // The latency of the native instruction is 4 cycles and it's broadly supported.
1024
+ // For single-precision floats it has a maximum relative error of 1.5*2^-12.
1025
+ // Higher precision isn't implemented on older CPUs. See `_simsimd_cos_normalize_f64_skylake` for that.
1026
+ __m128d squares = _mm_set_pd(a2, b2);
1027
+ __m128d rsqrts = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(squares)));
1028
+ // Newton-Raphson iteration for reciprocal square root:
1029
+ // https://en.wikipedia.org/wiki/Newton%27s_method
1030
+ rsqrts = _mm_add_pd( //
1031
+ _mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
1032
+ _mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
1033
+
1034
+ simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
1035
+ simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
1036
+ simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
1037
+ return result > 0 ? result : 0;
1038
+ }
1039
+
1040
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f32_t ab, simsimd_f32_t a2,
1041
+ simsimd_f32_t b2) {
744
1042
 
745
- #if SIMSIMD_TARGET_X86
1043
+ // If both vectors have magnitude 0, the distance is 0.
1044
+ if (a2 == 0.0f && b2 == 0.0f) return 0.0f;
1045
+ // If any one of the vectors is 0, the square root of the product is 0,
1046
+ // the division is illformed, and the result is 1.
1047
+ else if (ab == 0.0f)
1048
+ return 1.0f;
1049
+
1050
+ // Load the squares into an __m128 register for single-precision floating-point operations
1051
+ __m128 squares = _mm_set_ps(a2, b2, a2, b2); // We replicate to make use of full register
1052
+
1053
+ // Compute the reciprocal square root of the squares using _mm_rsqrt_ps (single-precision)
1054
+ __m128 rsqrts = _mm_rsqrt_ps(squares);
1055
+
1056
+ // Perform one iteration of Newton-Raphson refinement to improve the precision of rsqrt:
1057
+ // Formula: y' = y * (1.5 - 0.5 * x * y * y)
1058
+ __m128 half = _mm_set1_ps(0.5f);
1059
+ __m128 three_halves = _mm_set1_ps(1.5f);
1060
+ rsqrts =
1061
+ _mm_mul_ps(rsqrts, _mm_sub_ps(three_halves, _mm_mul_ps(half, _mm_mul_ps(squares, _mm_mul_ps(rsqrts, rsqrts)))));
1062
+
1063
+ // Extract the reciprocal square roots of a2 and b2 from the __m128 register
1064
+ simsimd_f32_t a2_reciprocal = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1065
+ simsimd_f32_t b2_reciprocal = _mm_cvtss_f32(rsqrts);
1066
+
1067
+ // Calculate the cosine distance: 1 - ab * a2_reciprocal * b2_reciprocal
1068
+ simsimd_distance_t result = 1.0f - ab * a2_reciprocal * b2_reciprocal;
1069
+ return result > 0 ? result : 0;
1070
+ }
1071
+
1072
+ #pragma clang attribute pop
1073
+ #pragma GCC pop_options
1074
+ #endif // SIMSIMD_TARGET_HASWELL
1075
+ #endif // _SIMSIMD_TARGET_X86
1076
+
1077
+ #if _SIMSIMD_TARGET_X86
746
1078
  #if SIMSIMD_TARGET_HASWELL
747
1079
  #pragma GCC push_options
748
1080
  #pragma GCC target("avx2", "f16c", "fma")
749
1081
  #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
750
1082
 
751
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
752
- simsimd_distance_t* result) {
1083
+ SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1084
+ simsimd_distance_t *result) {
1085
+ simsimd_l2sq_f16_haswell(a, b, n, result);
1086
+ *result = _simsimd_sqrt_f32_haswell(*result);
1087
+ }
1088
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1089
+ simsimd_distance_t *result) {
1090
+ __m256 a_vec, b_vec;
753
1091
  __m256 d2_vec = _mm256_setzero_ps();
754
- simsimd_size_t i = 0;
755
- for (; i + 8 <= n; i += 8) {
756
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
757
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
758
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
759
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
760
- }
761
1092
 
762
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
763
- // function will run. It is extremely slow, so even for the tail, let's combine serial
764
- // loads and stores with vectorized math.
765
- if (i < n) {
766
- union {
767
- __m128i f16_vec;
768
- simsimd_f16_t f16[8];
769
- } a_padded_tail, b_padded_tail;
770
- simsimd_size_t j = 0;
771
- for (; i < n; ++i, ++j)
772
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
773
- for (; j < 8; ++j)
774
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
775
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
776
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
777
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
778
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1093
+ simsimd_l2sq_f16_haswell_cycle:
1094
+ if (n < 8) {
1095
+ a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1096
+ b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1097
+ n = 0;
779
1098
  }
1099
+ else {
1100
+ a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
1101
+ b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
1102
+ n -= 8, a += 8, b += 8;
1103
+ }
1104
+ __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1105
+ d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1106
+ if (n) goto simsimd_l2sq_f16_haswell_cycle;
780
1107
 
781
- d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
782
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
783
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
784
-
785
- simsimd_f32_t f32_result;
786
- _mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
787
- *result = f32_result;
1108
+ *result = _simsimd_reduce_f32x8_haswell(d2_vec);
788
1109
  }
789
1110
 
790
- SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
791
- simsimd_distance_t* result) {
792
-
1111
+ SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1112
+ simsimd_distance_t *result) {
1113
+ __m256 a_vec, b_vec;
793
1114
  __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
794
- simsimd_size_t i = 0;
795
- for (; i + 8 <= n; i += 8) {
796
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
797
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
798
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
799
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
800
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
801
- }
802
1115
 
803
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
804
- // function will run. It is extremely slow, so even for the tail, let's combine serial
805
- // loads and stores with vectorized math.
806
- if (i < n) {
807
- union {
808
- __m128i f16_vec;
809
- simsimd_f16_t f16[8];
810
- } a_padded_tail, b_padded_tail;
811
- simsimd_size_t j = 0;
812
- for (; i < n; ++i, ++j)
813
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
814
- for (; j < 8; ++j)
815
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
816
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
817
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
818
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
819
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
820
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1116
+ simsimd_cos_f16_haswell_cycle:
1117
+ if (n < 8) {
1118
+ a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1119
+ b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1120
+ n = 0;
1121
+ }
1122
+ else {
1123
+ a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
1124
+ b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
1125
+ n -= 8, a += 8, b += 8;
821
1126
  }
1127
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1128
+ a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1129
+ b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1130
+ if (n) goto simsimd_cos_f16_haswell_cycle;
1131
+
1132
+ simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1133
+ simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1134
+ simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1135
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1136
+ }
822
1137
 
823
- // Horizontal reductions:
824
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
825
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
826
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
1138
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1139
+ simsimd_distance_t *result) {
1140
+ simsimd_l2sq_bf16_haswell(a, b, n, result);
1141
+ *result = _simsimd_sqrt_f32_haswell(*result);
1142
+ }
1143
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1144
+ simsimd_distance_t *result) {
1145
+ __m256 a_vec, b_vec;
1146
+ __m256 d2_vec = _mm256_setzero_ps();
827
1147
 
828
- a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
829
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
830
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
1148
+ simsimd_l2sq_bf16_haswell_cycle:
1149
+ if (n < 8) {
1150
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1151
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1152
+ n = 0;
1153
+ }
1154
+ else {
1155
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
1156
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
1157
+ n -= 8, a += 8, b += 8;
1158
+ }
1159
+ __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1160
+ d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1161
+ if (n) goto simsimd_l2sq_bf16_haswell_cycle;
831
1162
 
832
- b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
833
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
834
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
1163
+ *result = _simsimd_reduce_f32x8_haswell(d2_vec);
1164
+ }
835
1165
 
836
- simsimd_f32_t ab, a2, b2;
837
- _mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
838
- _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
839
- _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
1166
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1167
+ simsimd_distance_t *result) {
1168
+ __m256 a_vec, b_vec;
1169
+ __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
840
1170
 
841
- // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
842
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
843
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
844
- __m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
845
- result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
846
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1171
+ simsimd_cos_bf16_haswell_cycle:
1172
+ if (n < 8) {
1173
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1174
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1175
+ n = 0;
1176
+ }
1177
+ else {
1178
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
1179
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
1180
+ n -= 8, a += 8, b += 8;
1181
+ }
1182
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1183
+ a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1184
+ b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1185
+ if (n) goto simsimd_cos_bf16_haswell_cycle;
1186
+
1187
+ simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1188
+ simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1189
+ simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1190
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
847
1191
  }
848
1192
 
849
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
850
- simsimd_distance_t* result) {
851
- __m256 d2_vec = _mm256_setzero_ps();
1193
+ SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1194
+ simsimd_distance_t *result) {
1195
+ simsimd_l2sq_i8_haswell(a, b, n, result);
1196
+ *result = _simsimd_sqrt_f32_haswell(*result);
1197
+ }
1198
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1199
+ simsimd_distance_t *result) {
1200
+
1201
+ __m256i d2_i32_low_vec = _mm256_setzero_si256();
1202
+ __m256i d2_i32_high_vec = _mm256_setzero_si256();
1203
+
852
1204
  simsimd_size_t i = 0;
853
- for (; i + 8 <= n; i += 8) {
854
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
855
- // x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
856
- __m256 a_vec =
857
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
858
- __m256 b_vec =
859
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
860
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
861
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1205
+ for (; i + 32 <= n; i += 32) {
1206
+ __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1207
+ __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1208
+
1209
+ // Sign extend `i8` to `i16`
1210
+ __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_i8_vec));
1211
+ __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1212
+ __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_i8_vec));
1213
+ __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1214
+
1215
+ // Subtract
1216
+ // After this we will be squaring the values. The sign will be dropped
1217
+ // and each difference will be in the range [0, 255].
1218
+ __m256i d_i16_low_vec = _mm256_sub_epi16(a_i16_low_vec, b_i16_low_vec);
1219
+ __m256i d_i16_high_vec = _mm256_sub_epi16(a_i16_high_vec, b_i16_high_vec);
1220
+
1221
+ // Accumulate into `i32` vectors
1222
+ d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
1223
+ d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
862
1224
  }
863
1225
 
864
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
865
- // function will run. It is extremely slow, so even for the tail, let's combine serial
866
- // loads and stores with vectorized math.
867
- if (i < n) {
868
- union {
869
- __m128i bf16_vec;
870
- simsimd_bf16_t bf16[8];
871
- } a_padded_tail, b_padded_tail;
872
- simsimd_size_t j = 0;
873
- for (; i < n; ++i, ++j)
874
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
875
- for (; j < 8; ++j)
876
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
877
- __m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
878
- __m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
879
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
880
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
881
- }
1226
+ // Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
1227
+ int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
882
1228
 
883
- d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
884
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
885
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
1229
+ // Take care of the tail:
1230
+ for (; i < n; ++i) {
1231
+ int n = (int)(a[i]) - b[i];
1232
+ d2 += n * n;
1233
+ }
886
1234
 
887
- simsimd_f32_t f32_result;
888
- _mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
889
- *result = f32_result;
1235
+ *result = (simsimd_f64_t)d2;
890
1236
  }
891
1237
 
892
- SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
893
- simsimd_distance_t* result) {
1238
+ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1239
+ simsimd_distance_t *result) {
894
1240
 
895
- __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
1241
+ __m256i ab_i32_low_vec = _mm256_setzero_si256();
1242
+ __m256i ab_i32_high_vec = _mm256_setzero_si256();
1243
+ __m256i a2_i32_low_vec = _mm256_setzero_si256();
1244
+ __m256i a2_i32_high_vec = _mm256_setzero_si256();
1245
+ __m256i b2_i32_low_vec = _mm256_setzero_si256();
1246
+ __m256i b2_i32_high_vec = _mm256_setzero_si256();
1247
+
1248
+ // AVX2 has no instructions for 8-bit signed integer dot-products,
1249
+ // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1250
+ // So we need to normalize the first vector to its absolute value,
1251
+ // and shift the product sign into the second vector.
1252
+ //
1253
+ // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1254
+ // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1255
+ // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1256
+ //
1257
+ // The problem with this approach, however, is the `-128` value in the second vector.
1258
+ // Flipping it's sign will do nothing, and the result will be incorrect.
1259
+ // This can easily lead to noticeable numerical errors in the final result.
896
1260
  simsimd_size_t i = 0;
897
- for (; i + 8 <= n; i += 8) {
898
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
899
- // x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
900
- __m256 a_vec =
901
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
902
- __m256 b_vec =
903
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
904
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
905
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
906
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
907
- }
1261
+ for (; i + 32 <= n; i += 32) {
1262
+ __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1263
+ __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
908
1264
 
909
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
910
- // function will run. It is extremely slow, so even for the tail, let's combine serial
911
- // loads and stores with vectorized math.
912
- if (i < n) {
913
- union {
914
- __m128i bf16_vec;
915
- simsimd_bf16_t bf16[8];
916
- } a_padded_tail, b_padded_tail;
917
- simsimd_size_t j = 0;
918
- for (; i < n; ++i, ++j)
919
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
920
- for (; j < 8; ++j)
921
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
922
- __m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
923
- __m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
924
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
925
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
926
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1265
+ // Unpack `int8` to `int16`
1266
+ __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
1267
+ __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1268
+ __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
1269
+ __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1270
+
1271
+ // Multiply and accumulate as `int16`, accumulate products as `int32`:
1272
+ ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1273
+ ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1274
+ a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
1275
+ a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
1276
+ b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
1277
+ b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
927
1278
  }
928
1279
 
929
- // Horizontal reductions:
930
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
931
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
932
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
933
-
934
- a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
935
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
936
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
937
-
938
- b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
939
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
940
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
1280
+ // Further reduce to a single sum for each vector
1281
+ int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1282
+ int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
1283
+ int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
941
1284
 
942
- simsimd_f32_t ab, a2, b2;
943
- _mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
944
- _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
945
- _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
1285
+ // Take care of the tail:
1286
+ for (; i < n; ++i) {
1287
+ int ai = a[i], bi = b[i];
1288
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1289
+ }
946
1290
 
947
- // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
948
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
949
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
950
- __m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
951
- result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
952
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1291
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
953
1292
  }
954
1293
 
955
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
956
- simsimd_distance_t* result) {
1294
+ SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1295
+ simsimd_distance_t *result) {
1296
+ simsimd_l2sq_u8_haswell(a, b, n, result);
1297
+ *result = _simsimd_sqrt_f32_haswell(*result);
1298
+ }
1299
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1300
+ simsimd_distance_t *result) {
957
1301
 
958
- __m256i d2_high_vec = _mm256_setzero_si256();
959
- __m256i d2_low_vec = _mm256_setzero_si256();
1302
+ __m256i d2_i32_low_vec = _mm256_setzero_si256();
1303
+ __m256i d2_i32_high_vec = _mm256_setzero_si256();
1304
+ __m256i const zeros_vec = _mm256_setzero_si256();
960
1305
 
961
1306
  simsimd_size_t i = 0;
962
1307
  for (; i + 32 <= n; i += 32) {
963
- __m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
964
- __m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
1308
+ __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1309
+ __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
965
1310
 
966
- // Sign extend int8 to int16
967
- __m256i a_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_vec));
968
- __m256i a_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
969
- __m256i b_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_vec));
970
- __m256i b_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
1311
+ // Substracting unsigned vectors in AVX2 is done by saturating subtraction:
1312
+ __m256i d_u8_vec = _mm256_or_si256(_mm256_subs_epu8(a_u8_vec, b_u8_vec), _mm256_subs_epu8(b_u8_vec, a_u8_vec));
971
1313
 
972
- // Subtract and multiply
973
- __m256i d_low = _mm256_sub_epi16(a_low, b_low);
974
- __m256i d_high = _mm256_sub_epi16(a_high, b_high);
975
- __m256i d2_low_part = _mm256_madd_epi16(d_low, d_low);
976
- __m256i d2_high_part = _mm256_madd_epi16(d_high, d_high);
1314
+ // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1315
+ // instructions instead of extracts, as they are much faster and more efficient.
1316
+ __m256i d_i16_low_vec = _mm256_unpacklo_epi8(d_u8_vec, zeros_vec);
1317
+ __m256i d_i16_high_vec = _mm256_unpackhi_epi8(d_u8_vec, zeros_vec);
977
1318
 
978
- // Accumulate into int32 vectors
979
- d2_low_vec = _mm256_add_epi32(d2_low_vec, d2_low_part);
980
- d2_high_vec = _mm256_add_epi32(d2_high_vec, d2_high_part);
1319
+ // Multiply and accumulate at `int16` level, accumulate at `int32` level:
1320
+ d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
1321
+ d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
981
1322
  }
982
1323
 
983
- // Accumulate the 32-bit integers from `d2_high_vec` and `d2_low_vec`
984
- __m256i d2_vec = _mm256_add_epi32(d2_low_vec, d2_high_vec);
985
- __m128i d2_sum = _mm_add_epi32(_mm256_extracti128_si256(d2_vec, 0), _mm256_extracti128_si256(d2_vec, 1));
986
- d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
987
- d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
988
- int d2 = _mm_extract_epi32(d2_sum, 0);
1324
+ // Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
1325
+ int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
989
1326
 
990
1327
  // Take care of the tail:
991
1328
  for (; i < n; ++i) {
992
- int n = a[i] - b[i];
1329
+ int n = (int)(a[i]) - b[i];
993
1330
  d2 += n * n;
994
1331
  }
995
1332
 
996
1333
  *result = (simsimd_f64_t)d2;
997
1334
  }
998
1335
 
999
- SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1000
- simsimd_distance_t* result) {
1001
-
1002
- __m256i ab_low_vec = _mm256_setzero_si256();
1003
- __m256i ab_high_vec = _mm256_setzero_si256();
1004
- __m256i a2_low_vec = _mm256_setzero_si256();
1005
- __m256i a2_high_vec = _mm256_setzero_si256();
1006
- __m256i b2_low_vec = _mm256_setzero_si256();
1007
- __m256i b2_high_vec = _mm256_setzero_si256();
1008
-
1336
+ SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1337
+ simsimd_distance_t *result) {
1338
+
1339
+ __m256i ab_i32_low_vec = _mm256_setzero_si256();
1340
+ __m256i ab_i32_high_vec = _mm256_setzero_si256();
1341
+ __m256i a2_i32_low_vec = _mm256_setzero_si256();
1342
+ __m256i a2_i32_high_vec = _mm256_setzero_si256();
1343
+ __m256i b2_i32_low_vec = _mm256_setzero_si256();
1344
+ __m256i b2_i32_high_vec = _mm256_setzero_si256();
1345
+ __m256i const zeros_vec = _mm256_setzero_si256();
1346
+
1347
+ // AVX2 has no instructions for 8-bit signed integer dot-products,
1348
+ // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1349
+ // So we need to normalize the first vector to its absolute value,
1350
+ // and shift the product sign into the second vector.
1351
+ //
1352
+ // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1353
+ // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1354
+ // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1355
+ //
1356
+ // The problem with this approach, however, is the `-128` value in the second vector.
1357
+ // Flipping it's sign will do nothing, and the result will be incorrect.
1358
+ // This can easily lead to noticeable numerical errors in the final result.
1009
1359
  simsimd_size_t i = 0;
1010
1360
  for (; i + 32 <= n; i += 32) {
1011
- __m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
1012
- __m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
1361
+ __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1362
+ __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1013
1363
 
1014
- // Unpack `int8` to `int16`
1015
- __m256i a_low_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 0));
1016
- __m256i a_high_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
1017
- __m256i b_low_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 0));
1018
- __m256i b_high_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
1364
+ // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1365
+ // instructions instead of extracts, as they are much faster and more efficient.
1366
+ __m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
1367
+ __m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
1368
+ __m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
1369
+ __m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
1019
1370
 
1020
1371
  // Multiply and accumulate as `int16`, accumulate products as `int32`
1021
- ab_low_vec = _mm256_add_epi32(ab_low_vec, _mm256_madd_epi16(a_low_16, b_low_16));
1022
- ab_high_vec = _mm256_add_epi32(ab_high_vec, _mm256_madd_epi16(a_high_16, b_high_16));
1023
- a2_low_vec = _mm256_add_epi32(a2_low_vec, _mm256_madd_epi16(a_low_16, a_low_16));
1024
- a2_high_vec = _mm256_add_epi32(a2_high_vec, _mm256_madd_epi16(a_high_16, a_high_16));
1025
- b2_low_vec = _mm256_add_epi32(b2_low_vec, _mm256_madd_epi16(b_low_16, b_low_16));
1026
- b2_high_vec = _mm256_add_epi32(b2_high_vec, _mm256_madd_epi16(b_high_16, b_high_16));
1027
- }
1028
-
1029
- // Horizontal sum across the 256-bit register
1030
- __m256i ab_vec = _mm256_add_epi32(ab_low_vec, ab_high_vec);
1031
- __m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
1032
- ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
1033
- ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
1034
-
1035
- __m256i a2_vec = _mm256_add_epi32(a2_low_vec, a2_high_vec);
1036
- __m128i a2_sum = _mm_add_epi32(_mm256_extracti128_si256(a2_vec, 0), _mm256_extracti128_si256(a2_vec, 1));
1037
- a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
1038
- a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
1039
-
1040
- __m256i b2_vec = _mm256_add_epi32(b2_low_vec, b2_high_vec);
1041
- __m128i b2_sum = _mm_add_epi32(_mm256_extracti128_si256(b2_vec, 0), _mm256_extracti128_si256(b2_vec, 1));
1042
- b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
1043
- b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
1372
+ ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1373
+ ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1374
+ a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
1375
+ a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
1376
+ b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
1377
+ b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
1378
+ }
1044
1379
 
1045
1380
  // Further reduce to a single sum for each vector
1046
- int ab = _mm_extract_epi32(ab_sum, 0);
1047
- int a2 = _mm_extract_epi32(a2_sum, 0);
1048
- int b2 = _mm_extract_epi32(b2_sum, 0);
1381
+ int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1382
+ int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
1383
+ int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
1049
1384
 
1050
1385
  // Take care of the tail:
1051
1386
  for (; i < n; ++i) {
@@ -1053,18 +1388,16 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
1053
1388
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1054
1389
  }
1055
1390
 
1056
- // Compute the reciprocal of the square roots
1057
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
1058
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
1059
-
1060
- // Compute cosine similarity: ab / sqrt(a2 * b2)
1061
- __m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
1062
- __m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
1063
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1391
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1064
1392
  }
1065
1393
 
1066
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1067
- simsimd_distance_t* result) {
1394
+ SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1395
+ simsimd_distance_t *result) {
1396
+ simsimd_l2sq_f32_haswell(a, b, n, result);
1397
+ *result = _simsimd_sqrt_f32_haswell(*result);
1398
+ }
1399
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1400
+ simsimd_distance_t *result) {
1068
1401
 
1069
1402
  __m256 d2_vec = _mm256_setzero_ps();
1070
1403
  simsimd_size_t i = 0;
@@ -1075,7 +1408,7 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
1075
1408
  d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1076
1409
  }
1077
1410
 
1078
- simsimd_f64_t d2 = _mm256_reduce_add_ps_dbl(d2_vec);
1411
+ simsimd_f64_t d2 = _simsimd_reduce_f32x8_haswell(d2_vec);
1079
1412
  for (; i < n; ++i) {
1080
1413
  float d = a[i] - b[i];
1081
1414
  d2 += d * d;
@@ -1084,8 +1417,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
1084
1417
  *result = d2;
1085
1418
  }
1086
1419
 
1087
- SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1088
- simsimd_distance_t* result) {
1420
+ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1421
+ simsimd_distance_t *result) {
1089
1422
 
1090
1423
  __m256 ab_vec = _mm256_setzero_ps();
1091
1424
  __m256 a2_vec = _mm256_setzero_ps();
@@ -1098,22 +1431,66 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
1098
1431
  a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1099
1432
  b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1100
1433
  }
1101
- simsimd_f64_t ab = _mm256_reduce_add_ps_dbl(ab_vec);
1102
- simsimd_f64_t a2 = _mm256_reduce_add_ps_dbl(a2_vec);
1103
- simsimd_f64_t b2 = _mm256_reduce_add_ps_dbl(b2_vec);
1434
+
1435
+ simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1436
+ simsimd_f64_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1437
+ simsimd_f64_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1104
1438
  for (; i < n; ++i) {
1105
1439
  float ai = a[i], bi = b[i];
1106
1440
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1107
1441
  }
1442
+ *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1443
+ }
1108
1444
 
1109
- // Compute the reciprocal of the square roots
1110
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
1111
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
1445
+ SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1446
+ simsimd_distance_t *result) {
1447
+ simsimd_l2sq_f64_haswell(a, b, n, result);
1448
+ *result = _simsimd_sqrt_f64_haswell(*result);
1449
+ }
1450
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1451
+ simsimd_distance_t *result) {
1452
+
1453
+ __m256d d2_vec = _mm256_setzero_pd();
1454
+ simsimd_size_t i = 0;
1455
+ for (; i + 4 <= n; i += 4) {
1456
+ __m256d a_vec = _mm256_loadu_pd(a + i);
1457
+ __m256d b_vec = _mm256_loadu_pd(b + i);
1458
+ __m256d d_vec = _mm256_sub_pd(a_vec, b_vec);
1459
+ d2_vec = _mm256_fmadd_pd(d_vec, d_vec, d2_vec);
1460
+ }
1112
1461
 
1113
- // Compute cosine similarity: ab / sqrt(a2 * b2)
1114
- __m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
1115
- __m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
1116
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1462
+ simsimd_f64_t d2 = _simsimd_reduce_f64x4_haswell(d2_vec);
1463
+ for (; i < n; ++i) {
1464
+ simsimd_f64_t d = a[i] - b[i];
1465
+ d2 += d * d;
1466
+ }
1467
+
1468
+ *result = d2;
1469
+ }
1470
+
1471
+ SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1472
+ simsimd_distance_t *result) {
1473
+
1474
+ __m256d ab_vec = _mm256_setzero_pd();
1475
+ __m256d a2_vec = _mm256_setzero_pd();
1476
+ __m256d b2_vec = _mm256_setzero_pd();
1477
+ simsimd_size_t i = 0;
1478
+ for (; i + 4 <= n; i += 4) {
1479
+ __m256d a_vec = _mm256_loadu_pd(a + i);
1480
+ __m256d b_vec = _mm256_loadu_pd(b + i);
1481
+ ab_vec = _mm256_fmadd_pd(a_vec, b_vec, ab_vec);
1482
+ a2_vec = _mm256_fmadd_pd(a_vec, a_vec, a2_vec);
1483
+ b2_vec = _mm256_fmadd_pd(b_vec, b_vec, b2_vec);
1484
+ }
1485
+
1486
+ simsimd_f64_t ab = _simsimd_reduce_f64x4_haswell(ab_vec);
1487
+ simsimd_f64_t a2 = _simsimd_reduce_f64x4_haswell(a2_vec);
1488
+ simsimd_f64_t b2 = _simsimd_reduce_f64x4_haswell(b2_vec);
1489
+ for (; i < n; ++i) {
1490
+ simsimd_f64_t ai = a[i], bi = b[i];
1491
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1492
+ }
1493
+ *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1117
1494
  }
1118
1495
 
1119
1496
  #pragma clang attribute pop
@@ -1122,11 +1499,16 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
1122
1499
 
1123
1500
  #if SIMSIMD_TARGET_SKYLAKE
1124
1501
  #pragma GCC push_options
1125
- #pragma GCC target("avx512f", "avx512vl", "bmi2")
1126
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2"))), apply_to = function)
1502
+ #pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
1503
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
1127
1504
 
1128
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1129
- simsimd_distance_t* result) {
1505
+ SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1506
+ simsimd_distance_t *result) {
1507
+ simsimd_l2sq_f32_skylake(a, b, n, result);
1508
+ *result = _simsimd_sqrt_f64_haswell(*result);
1509
+ }
1510
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1511
+ simsimd_distance_t *result) {
1130
1512
  __m512 d2_vec = _mm512_setzero();
1131
1513
  __m512 a_vec, b_vec;
1132
1514
 
@@ -1136,21 +1518,64 @@ simsimd_l2sq_f32_skylake_cycle:
1136
1518
  a_vec = _mm512_maskz_loadu_ps(mask, a);
1137
1519
  b_vec = _mm512_maskz_loadu_ps(mask, b);
1138
1520
  n = 0;
1139
- } else {
1521
+ }
1522
+ else {
1140
1523
  a_vec = _mm512_loadu_ps(a);
1141
1524
  b_vec = _mm512_loadu_ps(b);
1142
1525
  a += 16, b += 16, n -= 16;
1143
1526
  }
1144
1527
  __m512 d_vec = _mm512_sub_ps(a_vec, b_vec);
1145
1528
  d2_vec = _mm512_fmadd_ps(d_vec, d_vec, d2_vec);
1146
- if (n)
1147
- goto simsimd_l2sq_f32_skylake_cycle;
1529
+ if (n) goto simsimd_l2sq_f32_skylake_cycle;
1530
+
1531
+ *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1532
+ }
1148
1533
 
1149
- *result = _mm512_reduce_add_ps(d2_vec);
1534
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f64_t ab, simsimd_f64_t a2,
1535
+ simsimd_f64_t b2) {
1536
+
1537
+ // If both vectors have magnitude 0, the distance is 0.
1538
+ if (a2 == 0 && b2 == 0) return 0;
1539
+ // If any one of the vectors is 0, the square root of the product is 0,
1540
+ // the division is illformed, and the result is 1.
1541
+ else if (ab == 0)
1542
+ return 1;
1543
+
1544
+ // We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
1545
+ // https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
1546
+ // The maximum relative error for this approximation is less than 2^-14, which is 6x lower than
1547
+ // for single-precision floats in the `_simsimd_cos_normalize_f64_haswell` implementation.
1548
+ // Mysteriously, MSVC has no `_mm_rsqrt14_pd` intrinsic, but has it's masked variants,
1549
+ // so let's use `_mm_maskz_rsqrt14_pd(0xFF, ...)` instead.
1550
+ __m128d squares = _mm_set_pd(a2, b2);
1551
+ __m128d rsqrts = _mm_maskz_rsqrt14_pd(0xFF, squares);
1552
+
1553
+ // Let's implement a single Newton-Raphson iteration to refine the result.
1554
+ // This is how it affects downstream applications:
1555
+ //
1556
+ // +--------+------+----------+---------------------+---------------------+---------------------+
1557
+ // | Metric | NDim | DType | Baseline Error | Old SimSIMD Error | New SimSIMD Error |
1558
+ // +--------+------+----------+---------------------+---------------------+---------------------+
1559
+ // | cosine | 1536 | bfloat16 | 1.89e-08 ± 1.59e-08 | 3.07e-07 ± 3.09e-07 | 3.53e-09 ± 2.70e-09 |
1560
+ // | cosine | 1536 | float16 | 1.67e-02 ± 1.44e-02 | 2.68e-05 ± 1.95e-05 | 2.02e-05 ± 1.39e-05 |
1561
+ // | cosine | 1536 | float32 | 2.21e-08 ± 1.65e-08 | 3.47e-07 ± 3.49e-07 | 3.77e-09 ± 2.84e-09 |
1562
+ // | cosine | 1536 | float64 | 0.00e+00 ± 0.00e+00 | 3.80e-07 ± 4.50e-07 | 1.35e-11 ± 1.85e-11 |
1563
+ // | cosine | 1536 | int8 | 0.00e+00 ± 0.00e+00 | 4.60e-04 ± 3.36e-04 | 4.20e-04 ± 4.88e-04 |
1564
+ // +--------+------+----------+---------------------+---------------------+---------------------+
1565
+ //
1566
+ // https://en.wikipedia.org/wiki/Newton%27s_method
1567
+ rsqrts = _mm_add_pd( //
1568
+ _mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
1569
+ _mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
1570
+
1571
+ simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
1572
+ simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
1573
+ simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
1574
+ return result > 0 ? result : 0;
1150
1575
  }
1151
1576
 
1152
- SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1153
- simsimd_distance_t* result) {
1577
+ SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1578
+ simsimd_distance_t *result) {
1154
1579
  __m512 ab_vec = _mm512_setzero();
1155
1580
  __m512 a2_vec = _mm512_setzero();
1156
1581
  __m512 b2_vec = _mm512_setzero();
@@ -1162,7 +1587,8 @@ simsimd_cos_f32_skylake_cycle:
1162
1587
  a_vec = _mm512_maskz_loadu_ps(mask, a);
1163
1588
  b_vec = _mm512_maskz_loadu_ps(mask, b);
1164
1589
  n = 0;
1165
- } else {
1590
+ }
1591
+ else {
1166
1592
  a_vec = _mm512_loadu_ps(a);
1167
1593
  b_vec = _mm512_loadu_ps(b);
1168
1594
  a += 16, b += 16, n -= 16;
@@ -1170,24 +1596,21 @@ simsimd_cos_f32_skylake_cycle:
1170
1596
  ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
1171
1597
  a2_vec = _mm512_fmadd_ps(a_vec, a_vec, a2_vec);
1172
1598
  b2_vec = _mm512_fmadd_ps(b_vec, b_vec, b2_vec);
1173
- if (n)
1174
- goto simsimd_cos_f32_skylake_cycle;
1599
+ if (n) goto simsimd_cos_f32_skylake_cycle;
1175
1600
 
1176
- simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
1177
- simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
1178
- simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
1179
-
1180
- // Compute the reciprocal square roots of a2 and b2
1181
- // Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
1182
- // so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
1183
- __m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1184
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1185
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1186
- *result = 1 - ab * rsqrt_a2 * rsqrt_b2;
1601
+ simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1602
+ simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
1603
+ simsimd_f64_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
1604
+ *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1187
1605
  }
1188
1606
 
1189
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1190
- simsimd_distance_t* result) {
1607
+ SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1608
+ simsimd_distance_t *result) {
1609
+ simsimd_l2sq_f64_skylake(a, b, n, result);
1610
+ *result = _simsimd_sqrt_f64_haswell(*result);
1611
+ }
1612
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1613
+ simsimd_distance_t *result) {
1191
1614
  __m512d d2_vec = _mm512_setzero_pd();
1192
1615
  __m512d a_vec, b_vec;
1193
1616
 
@@ -1197,21 +1620,21 @@ simsimd_l2sq_f64_skylake_cycle:
1197
1620
  a_vec = _mm512_maskz_loadu_pd(mask, a);
1198
1621
  b_vec = _mm512_maskz_loadu_pd(mask, b);
1199
1622
  n = 0;
1200
- } else {
1623
+ }
1624
+ else {
1201
1625
  a_vec = _mm512_loadu_pd(a);
1202
1626
  b_vec = _mm512_loadu_pd(b);
1203
1627
  a += 8, b += 8, n -= 8;
1204
1628
  }
1205
1629
  __m512d d_vec = _mm512_sub_pd(a_vec, b_vec);
1206
1630
  d2_vec = _mm512_fmadd_pd(d_vec, d_vec, d2_vec);
1207
- if (n)
1208
- goto simsimd_l2sq_f64_skylake_cycle;
1631
+ if (n) goto simsimd_l2sq_f64_skylake_cycle;
1209
1632
 
1210
1633
  *result = _mm512_reduce_add_pd(d2_vec);
1211
1634
  }
1212
1635
 
1213
- SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1214
- simsimd_distance_t* result) {
1636
+ SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1637
+ simsimd_distance_t *result) {
1215
1638
  __m512d ab_vec = _mm512_setzero_pd();
1216
1639
  __m512d a2_vec = _mm512_setzero_pd();
1217
1640
  __m512d b2_vec = _mm512_setzero_pd();
@@ -1223,7 +1646,8 @@ simsimd_cos_f64_skylake_cycle:
1223
1646
  a_vec = _mm512_maskz_loadu_pd(mask, a);
1224
1647
  b_vec = _mm512_maskz_loadu_pd(mask, b);
1225
1648
  n = 0;
1226
- } else {
1649
+ }
1650
+ else {
1227
1651
  a_vec = _mm512_loadu_pd(a);
1228
1652
  b_vec = _mm512_loadu_pd(b);
1229
1653
  a += 8, b += 8, n -= 8;
@@ -1231,20 +1655,12 @@ simsimd_cos_f64_skylake_cycle:
1231
1655
  ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
1232
1656
  a2_vec = _mm512_fmadd_pd(a_vec, a_vec, a2_vec);
1233
1657
  b2_vec = _mm512_fmadd_pd(b_vec, b_vec, b2_vec);
1234
- if (n)
1235
- goto simsimd_cos_f64_skylake_cycle;
1236
-
1237
- simsimd_f32_t ab = (simsimd_f32_t)_mm512_reduce_add_pd(ab_vec);
1238
- simsimd_f32_t a2 = (simsimd_f32_t)_mm512_reduce_add_pd(a2_vec);
1239
- simsimd_f32_t b2 = (simsimd_f32_t)_mm512_reduce_add_pd(b2_vec);
1658
+ if (n) goto simsimd_cos_f64_skylake_cycle;
1240
1659
 
1241
- // Compute the reciprocal square roots of a2 and b2
1242
- // Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
1243
- // so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
1244
- __m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1245
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1246
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1247
- *result = 1 - ab * rsqrt_a2 * rsqrt_b2;
1660
+ simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
1661
+ simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
1662
+ simsimd_f64_t b2 = _mm512_reduce_add_pd(b2_vec);
1663
+ *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1248
1664
  }
1249
1665
 
1250
1666
  #pragma clang attribute pop
@@ -1253,15 +1669,71 @@ simsimd_cos_f64_skylake_cycle:
1253
1669
 
1254
1670
  #if SIMSIMD_TARGET_GENOA
1255
1671
  #pragma GCC push_options
1256
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1257
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), apply_to = function)
1258
-
1259
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1260
- simsimd_distance_t* result) {
1261
- __m512 d2_top_vec = _mm512_setzero_ps(), d2_bot_vec = _mm512_setzero_ps();
1262
- __m512 d_top_vec = _mm512_setzero_ps(), d_bot_vec = _mm512_setzero_ps();
1263
- __m512 a_f32_top_vec, a_f32_bot_vec, b_f32_top_vec, b_f32_bot_vec;
1264
- __m512i a_i16_vec, b_i16_vec;
1672
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1673
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1674
+ apply_to = function)
1675
+
1676
+ SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
1677
+
1678
+ union {
1679
+ __m512 fvec;
1680
+ __m512i ivec;
1681
+ simsimd_f32_t f32[16];
1682
+ simsimd_u16_t u16[32];
1683
+ simsimd_bf16_t bf16[32];
1684
+ } d_odd, d_even, d, a_f32_even, b_f32_even, d_f32_even, a_f32_odd, b_f32_odd, d_f32_odd, a, b;
1685
+ a.ivec = a_i16;
1686
+ b.ivec = b_i16;
1687
+
1688
+ // There are several approaches to perform subtraction in `bf16`. The first one is:
1689
+ //
1690
+ // Perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
1691
+ // expand it to 32-bit integers, then shift the bits by 16 to the left.
1692
+ // Then subtract as floats, and shift back. During expansion, we will double the space,
1693
+ // and should use separate registers for top and bottom halves.
1694
+ // Some compilers don't have `_mm512_extracti32x8_epi32`, so we use `_mm512_extracti64x4_epi64`:
1695
+ //
1696
+ // a_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
1697
+ // _mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16)), 16));
1698
+ // b_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
1699
+ // _mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16)), 16));
1700
+ // a_f32_top.fvec =_mm512_castsi512_ps(
1701
+ // _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16, 1)), 16));
1702
+ // b_f32_top.fvec =_mm512_castsi512_ps(
1703
+ // _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16, 1)), 16));
1704
+ // d_f32_top.fvec = _mm512_sub_ps(a_f32_top.fvec, b_f32_top.fvec);
1705
+ // d_f32_bot.fvec = _mm512_sub_ps(a_f32_bot.fvec, b_f32_bot.fvec);
1706
+ // d.ivec = _mm512_castsi256_si512(_mm512_cvtepi32_epi16(
1707
+ // _mm512_srli_epi32(_mm512_castps_si512(d_f32_bot.fvec), 16)));
1708
+ // d.ivec = _mm512_inserti64x4(d.ivec, _mm512_cvtepi32_epi16(
1709
+ // _mm512_srli_epi32(_mm512_castps_si512(d_f32_top.fvec), 16)), 1);
1710
+ //
1711
+ // Instead of using multple shifts and an insertion, we can achieve similar result with fewer expensive
1712
+ // calls to `_mm512_permutex2var_epi16`, or a cheap `_mm512_mask_shuffle_epi8` and blend:
1713
+ //
1714
+ a_f32_odd.ivec = _mm512_and_si512(a_i16, _mm512_set1_epi32(0xFFFF0000));
1715
+ a_f32_even.ivec = _mm512_slli_epi32(a_i16, 16);
1716
+ b_f32_odd.ivec = _mm512_and_si512(b_i16, _mm512_set1_epi32(0xFFFF0000));
1717
+ b_f32_even.ivec = _mm512_slli_epi32(b_i16, 16);
1718
+
1719
+ d_f32_odd.fvec = _mm512_sub_ps(a_f32_odd.fvec, b_f32_odd.fvec);
1720
+ d_f32_even.fvec = _mm512_sub_ps(a_f32_even.fvec, b_f32_even.fvec);
1721
+
1722
+ d_f32_even.ivec = _mm512_srli_epi32(d_f32_even.ivec, 16);
1723
+ d.ivec = _mm512_mask_blend_epi16(0x55555555, d_f32_odd.ivec, d_f32_even.ivec);
1724
+
1725
+ return d.ivec;
1726
+ }
1727
+
1728
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1729
+ simsimd_distance_t *result) {
1730
+ simsimd_l2sq_bf16_genoa(a, b, n, result);
1731
+ *result = _simsimd_sqrt_f32_haswell(*result);
1732
+ }
1733
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1734
+ simsimd_distance_t *result) {
1735
+ __m512 d2_vec = _mm512_setzero_ps();
1736
+ __m512i a_i16_vec, b_i16_vec, d_i16_vec;
1265
1737
 
1266
1738
  simsimd_l2sq_bf16_genoa_cycle:
1267
1739
  if (n < 32) {
@@ -1269,44 +1741,21 @@ simsimd_l2sq_bf16_genoa_cycle:
1269
1741
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1270
1742
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1271
1743
  n = 0;
1272
- } else {
1744
+ }
1745
+ else {
1273
1746
  a_i16_vec = _mm512_loadu_epi16(a);
1274
1747
  b_i16_vec = _mm512_loadu_epi16(b);
1275
1748
  a += 32, b += 32, n -= 32;
1276
1749
  }
1277
- // Let's perform the subtraction with single-precision, while the dot-product with half-precision.
1278
- // For that we need to perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
1279
- // expand it to 32-bit integers, then shift the bits by 16 to the left. Then subtract as floats,
1280
- // and shift back. During expansion, we will double the space, and should use separate registers
1281
- // for top and bottom halves.
1282
- a_f32_bot_vec =
1283
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16_vec)), 16));
1284
- b_f32_bot_vec =
1285
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16_vec)), 16));
1286
-
1287
- // Some compilers don't have `_mm512_extracti32x8_epi32`, so we need to use `_mm512_extracti64x4_epi64`
1288
- a_f32_top_vec =
1289
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16_vec, 1)), 16));
1290
- b_f32_top_vec =
1291
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16_vec, 1)), 16));
1292
-
1293
- // Subtract and cast back
1294
- d_top_vec = _mm512_sub_ps(a_f32_top_vec, b_f32_top_vec);
1295
- d_bot_vec = _mm512_sub_ps(a_f32_bot_vec, b_f32_bot_vec);
1296
- d_top_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_top_vec), 16));
1297
- d_bot_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_bot_vec), 16));
1298
-
1299
- // Square and accumulate
1300
- d2_top_vec = _mm512_dpbf16_ps(d2_top_vec, (__m512bh)(d_top_vec), (__m512bh)(d_top_vec));
1301
- d2_bot_vec = _mm512_dpbf16_ps(d2_bot_vec, (__m512bh)(d_bot_vec), (__m512bh)(d_bot_vec));
1302
- if (n)
1303
- goto simsimd_l2sq_bf16_genoa_cycle;
1304
-
1305
- *result = _mm512_reduce_add_ps(d2_top_vec) + _mm512_reduce_add_ps(d2_bot_vec);
1306
- }
1307
-
1308
- SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1309
- simsimd_distance_t* result) {
1750
+ d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
1751
+ d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
1752
+ if (n) goto simsimd_l2sq_bf16_genoa_cycle;
1753
+
1754
+ *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1755
+ }
1756
+
1757
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1758
+ simsimd_distance_t *result) {
1310
1759
  __m512 ab_vec = _mm512_setzero_ps();
1311
1760
  __m512 a2_vec = _mm512_setzero_ps();
1312
1761
  __m512 b2_vec = _mm512_setzero_ps();
@@ -1318,7 +1767,8 @@ simsimd_cos_bf16_genoa_cycle:
1318
1767
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1319
1768
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1320
1769
  n = 0;
1321
- } else {
1770
+ }
1771
+ else {
1322
1772
  a_i16_vec = _mm512_loadu_epi16(a);
1323
1773
  b_i16_vec = _mm512_loadu_epi16(b);
1324
1774
  a += 32, b += 32, n -= 32;
@@ -1326,18 +1776,12 @@ simsimd_cos_bf16_genoa_cycle:
1326
1776
  ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
1327
1777
  a2_vec = _mm512_dpbf16_ps(a2_vec, (__m512bh)(a_i16_vec), (__m512bh)(a_i16_vec));
1328
1778
  b2_vec = _mm512_dpbf16_ps(b2_vec, (__m512bh)(b_i16_vec), (__m512bh)(b_i16_vec));
1329
- if (n)
1330
- goto simsimd_cos_bf16_genoa_cycle;
1331
-
1332
- simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
1333
- simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
1334
- simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
1779
+ if (n) goto simsimd_cos_bf16_genoa_cycle;
1335
1780
 
1336
- // Compute the reciprocal square roots of a2 and b2
1337
- __m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1338
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1339
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1340
- *result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
1781
+ simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1782
+ simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
1783
+ simsimd_f32_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
1784
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1341
1785
  }
1342
1786
 
1343
1787
  #pragma clang attribute pop
@@ -1346,11 +1790,16 @@ simsimd_cos_bf16_genoa_cycle:
1346
1790
 
1347
1791
  #if SIMSIMD_TARGET_SAPPHIRE
1348
1792
  #pragma GCC push_options
1349
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512fp16")
1350
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
1793
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
1794
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
1351
1795
 
1352
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1353
- simsimd_distance_t* result) {
1796
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1797
+ simsimd_distance_t *result) {
1798
+ simsimd_l2sq_f16_sapphire(a, b, n, result);
1799
+ *result = _simsimd_sqrt_f32_haswell(*result);
1800
+ }
1801
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1802
+ simsimd_distance_t *result) {
1354
1803
  __m512h d2_vec = _mm512_setzero_ph();
1355
1804
  __m512i a_i16_vec, b_i16_vec;
1356
1805
 
@@ -1360,21 +1809,21 @@ simsimd_l2sq_f16_sapphire_cycle:
1360
1809
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1361
1810
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1362
1811
  n = 0;
1363
- } else {
1812
+ }
1813
+ else {
1364
1814
  a_i16_vec = _mm512_loadu_epi16(a);
1365
1815
  b_i16_vec = _mm512_loadu_epi16(b);
1366
1816
  a += 32, b += 32, n -= 32;
1367
1817
  }
1368
1818
  __m512h d_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec));
1369
1819
  d2_vec = _mm512_fmadd_ph(d_vec, d_vec, d2_vec);
1370
- if (n)
1371
- goto simsimd_l2sq_f16_sapphire_cycle;
1820
+ if (n) goto simsimd_l2sq_f16_sapphire_cycle;
1372
1821
 
1373
1822
  *result = _mm512_reduce_add_ph(d2_vec);
1374
1823
  }
1375
1824
 
1376
- SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1377
- simsimd_distance_t* result) {
1825
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1826
+ simsimd_distance_t *result) {
1378
1827
  __m512h ab_vec = _mm512_setzero_ph();
1379
1828
  __m512h a2_vec = _mm512_setzero_ph();
1380
1829
  __m512h b2_vec = _mm512_setzero_ph();
@@ -1386,7 +1835,8 @@ simsimd_cos_f16_sapphire_cycle:
1386
1835
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1387
1836
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1388
1837
  n = 0;
1389
- } else {
1838
+ }
1839
+ else {
1390
1840
  a_i16_vec = _mm512_loadu_epi16(a);
1391
1841
  b_i16_vec = _mm512_loadu_epi16(b);
1392
1842
  a += 32, b += 32, n -= 32;
@@ -1394,18 +1844,12 @@ simsimd_cos_f16_sapphire_cycle:
1394
1844
  ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
1395
1845
  a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(a_i16_vec), a2_vec);
1396
1846
  b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_i16_vec), _mm512_castsi512_ph(b_i16_vec), b2_vec);
1397
- if (n)
1398
- goto simsimd_cos_f16_sapphire_cycle;
1847
+ if (n) goto simsimd_cos_f16_sapphire_cycle;
1399
1848
 
1400
1849
  simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
1401
1850
  simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
1402
1851
  simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
1403
-
1404
- // Compute the reciprocal square roots of a2 and b2
1405
- __m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1406
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1407
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1408
- *result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
1852
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1409
1853
  }
1410
1854
 
1411
1855
  #pragma clang attribute pop
@@ -1414,94 +1858,464 @@ simsimd_cos_f16_sapphire_cycle:
1414
1858
 
1415
1859
  #if SIMSIMD_TARGET_ICE
1416
1860
  #pragma GCC push_options
1417
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1418
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), apply_to = function)
1419
-
1420
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1421
- simsimd_distance_t* result) {
1422
- __m512i d2_i32s_vec = _mm512_setzero_si512();
1423
- __m512i a_vec, b_vec, d_i16s_vec;
1861
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1862
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1863
+ apply_to = function)
1864
+
1865
+ SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1866
+ simsimd_distance_t *result) {
1867
+ simsimd_l2sq_i8_ice(a, b, n, result);
1868
+ *result = _simsimd_sqrt_f32_haswell(*result);
1869
+ }
1870
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1871
+ simsimd_distance_t *result) {
1872
+ __m512i d2_i32_vec = _mm512_setzero_si512();
1873
+ __m512i a_i16_vec, b_i16_vec, d_i16s_vec;
1424
1874
 
1425
1875
  simsimd_l2sq_i8_ice_cycle:
1426
- if (n < 32) {
1876
+ if (n < 32) { // TODO: Avoid eaarly i16 upcast to step through 64 values at a time
1427
1877
  __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1428
- a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1429
- b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1878
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1879
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1430
1880
  n = 0;
1431
- } else {
1432
- a_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(a));
1433
- b_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(b));
1881
+ }
1882
+ else {
1883
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
1884
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
1434
1885
  a += 32, b += 32, n -= 32;
1435
1886
  }
1436
- d_i16s_vec = _mm512_sub_epi16(a_vec, b_vec);
1437
- d2_i32s_vec = _mm512_dpwssd_epi32(d2_i32s_vec, d_i16s_vec, d_i16s_vec);
1438
- if (n)
1439
- goto simsimd_l2sq_i8_ice_cycle;
1887
+ d_i16s_vec = _mm512_sub_epi16(a_i16_vec, b_i16_vec);
1888
+ d2_i32_vec = _mm512_dpwssd_epi32(d2_i32_vec, d_i16s_vec, d_i16s_vec);
1889
+ if (n) goto simsimd_l2sq_i8_ice_cycle;
1440
1890
 
1441
- *result = _mm512_reduce_add_epi32(d2_i32s_vec);
1891
+ *result = _mm512_reduce_add_epi32(d2_i32_vec);
1442
1892
  }
1443
1893
 
1444
- SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1445
- simsimd_distance_t* result) {
1446
- __m512i ab_low_i32s_vec = _mm512_setzero_si512();
1447
- __m512i ab_high_i32s_vec = _mm512_setzero_si512();
1448
- __m512i a2_i32s_vec = _mm512_setzero_si512();
1449
- __m512i b2_i32s_vec = _mm512_setzero_si512();
1450
- __m512i a_vec, b_vec;
1451
- __m512i a_abs_vec, b_abs_vec;
1894
+ SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1895
+ simsimd_distance_t *result) {
1452
1896
 
1897
+ __m512i ab_i32_vec = _mm512_setzero_si512();
1898
+ __m512i a2_i32_vec = _mm512_setzero_si512();
1899
+ __m512i b2_i32_vec = _mm512_setzero_si512();
1900
+ __m512i a_i16_vec, b_i16_vec;
1453
1901
  simsimd_cos_i8_ice_cycle:
1454
- if (n < 64) {
1455
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1456
- a_vec = _mm512_maskz_loadu_epi8(mask, a);
1457
- b_vec = _mm512_maskz_loadu_epi8(mask, b);
1902
+ if (n < 32) {
1903
+ __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1904
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1905
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1458
1906
  n = 0;
1459
- } else {
1460
- a_vec = _mm512_loadu_epi8(a);
1461
- b_vec = _mm512_loadu_epi8(b);
1462
- a += 64, b += 64, n -= 64;
1907
+ }
1908
+ else {
1909
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
1910
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
1911
+ a += 32, b += 32, n -= 32;
1463
1912
  }
1464
1913
 
1465
1914
  // We can't directly use the `_mm512_dpbusd_epi32` intrinsic everywhere,
1466
1915
  // as it's asymmetric with respect to the sign of the input arguments:
1916
+ //
1467
1917
  // Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
1468
- // Luckily to compute the squares, we just drop the sign bit of the second argument.
1469
- a_abs_vec = _mm512_abs_epi8(a_vec);
1470
- b_abs_vec = _mm512_abs_epi8(b_vec);
1471
- a2_i32s_vec = _mm512_dpbusds_epi32(a2_i32s_vec, a_abs_vec, a_abs_vec);
1472
- b2_i32s_vec = _mm512_dpbusds_epi32(b2_i32s_vec, b_abs_vec, b_abs_vec);
1918
+ //
1919
+ // To compute the squares, we could just drop the sign bit of the second argument.
1920
+ // But this would lead to big-big problems on values like `-128`!
1921
+ // For dot-products we don't have the luxury of optimizing the sign bit away.
1922
+ // Assuming this is an approximate kernel (with reciprocal square root approximations)
1923
+ // in the end, we can allow clamping the value to [-127, 127] range.
1924
+ //
1925
+ // On Ice Lake:
1926
+ //
1927
+ // 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can only execute on port 0, with 5 cycle latency.
1928
+ // 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also only execute on port 0, with 5 cycle latency.
1929
+ // 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 5, with 5 cycle latency.
1930
+ //
1931
+ // On Zen4 Genoa:
1932
+ //
1933
+ // 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 4 cycle latency.
1934
+ // 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also execute on ports 0 and 1, with 4 cycle latency.
1935
+ // 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 3 cycle latency.
1936
+ //
1937
+ // The old solution was complex replied on 1. and 2.:
1938
+ //
1939
+ // a_i8_abs_vec = _mm512_abs_epi8(a_i8_vec);
1940
+ // b_i8_abs_vec = _mm512_abs_epi8(b_i8_vec);
1941
+ // a2_i32_vec = _mm512_dpbusds_epi32(a2_i32_vec, a_i8_abs_vec, a_i8_abs_vec);
1942
+ // b2_i32_vec = _mm512_dpbusds_epi32(b2_i32_vec, b_i8_abs_vec, b_i8_abs_vec);
1943
+ // ab_i32_low_vec = _mm512_dpwssds_epi32( //
1944
+ // ab_i32_low_vec, //
1945
+ // _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)), //
1946
+ // _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1947
+ // ab_i32_high_vec = _mm512_dpwssds_epi32( //
1948
+ // ab_i32_high_vec, //
1949
+ // _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)), //
1950
+ // _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1951
+ //
1952
+ // The new solution is simpler and relies on 3.:
1953
+ ab_i32_vec = _mm512_add_epi32(ab_i32_vec, _mm512_madd_epi16(a_i16_vec, b_i16_vec));
1954
+ a2_i32_vec = _mm512_add_epi32(a2_i32_vec, _mm512_madd_epi16(a_i16_vec, a_i16_vec));
1955
+ b2_i32_vec = _mm512_add_epi32(b2_i32_vec, _mm512_madd_epi16(b_i16_vec, b_i16_vec));
1956
+ if (n) goto simsimd_cos_i8_ice_cycle;
1957
+
1958
+ int ab = _mm512_reduce_add_epi32(ab_i32_vec);
1959
+ int a2 = _mm512_reduce_add_epi32(a2_i32_vec);
1960
+ int b2 = _mm512_reduce_add_epi32(b2_i32_vec);
1961
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1962
+ }
1963
+ SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1964
+ simsimd_distance_t *result) {
1965
+ simsimd_l2sq_u8_ice(a, b, n, result);
1966
+ *result = _simsimd_sqrt_f32_haswell(*result);
1967
+ }
1968
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1969
+ simsimd_distance_t *result) {
1970
+ __m512i d2_i32_low_vec = _mm512_setzero_si512();
1971
+ __m512i d2_i32_high_vec = _mm512_setzero_si512();
1972
+ __m512i const zeros_vec = _mm512_setzero_si512();
1973
+ __m512i d_i16_low_vec, d_i16_high_vec;
1974
+ __m512i a_u8_vec, b_u8_vec, d_u8_vec;
1975
+
1976
+ simsimd_l2sq_u8_ice_cycle:
1977
+ if (n < 64) {
1978
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1979
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1980
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1981
+ n = 0;
1982
+ }
1983
+ else {
1984
+ a_u8_vec = _mm512_loadu_si512(a);
1985
+ b_u8_vec = _mm512_loadu_si512(b);
1986
+ a += 64, b += 64, n -= 64;
1987
+ }
1988
+
1989
+ // Substracting unsigned vectors in AVX-512 is done by saturating subtraction:
1990
+ d_u8_vec = _mm512_or_si512(_mm512_subs_epu8(a_u8_vec, b_u8_vec), _mm512_subs_epu8(b_u8_vec, a_u8_vec));
1991
+ d_i16_low_vec = _mm512_unpacklo_epi8(d_u8_vec, zeros_vec);
1992
+ d_i16_high_vec = _mm512_unpackhi_epi8(d_u8_vec, zeros_vec);
1993
+
1994
+ // Multiply and accumulate at `int16` level, accumulate at `int32` level:
1995
+ d2_i32_low_vec = _mm512_dpwssd_epi32(d2_i32_low_vec, d_i16_low_vec, d_i16_low_vec);
1996
+ d2_i32_high_vec = _mm512_dpwssd_epi32(d2_i32_high_vec, d_i16_high_vec, d_i16_high_vec);
1997
+ if (n) goto simsimd_l2sq_u8_ice_cycle;
1998
+
1999
+ *result = _mm512_reduce_add_epi32(_mm512_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
2000
+ }
2001
+
2002
+ SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
2003
+ simsimd_distance_t *result) {
2004
+
2005
+ __m512i ab_i32_low_vec = _mm512_setzero_si512();
2006
+ __m512i ab_i32_high_vec = _mm512_setzero_si512();
2007
+ __m512i a2_i32_low_vec = _mm512_setzero_si512();
2008
+ __m512i a2_i32_high_vec = _mm512_setzero_si512();
2009
+ __m512i b2_i32_low_vec = _mm512_setzero_si512();
2010
+ __m512i b2_i32_high_vec = _mm512_setzero_si512();
2011
+ __m512i const zeros_vec = _mm512_setzero_si512();
2012
+ __m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
2013
+ __m512i a_u8_vec, b_u8_vec;
2014
+
2015
+ simsimd_cos_u8_ice_cycle:
2016
+ if (n < 64) {
2017
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
2018
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
2019
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
2020
+ n = 0;
2021
+ }
2022
+ else {
2023
+ a_u8_vec = _mm512_loadu_si512(a);
2024
+ b_u8_vec = _mm512_loadu_si512(b);
2025
+ a += 64, b += 64, n -= 64;
2026
+ }
2027
+
2028
+ // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
2029
+ // instructions instead of extracts, as they are much faster and more efficient.
2030
+ a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
2031
+ a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
2032
+ b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
2033
+ b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
2034
+
2035
+ // Multiply and accumulate as `int16`, accumulate products as `int32`:
2036
+ ab_i32_low_vec = _mm512_dpwssds_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
2037
+ ab_i32_high_vec = _mm512_dpwssds_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
2038
+ a2_i32_low_vec = _mm512_dpwssds_epi32(a2_i32_low_vec, a_i16_low_vec, a_i16_low_vec);
2039
+ a2_i32_high_vec = _mm512_dpwssds_epi32(a2_i32_high_vec, a_i16_high_vec, a_i16_high_vec);
2040
+ b2_i32_low_vec = _mm512_dpwssds_epi32(b2_i32_low_vec, b_i16_low_vec, b_i16_low_vec);
2041
+ b2_i32_high_vec = _mm512_dpwssds_epi32(b2_i32_high_vec, b_i16_high_vec, b_i16_high_vec);
2042
+ if (n) goto simsimd_cos_u8_ice_cycle;
2043
+
2044
+ int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
2045
+ int a2 = _mm512_reduce_add_epi32(_mm512_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
2046
+ int b2 = _mm512_reduce_add_epi32(_mm512_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
2047
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2048
+ }
2049
+
2050
+ SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2051
+ simsimd_distance_t *result) {
2052
+ simsimd_l2sq_i4x2_ice(a, b, n_words, result);
2053
+ *result = _simsimd_sqrt_f32_haswell(*result);
2054
+ }
2055
+ SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2056
+ simsimd_distance_t *result) {
2057
+
2058
+ // While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
2059
+ // The absolute difference between two 4-bit integers is at most 15 and it is always a `uint4_t` value!
2060
+ // Moreover, it's square is at most 225, which fits into `uint8_t` and can be computed with a single
2061
+ // lookup table. Accumulating those values is similar to checksumming, a piece of cake for SIMD!
2062
+ __m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
2063
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2064
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2065
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2066
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
2067
+ __m512i const u4_squares_lookup_vec = _mm512_set_epi8( //
2068
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2069
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2070
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2071
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0);
2072
+
2073
+ /// The mask used to take the low nibble of each byte.
2074
+ __m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
2075
+
2076
+ // Temporaries:
2077
+ __m512i a_i4x2_vec, b_i4x2_vec;
2078
+ __m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
2079
+ __m512i d_u8_low_vec, d_u8_high_vec; //! Only the low 4 bits are actually used
2080
+ __m512i d2_u8_low_vec, d2_u8_high_vec;
2081
+ __m512i d2_u16_low_vec, d2_u16_high_vec;
2082
+
2083
+ // Accumulators:
2084
+ __m512i d2_u32_vec = _mm512_setzero_si512();
2085
+
2086
+ simsimd_l2sq_i4x2_ice_cycle:
2087
+ if (n_words < 64) {
2088
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
2089
+ a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
2090
+ b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
2091
+ n_words = 0;
2092
+ }
2093
+ else {
2094
+ a_i4x2_vec = _mm512_loadu_epi8(a);
2095
+ b_i4x2_vec = _mm512_loadu_epi8(b);
2096
+ a += 64, b += 64, n_words -= 64;
2097
+ }
2098
+
2099
+ // Unpack the 4-bit values into 8-bit values with an empty top nibble.
2100
+ a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
2101
+ a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
2102
+ b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
2103
+ b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
2104
+ a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
2105
+ a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
2106
+ b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
2107
+ b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
2108
+
2109
+ // We can implement subtraction with a lookup table, or using `_mm512_sub_epi8`.
2110
+ d_u8_low_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_low_vec, b_i8_low_vec));
2111
+ d_u8_high_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_high_vec, b_i8_high_vec));
2112
+
2113
+ // Now we can use the lookup table to compute the squares of the 4-bit unsigned integers
2114
+ // in the low nibbles of the `d_u8_low_vec` and `d_u8_high_vec` vectors.
2115
+ d2_u8_low_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_low_vec);
2116
+ d2_u8_high_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_high_vec);
2117
+
2118
+ // Aggregating into 16-bit integers, we need to first upcast our 8-bit values to 16 bits.
2119
+ // After that, we will perform one more operation, upcasting further into 32-bit integers.
2120
+ d2_u16_low_vec = //
2121
+ _mm512_add_epi16( //
2122
+ _mm512_unpacklo_epi8(d2_u8_low_vec, _mm512_setzero_si512()),
2123
+ _mm512_unpackhi_epi8(d2_u8_low_vec, _mm512_setzero_si512()));
2124
+ d2_u16_high_vec = //
2125
+ _mm512_add_epi16( //
2126
+ _mm512_unpacklo_epi8(d2_u8_high_vec, _mm512_setzero_si512()),
2127
+ _mm512_unpackhi_epi8(d2_u8_high_vec, _mm512_setzero_si512()));
2128
+ d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_low_vec, _mm512_setzero_si512()));
2129
+ d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_high_vec, _mm512_setzero_si512()));
2130
+ if (n_words) goto simsimd_l2sq_i4x2_ice_cycle;
2131
+
2132
+ // Finally, we can reduce the 16-bit integers to 32-bit integers and sum them up.
2133
+ int d2 = _mm512_reduce_add_epi32(d2_u32_vec);
2134
+ *result = d2;
2135
+ }
2136
+ SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2137
+ simsimd_distance_t *result) {
2138
+
2139
+ // We need to compose a lookup table for all the scalar products of 4-bit integers.
2140
+ // While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
2141
+ // Practically speaking, the product of two 4-bit signed integers is a 7-bit integer,
2142
+ // as the maximum absolute value of the product is `abs(-8 * -8) == 64`.
2143
+ //
2144
+ // To store 128 possible values of 2^7 bits we only need 128 single-byte scalars,
2145
+ // or just 2x ZMM registers. In that case our lookup will only take `vpermi2b` instruction,
2146
+ // easily inokable with `_mm512_permutex2var_epi8` intrinsic with latency of 6 on Sapphire Rapids.
2147
+ // The problem is converting 2d indices of our symmetric matrix into 1d offsets in the dense array.
2148
+ //
2149
+ // Alternatively, we can take the entire symmetric (16 x 16) matrix of products,
2150
+ // put into 4x ZMM registers, and use it with `_mm512_shuffle_epi8`, remembering
2151
+ // that it can only lookup with 128-bit lanes (16x 8-bit values).
2152
+ // That intrinsic has latency 1, but will need to be repeated and combined with
2153
+ // multiple iterations of `_mm512_shuffle_i64x2` that has latency 3.
2154
+ //
2155
+ // Altenatively, we can get down to 3 cycles per lookup with `vpermb` and `_mm512_permutexvar_epi8` intrinsics.
2156
+ // For that we can split our (16 x 16) matrix into 4x (8 x 8) submatrices, and use 4x ZMM registers.
2157
+ //
2158
+ // Still, all of those solutions are quite heavy compared to two parallel calls to `_mm512_dpbusds_epi32`
2159
+ // for the dot product. But we can still use the `_mm512_permutexvar_epi8` to compute the squares of the
2160
+ // 16 possible `int4_t` values faster.
2161
+ //
2162
+ // Here is how our `int4_t` range looks:
2163
+ //
2164
+ // dec: 0 1 2 3 4 5 6 7 -8 -7 -6 -5 -4 -3 -2 -1
2165
+ // hex: 0 1 2 3 4 5 6 7 8 9 A B C D E F
2166
+ //
2167
+ // Squared:
2168
+ //
2169
+ // dec2: 0 1 4 9 16 25 36 49 64 49 36 25 16 9 4 1
2170
+ // hex2: 0 1 4 9 10 19 24 31 40 31 24 19 10 9 4 1
2171
+ //
2172
+ // Broadcast it to every lane, so that: `square(x) == _mm512_shuffle_epi8(i4_squares_lookup_vec, x)`.
2173
+ __m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
2174
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2175
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2176
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2177
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
2178
+ __m512i const i4_squares_lookup_vec = _mm512_set_epi8( //
2179
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2180
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2181
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2182
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0);
2183
+
2184
+ /// The mask used to take the low nibble of each byte.
2185
+ __m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
2186
+
2187
+ // Temporaries:
2188
+ __m512i a_i4x2_vec, b_i4x2_vec;
2189
+ __m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
2190
+ __m512i a2_u8_vec, b2_u8_vec;
2191
+
2192
+ // Accumulators:
2193
+ __m512i a2_u16_low_vec = _mm512_setzero_si512();
2194
+ __m512i a2_u16_high_vec = _mm512_setzero_si512();
2195
+ __m512i b2_u16_low_vec = _mm512_setzero_si512();
2196
+ __m512i b2_u16_high_vec = _mm512_setzero_si512();
2197
+ __m512i ab_i32_low_vec = _mm512_setzero_si512();
2198
+ __m512i ab_i32_high_vec = _mm512_setzero_si512();
2199
+
2200
+ simsimd_cos_i4x2_ice_cycle:
2201
+ if (n_words < 64) {
2202
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
2203
+ a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
2204
+ b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
2205
+ n_words = 0;
2206
+ }
2207
+ else {
2208
+ a_i4x2_vec = _mm512_loadu_epi8(a);
2209
+ b_i4x2_vec = _mm512_loadu_epi8(b);
2210
+ a += 64, b += 64, n_words -= 64;
2211
+ }
2212
+
2213
+ // Unpack the 4-bit values into 8-bit values with an empty top nibble.
2214
+ // For now, they are not really 8-bit integers, as they are not sign-extended.
2215
+ // That part will come later, using the `i4_to_i8_lookup_vec` lookup.
2216
+ a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
2217
+ a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
2218
+ b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
2219
+ b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
2220
+
2221
+ // Compute the squares of the 4-bit integers.
2222
+ // For symmetry we could have used 4 registers, aka "a2_i8_low_vec", "a2_i8_high_vec", "b2_i8_low_vec",
2223
+ // "b2_i8_high_vec". But the largest square value is just 64, so we can safely aggregate into 8-bit unsigned values.
2224
+ a2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_low_vec),
2225
+ _mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_high_vec));
2226
+ b2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_low_vec),
2227
+ _mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_high_vec));
2228
+
2229
+ // We can safely aggregate into just 16-bit sums without overflow, if the vectors have less than:
2230
+ // (2 scalars / byte) * (64 bytes / register) * (256 non-overflowing 8-bit additions in 16-bit intesgers)
2231
+ // = 32'768 dimensions.
2232
+ //
2233
+ // We use saturated addition here to clearly inform in case of overflow.
2234
+ a2_u16_low_vec = _mm512_adds_epu16(a2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
2235
+ a2_u16_high_vec = _mm512_adds_epu16(a2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
2236
+ b2_u16_low_vec = _mm512_adds_epu16(b2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
2237
+ b2_u16_high_vec = _mm512_adds_epu16(b2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
2238
+
2239
+ // Time to perform the proper sign extension of the 4-bit integers to 8-bit integers.
2240
+ a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
2241
+ a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
2242
+ b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
2243
+ b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
1473
2244
 
1474
2245
  // The same trick won't work for the primary dot-product, as the signs vector
1475
- // components may differ significantly. So we have to use two `_mm512_dpwssd_epi32`
2246
+ // components may differ significantly. So we have to use two `_mm512_dpwssds_epi32`
1476
2247
  // intrinsics instead, upcasting four chunks to 16-bit integers beforehand!
1477
- ab_low_i32s_vec = _mm512_dpwssds_epi32( //
1478
- ab_low_i32s_vec, //
1479
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_vec)), //
1480
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_vec)));
1481
- ab_high_i32s_vec = _mm512_dpwssds_epi32( //
1482
- ab_high_i32s_vec, //
1483
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_vec, 1)), //
1484
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_vec, 1)));
1485
- if (n)
1486
- goto simsimd_cos_i8_ice_cycle;
1487
-
1488
- int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_low_i32s_vec, ab_high_i32s_vec));
1489
- int a2 = _mm512_reduce_add_epi32(a2_i32s_vec);
1490
- int b2 = _mm512_reduce_add_epi32(b2_i32s_vec);
1491
-
1492
- // Compute the reciprocal square roots of a2 and b2
1493
- // Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
1494
- // so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
1495
- __m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1496
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1497
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1498
- *result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
2248
+ // Alternatively, we can flip the signs of the second argument and use `_mm512_dpbusds_epi32`,
2249
+ // but it ends up taking more instructions.
2250
+ ab_i32_low_vec = _mm512_dpwssds_epi32( //
2251
+ ab_i32_low_vec, //
2252
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_low_vec)), //
2253
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_low_vec)));
2254
+ ab_i32_low_vec = _mm512_dpwssds_epi32( //
2255
+ ab_i32_low_vec, //
2256
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_low_vec, 1)), //
2257
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_low_vec, 1)));
2258
+ ab_i32_high_vec = _mm512_dpwssds_epi32( //
2259
+ ab_i32_high_vec, //
2260
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_high_vec)), //
2261
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_high_vec)));
2262
+ ab_i32_high_vec = _mm512_dpwssds_epi32( //
2263
+ ab_i32_high_vec, //
2264
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_high_vec, 1)), //
2265
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_high_vec, 1)));
2266
+ if (n_words) goto simsimd_cos_i4x2_ice_cycle;
2267
+
2268
+ int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
2269
+ unsigned short a2_u16[32], b2_u16[32];
2270
+ _mm512_storeu_si512(a2_u16, _mm512_add_epi16(a2_u16_low_vec, a2_u16_high_vec));
2271
+ unsigned int a2 = 0, b2 = 0;
2272
+ for (int i = 0; i < 32; ++i) a2 += a2_u16[i], b2 += b2_u16[i];
2273
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1499
2274
  }
1500
2275
 
1501
2276
  #pragma clang attribute pop
1502
2277
  #pragma GCC pop_options
1503
2278
  #endif // SIMSIMD_TARGET_ICE
1504
- #endif // SIMSIMD_TARGET_X86
2279
+
2280
+ #if SIMSIMD_TARGET_SIERRA
2281
+ #pragma GCC push_options
2282
+ #pragma GCC target("avx2", "bmi2", "avx2vnni")
2283
+ #pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
2284
+
2285
+ SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
2286
+ simsimd_distance_t *result) {
2287
+
2288
+ __m256i ab_i32_vec = _mm256_setzero_si256();
2289
+ __m256i a2_i32_vec = _mm256_setzero_si256();
2290
+ __m256i b2_i32_vec = _mm256_setzero_si256();
2291
+
2292
+ simsimd_size_t i = 0;
2293
+ for (; i + 32 <= n; i += 32) {
2294
+ __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
2295
+ __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
2296
+ ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
2297
+ a2_i32_vec = _mm256_dpbssds_epi32(a2_i32_vec, a_i8_vec, a_i8_vec);
2298
+ b2_i32_vec = _mm256_dpbssds_epi32(b2_i32_vec, b_i8_vec, b_i8_vec);
2299
+ }
2300
+
2301
+ // Further reduce to a single sum for each vector
2302
+ int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
2303
+ int a2 = _simsimd_reduce_i32x8_haswell(a2_i32_vec);
2304
+ int b2 = _simsimd_reduce_i32x8_haswell(b2_i32_vec);
2305
+
2306
+ // Take care of the tail:
2307
+ for (; i < n; ++i) {
2308
+ int ai = a[i], bi = b[i];
2309
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
2310
+ }
2311
+
2312
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2313
+ }
2314
+
2315
+ #pragma clang attribute pop
2316
+ #pragma GCC pop_options
2317
+ #endif // SIMSIMD_TARGET_SIERRA
2318
+ #endif // _SIMSIMD_TARGET_X86
1505
2319
 
1506
2320
  #ifdef __cplusplus
1507
2321
  }