usearch 2.15.1 → 2.15.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,8 +5,8 @@
5
5
  * @date March 14, 2023
6
6
  *
7
7
  * Contains:
8
- * - L2 (Euclidean) squared distance
9
- * - Cosine (Angular) similarity
8
+ * - L2 (Euclidean) regular and squared distance
9
+ * - Cosine (Angular) distance - @b not similarity!
10
10
  *
11
11
  * For datatypes:
12
12
  * - 64-bit IEEE floating point numbers
@@ -14,6 +14,7 @@
14
14
  * - 16-bit IEEE floating point numbers
15
15
  * - 16-bit brain floating point numbers
16
16
  * - 8-bit signed integral numbers
17
+ * - 4-bit signed integral numbers
17
18
  *
18
19
  * For hardware architectures:
19
20
  * - Arm (NEON, SVE)
@@ -27,7 +28,7 @@
27
28
 
28
29
  #include "types.h"
29
30
 
30
- #include "dot.h" // `_mm256_reduce_add_ps_dbl`
31
+ #include "dot.h" // `_simsimd_reduce_f32x8_haswell`
31
32
 
32
33
  #ifdef __cplusplus
33
34
  extern "C" {
@@ -39,26 +40,35 @@ extern "C" {
39
40
  * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
40
41
  * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
41
42
  */
43
+ SIMSIMD_PUBLIC void simsimd_l2_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
42
44
  SIMSIMD_PUBLIC void simsimd_l2sq_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
43
45
  SIMSIMD_PUBLIC void simsimd_cos_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
46
+ SIMSIMD_PUBLIC void simsimd_l2_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
44
47
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
45
48
  SIMSIMD_PUBLIC void simsimd_cos_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
49
+ SIMSIMD_PUBLIC void simsimd_l2_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
46
50
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
47
51
  SIMSIMD_PUBLIC void simsimd_cos_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
52
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
48
53
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
49
54
  SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
55
+ SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
50
56
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
51
57
  SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
52
58
 
53
59
  /* Double-precision serial backends for all numeric types.
54
60
  * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
55
61
  */
62
+ SIMSIMD_PUBLIC void simsimd_l2_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
56
63
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
57
64
  SIMSIMD_PUBLIC void simsimd_cos_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
65
+ SIMSIMD_PUBLIC void simsimd_l2_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
58
66
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
59
67
  SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
68
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
60
69
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
61
70
  SIMSIMD_PUBLIC void simsimd_cos_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
71
+ SIMSIMD_PUBLIC void simsimd_l2_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
62
72
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
63
73
  SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
64
74
 
@@ -66,22 +76,37 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t
66
76
  * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
67
77
  * server CPUs produced before 2023.
68
78
  */
79
+ SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
80
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
81
+ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
82
+ SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
69
83
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
70
84
  SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
85
+ SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
71
86
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
72
87
  SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
88
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
89
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
90
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
91
+ SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
73
92
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
74
93
  SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
75
94
 
76
95
  /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
77
96
  * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
78
97
  */
79
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
80
- SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
81
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
82
- SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
83
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
84
- SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
98
+ SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
99
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
100
+ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
101
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
102
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
103
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
104
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
105
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
106
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
107
+ SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
108
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
109
+ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
85
110
 
86
111
  /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
87
112
  * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
@@ -89,98 +114,189 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
89
114
  * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
90
115
  * properly vectorized by recent compilers.
91
116
  */
92
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
93
- SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
94
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
95
- SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
96
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
97
- SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
98
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
99
- SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
117
+ SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
118
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
119
+ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
120
+ SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
121
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
122
+ SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
123
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
124
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
125
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
126
+ SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
127
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
128
+ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
100
129
 
101
130
  /* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
102
131
  * Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
103
132
  * as well as masked operations. This is enough to supersede auto-vectorization on `f32` and `f64` types.
104
133
  */
105
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
106
- SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t*);
107
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
108
- SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t*);
134
+ SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
135
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
136
+ SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
137
+ SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
138
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
139
+ SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
109
140
 
110
141
  /* SIMD-powered backends for AVX512 CPUs of Ice Lake generation and newer, using mixed arithmetic over 512-bit words.
111
142
  * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
112
143
  * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
113
144
  */
114
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
115
- SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
116
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
117
- SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t*);
118
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
119
- SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t*);
145
+ SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
146
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
147
+ SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
148
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
149
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
150
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
151
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
152
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
153
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
120
154
 
121
155
  // clang-format on
122
156
 
123
- #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, converter) \
157
+ #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
124
158
  SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const* a, \
125
159
  simsimd_##input_type##_t const* b, simsimd_size_t n, \
126
160
  simsimd_distance_t* result) { \
127
161
  simsimd_##accumulator_type##_t d2 = 0; \
128
162
  for (simsimd_size_t i = 0; i != n; ++i) { \
129
- simsimd_##accumulator_type##_t ai = converter(a[i]); \
130
- simsimd_##accumulator_type##_t bi = converter(b[i]); \
163
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
164
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
131
165
  d2 += (ai - bi) * (ai - bi); \
132
166
  } \
133
167
  *result = d2; \
134
168
  }
135
169
 
136
- #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, converter) \
170
+ #define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
171
+ SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const* a, \
172
+ simsimd_##input_type##_t const* b, simsimd_size_t n, \
173
+ simsimd_distance_t* result) { \
174
+ simsimd_l2sq_##input_type##_##name(a, b, n, result); \
175
+ *result = SIMSIMD_SQRT(*result); \
176
+ }
177
+
178
+ #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
137
179
  SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const* a, \
138
180
  simsimd_##input_type##_t const* b, simsimd_size_t n, \
139
181
  simsimd_distance_t* result) { \
140
182
  simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
141
183
  for (simsimd_size_t i = 0; i != n; ++i) { \
142
- simsimd_##accumulator_type##_t ai = converter(a[i]); \
143
- simsimd_##accumulator_type##_t bi = converter(b[i]); \
184
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
185
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
144
186
  ab += ai * bi; \
145
187
  a2 += ai * ai; \
146
188
  b2 += bi * bi; \
147
189
  } \
148
- *result = ab != 0 ? (1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2)) : 1; \
190
+ if (a2 == 0 && b2 == 0) { \
191
+ *result = 0; \
192
+ } else if (ab == 0) { \
193
+ *result = 1; \
194
+ } else { \
195
+ simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
196
+ *result = unclipped_result > 0 ? unclipped_result : 0; \
197
+ } \
149
198
  }
150
199
 
151
- SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_l2sq_f64_serial
152
- SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_IDENTIFY) // simsimd_cos_f64_serial
200
+ SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
201
+ SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f64_serial
202
+ SIMSIMD_MAKE_L2(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f64_serial
153
203
 
154
- SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_l2sq_f32_serial
155
- SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_IDENTIFY) // simsimd_cos_f32_serial
204
+ SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_serial
205
+ SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_serial
206
+ SIMSIMD_MAKE_L2(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_serial
156
207
 
157
- SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_l2sq_f16_serial
158
- SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_UNCOMPRESS_F16) // simsimd_cos_f16_serial
208
+ SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_serial
209
+ SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_serial
210
+ SIMSIMD_MAKE_L2(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_serial
159
211
 
160
- SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_l2sq_bf16_serial
161
- SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_UNCOMPRESS_BF16) // simsimd_cos_bf16_serial
212
+ SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_serial
213
+ SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_serial
214
+ SIMSIMD_MAKE_L2(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_serial
162
215
 
163
- SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_IDENTIFY) // simsimd_l2sq_i8_serial
164
- SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_IDENTIFY) // simsimd_cos_i8_serial
216
+ SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
217
+ SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
218
+ SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
165
219
 
166
- SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_l2sq_f32_accurate
167
- SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_IDENTIFY) // simsimd_cos_f32_accurate
220
+ SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
221
+ SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
222
+ SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
168
223
 
169
- SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_l2sq_f16_accurate
170
- SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16) // simsimd_cos_f16_accurate
224
+ SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_accurate
225
+ SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_accurate
226
+ SIMSIMD_MAKE_L2(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_accurate
171
227
 
172
- SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_l2sq_bf16_accurate
173
- SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_UNCOMPRESS_BF16) // simsimd_cos_bf16_accurate
228
+ SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_accurate
229
+ SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
230
+ SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
174
231
 
175
- SIMSIMD_MAKE_L2SQ(accurate, i8, i32, SIMSIMD_IDENTIFY) // simsimd_l2sq_i8_accurate
176
- SIMSIMD_MAKE_COS(accurate, i8, i32, SIMSIMD_IDENTIFY) // simsimd_cos_i8_accurate
232
+ SIMSIMD_MAKE_COS(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_accurate
233
+ SIMSIMD_MAKE_L2SQ(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_accurate
234
+ SIMSIMD_MAKE_L2(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_accurate
177
235
 
178
236
  #if SIMSIMD_TARGET_ARM
179
237
  #if SIMSIMD_TARGET_NEON
180
238
  #pragma GCC push_options
181
- #pragma GCC target("+simd")
182
- #pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
239
+ #pragma GCC target("arch=armv8.2-a+simd")
240
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
241
+
242
+ SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_neon(simsimd_f32_t x) {
243
+ return vget_lane_f32(vsqrt_f32(vdup_n_f32(x)), 0);
244
+ }
245
+ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
246
+ return vget_lane_f64(vsqrt_f64(vdup_n_f64(x)), 0);
247
+ }
248
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
249
+ simsimd_f32_t b2) {
250
+ if (a2 == 0 && b2 == 0)
251
+ return 0;
252
+ if (ab == 0)
253
+ return 1;
254
+ simsimd_f32_t squares_arr[2] = {a2, b2};
255
+ float32x2_t squares = vld1_f32(squares_arr);
256
+ // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
257
+ // Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
258
+ // One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
259
+ // https://github.com/lighttransport/embree-aarch64/issues/24
260
+ // https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
261
+ float32x2_t rsqrts = vrsqrte_f32(squares);
262
+ // Perform two rounds of Newton-Raphson refinement:
263
+ // https://en.wikipedia.org/wiki/Newton%27s_method
264
+ rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
265
+ rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
266
+ vst1_f32(squares_arr, rsqrts);
267
+ simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
268
+ return result > 0 ? result : 0;
269
+ }
270
+
271
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
272
+ simsimd_f64_t b2) {
273
+ if (a2 == 0 && b2 == 0)
274
+ return 0;
275
+ if (ab == 0)
276
+ return 1;
277
+ simsimd_f64_t squares_arr[2] = {a2, b2};
278
+ float64x2_t squares = vld1q_f64(squares_arr);
279
+
280
+ // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
281
+ // Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
282
+ // One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
283
+ // https://github.com/lighttransport/embree-aarch64/issues/24
284
+ // https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
285
+ float64x2_t rsqrts = vrsqrteq_f64(squares);
286
+ // Perform two rounds of Newton-Raphson refinement:
287
+ // https://en.wikipedia.org/wiki/Newton%27s_method
288
+ rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
289
+ rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
290
+ vst1q_f64(squares_arr, rsqrts);
291
+ simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
292
+ return result > 0 ? result : 0;
293
+ }
183
294
 
295
+ SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
296
+ simsimd_distance_t* result) {
297
+ simsimd_l2sq_f32_neon(a, b, n, result);
298
+ *result = _simsimd_sqrt_f64_neon(*result);
299
+ }
184
300
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
185
301
  simsimd_distance_t* result) {
186
302
  float32x4_t sum_vec = vdupq_n_f32(0);
@@ -216,95 +332,117 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
216
332
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
217
333
  }
218
334
 
219
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
220
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
221
- vst1_f32(a2_b2_arr, vrsqrte_f32(vld1_f32(a2_b2_arr)));
222
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
335
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
336
+ }
337
+
338
+ SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
339
+ simsimd_distance_t* result) {
340
+ simsimd_l2sq_f64_neon(a, b, n, result);
341
+ *result = _simsimd_sqrt_f64_neon(*result);
342
+ }
343
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
344
+ simsimd_distance_t* result) {
345
+ float64x2_t sum_vec = vdupq_n_f64(0);
346
+ simsimd_size_t i = 0;
347
+ for (; i + 2 <= n; i += 2) {
348
+ float64x2_t a_vec = vld1q_f64(a + i);
349
+ float64x2_t b_vec = vld1q_f64(b + i);
350
+ float64x2_t diff_vec = vsubq_f64(a_vec, b_vec);
351
+ sum_vec = vfmaq_f64(sum_vec, diff_vec, diff_vec);
352
+ }
353
+ simsimd_f64_t sum = vaddvq_f64(sum_vec);
354
+ for (; i < n; ++i) {
355
+ simsimd_f64_t diff = a[i] - b[i];
356
+ sum += diff * diff;
357
+ }
358
+ *result = sum;
359
+ }
360
+
361
+ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
362
+ simsimd_distance_t* result) {
363
+ float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
364
+ simsimd_size_t i = 0;
365
+ for (; i + 2 <= n; i += 2) {
366
+ float64x2_t a_vec = vld1q_f64(a + i);
367
+ float64x2_t b_vec = vld1q_f64(b + i);
368
+ ab_vec = vfmaq_f64(ab_vec, a_vec, b_vec);
369
+ a2_vec = vfmaq_f64(a2_vec, a_vec, a_vec);
370
+ b2_vec = vfmaq_f64(b2_vec, b_vec, b_vec);
371
+ }
372
+ simsimd_f64_t ab = vaddvq_f64(ab_vec), a2 = vaddvq_f64(a2_vec), b2 = vaddvq_f64(b2_vec);
373
+ for (; i < n; ++i) {
374
+ simsimd_f64_t ai = a[i], bi = b[i];
375
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
376
+ }
377
+
378
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
223
379
  }
224
380
 
225
381
  #pragma clang attribute pop
226
382
  #pragma GCC pop_options
383
+ #endif // SIMSIMD_TARGET_NEON
227
384
 
385
+ #if SIMSIMD_TARGET_NEON_F16
228
386
  #pragma GCC push_options
229
- #pragma GCC target("+simd+fp16")
230
- #pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
387
+ #pragma GCC target("arch=armv8.2-a+simd+fp16")
388
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
231
389
 
390
+ SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
391
+ simsimd_distance_t* result) {
392
+ simsimd_l2sq_f16_neon(a, b, n, result);
393
+ *result = _simsimd_sqrt_f32_neon(*result);
394
+ }
232
395
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
233
396
  simsimd_distance_t* result) {
397
+ float32x4_t a_vec, b_vec;
234
398
  float32x4_t sum_vec = vdupq_n_f32(0);
235
- simsimd_size_t i = 0;
236
- for (; i + 4 <= n; i += 4) {
237
- float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
238
- float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
239
- float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
240
- sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
241
- }
242
399
 
243
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
244
- // function will run. It is extremely slow, so even for the tail, let's combine serial
245
- // loads and stores with vectorized math.
246
- if (i < n) {
247
- union {
248
- float16x4_t f16_vec;
249
- simsimd_f16_t f16[4];
250
- } a_padded_tail, b_padded_tail;
251
- simsimd_size_t j = 0;
252
- for (; i < n; ++i, ++j)
253
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
254
- for (; j < 4; ++j)
255
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
256
- float32x4_t diff_vec = vsubq_f32(vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
257
- sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
400
+ simsimd_l2sq_f16_neon_cycle:
401
+ if (n < 4) {
402
+ a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
403
+ b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
404
+ n = 0;
405
+ } else {
406
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
407
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
408
+ n -= 4, a += 4, b += 4;
258
409
  }
410
+ float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
411
+ sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
412
+ if (n)
413
+ goto simsimd_l2sq_f16_neon_cycle;
259
414
 
260
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
261
- *result = sum;
415
+ *result = vaddvq_f32(sum_vec);
262
416
  }
263
417
 
264
418
  SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
265
419
  simsimd_distance_t* result) {
266
420
  float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
267
- simsimd_size_t i = 0;
268
- for (; i + 4 <= n; i += 4) {
269
- float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
270
- float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
271
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
272
- a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
273
- b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
274
- }
421
+ float32x4_t a_vec, b_vec;
275
422
 
276
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
277
- // function will run. It is extremely slow, so even for the tail, let's combine serial
278
- // loads and stores with vectorized math.
279
- if (i < n) {
280
- union {
281
- float16x4_t f16_vec;
282
- simsimd_f16_t f16[4];
283
- } a_padded_tail, b_padded_tail;
284
- simsimd_size_t j = 0;
285
- for (; i < n; ++i, ++j)
286
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
287
- for (; j < 4; ++j)
288
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
289
- float32x4_t a_vec = vcvt_f32_f16(a_padded_tail.f16_vec);
290
- float32x4_t b_vec = vcvt_f32_f16(b_padded_tail.f16_vec);
291
- ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
292
- a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
293
- b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
423
+ simsimd_cos_f16_neon_cycle:
424
+ if (n < 4) {
425
+ a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
426
+ b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
427
+ n = 0;
428
+ } else {
429
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
430
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
431
+ n -= 4, a += 4, b += 4;
294
432
  }
433
+ ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
434
+ a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
435
+ b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
436
+ if (n)
437
+ goto simsimd_cos_f16_neon_cycle;
295
438
 
296
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
297
439
  simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
298
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
299
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
300
- a2_b2 = vrsqrte_f32(a2_b2);
301
- vst1_f32(a2_b2_arr, a2_b2);
302
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
440
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
303
441
  }
304
442
 
305
443
  #pragma clang attribute pop
306
444
  #pragma GCC pop_options
307
- #endif // SIMSIMD_TARGET_NEON
445
+ #endif // SIMSIMD_TARGET_NEON_F16
308
446
 
309
447
  #if SIMSIMD_TARGET_NEON_BF16
310
448
  #pragma GCC push_options
@@ -341,95 +479,76 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_
341
479
  // }
342
480
  // float32x4_t products_vec = vaddq_f32(products_high_vec, products_low_vec);
343
481
  // simsimd_f32_t a2 = products_vec[0], ab = products_vec[1], b2 = products_vec[3];
482
+ //
483
+ // Another way of accomplishing the same thing is to process the odd and even elements separately,
484
+ // using special `vbfmlaltq_f32` and `vbfmlalbq_f32` intrinsics:
485
+ //
486
+ // ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
487
+ // ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
488
+ // a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
489
+ // a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
490
+ // b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
491
+ // b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
492
+ //
344
493
 
345
- float32x4_t ab_high_vec = vdupq_n_f32(0), ab_low_vec = vdupq_n_f32(0);
346
- float32x4_t a2_high_vec = vdupq_n_f32(0), a2_low_vec = vdupq_n_f32(0);
347
- float32x4_t b2_high_vec = vdupq_n_f32(0), b2_low_vec = vdupq_n_f32(0);
348
- simsimd_size_t i = 0;
349
- for (; i + 8 <= n; i += 8) {
350
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
351
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
352
- ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
353
- ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
354
- a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
355
- a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
356
- b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
357
- b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
358
- }
494
+ float32x4_t ab_vec = vdupq_n_f32(0);
495
+ float32x4_t a2_vec = vdupq_n_f32(0);
496
+ float32x4_t b2_vec = vdupq_n_f32(0);
497
+ bfloat16x8_t a_vec, b_vec;
359
498
 
360
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
361
- // function will run. It is extremely slow, so even for the tail, let's combine serial
362
- // loads and stores with vectorized math.
363
- if (i < n) {
364
- union {
365
- bfloat16x8_t bf16_vec;
366
- simsimd_bf16_t bf16[8];
367
- } a_padded_tail, b_padded_tail;
368
- simsimd_size_t j = 0;
369
- for (; i < n; ++i, ++j)
370
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
371
- for (; j < 8; ++j)
372
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
373
- ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
374
- ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
375
- a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
376
- a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
377
- b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
378
- b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
499
+ simsimd_cos_bf16_neon_cycle:
500
+ if (n < 8) {
501
+ a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
502
+ b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
503
+ n = 0;
504
+ } else {
505
+ a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
506
+ b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
507
+ n -= 8, a += 8, b += 8;
379
508
  }
509
+ ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
510
+ a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
511
+ b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
512
+ if (n)
513
+ goto simsimd_cos_bf16_neon_cycle;
380
514
 
381
515
  // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
382
- simsimd_f32_t ab = vaddvq_f32(vaddq_f32(ab_high_vec, ab_low_vec)),
383
- a2 = vaddvq_f32(vaddq_f32(a2_high_vec, a2_low_vec)),
384
- b2 = vaddvq_f32(vaddq_f32(b2_high_vec, b2_low_vec));
385
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
386
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
387
- a2_b2 = vrsqrte_f32(a2_b2);
388
- vst1_f32(a2_b2_arr, a2_b2);
389
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
516
+ simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
517
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
390
518
  }
391
519
 
520
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
521
+ simsimd_distance_t* result) {
522
+ simsimd_l2sq_bf16_neon(a, b, n, result);
523
+ *result = _simsimd_sqrt_f64_neon(*result);
524
+ }
392
525
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
393
526
  simsimd_distance_t* result) {
394
- float32x4_t diff_high_vec = vdupq_n_f32(0), diff_low_vec = vdupq_n_f32(0);
527
+ float32x4_t diff_high_vec, diff_low_vec;
395
528
  float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
396
- simsimd_size_t i = 0;
397
- for (; i + 8 <= n; i += 8) {
398
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a + i);
399
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b + i);
400
- // We can't perform subtraction in `bf16`. One option would be to upcast to `f32`
401
- // and then subtract, converting back to `bf16` for computing the squared difference.
529
+
530
+ simsimd_l2sq_bf16_neon_cycle:
531
+ if (n < 8) {
532
+ bfloat16x8_t a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
533
+ bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
402
534
  diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
403
535
  diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
404
- sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
405
- sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
406
- }
407
-
408
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
409
- // function will run. It is extremely slow, so even for the tail, let's combine serial
410
- // loads and stores with vectorized math.
411
- if (i < n) {
412
- union {
413
- bfloat16x8_t bf16_vec;
414
- simsimd_bf16_t bf16[8];
415
- } a_padded_tail, b_padded_tail;
416
- simsimd_size_t j = 0;
417
- for (; i < n; ++i, ++j)
418
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
419
- for (; j < 8; ++j)
420
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
421
- diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_padded_tail.bf16_vec)),
422
- vcvt_f32_bf16(vget_high_bf16(b_padded_tail.bf16_vec)));
423
- diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_padded_tail.bf16_vec)),
424
- vcvt_f32_bf16(vget_low_bf16(b_padded_tail.bf16_vec)));
425
- sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
426
- sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
536
+ n = 0;
537
+ } else {
538
+ bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
539
+ bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
540
+ diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
541
+ diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
542
+ n -= 8, a += 8, b += 8;
427
543
  }
544
+ sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
545
+ sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
546
+ if (n)
547
+ goto simsimd_l2sq_bf16_neon_cycle;
428
548
 
429
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
430
- simsimd_f32_t sum = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
431
- *result = sum;
549
+ *result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
432
550
  }
551
+
433
552
  #pragma clang attribute pop
434
553
  #pragma GCC pop_options
435
554
  #endif // SIMSIMD_TARGET_NEON_BF16
@@ -439,6 +558,11 @@ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16
439
558
  #pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
440
559
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
441
560
 
561
+ SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
562
+ simsimd_distance_t* result) {
563
+ simsimd_l2sq_i8_neon(a, b, n, result);
564
+ *result = _simsimd_sqrt_f32_neon(*result);
565
+ }
442
566
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
443
567
  simsimd_distance_t* result) {
444
568
  int32x4_t d2_vec = vdupq_n_s32(0);
@@ -455,7 +579,7 @@ SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t con
455
579
  }
456
580
  int32_t d2 = vaddvq_s32(d2_vec);
457
581
  for (; i < n; ++i) {
458
- int32_t n = a[i] - b[i];
582
+ int32_t n = (int32_t)a[i] - b[i];
459
583
  d2 += n * n;
460
584
  }
461
585
  *result = d2;
@@ -577,12 +701,7 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
577
701
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
578
702
  }
579
703
 
580
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
581
- simsimd_f32_t a2_b2_arr[2] = {(simsimd_f32_t)a2, (simsimd_f32_t)b2};
582
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
583
- a2_b2 = vrsqrte_f32(a2_b2);
584
- vst1_f32(a2_b2_arr, a2_b2);
585
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
704
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
586
705
  }
587
706
 
588
707
  #pragma clang attribute pop
@@ -591,9 +710,14 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
591
710
 
592
711
  #if SIMSIMD_TARGET_SVE
593
712
  #pragma GCC push_options
594
- #pragma GCC target("+sve")
595
- #pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
713
+ #pragma GCC target("arch=armv8.2-a+sve")
714
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
596
715
 
716
+ SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
717
+ simsimd_distance_t* result) {
718
+ simsimd_l2sq_f32_sve(a, b, n, result);
719
+ *result = _simsimd_sqrt_f64_neon(*result);
720
+ }
597
721
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
598
722
  simsimd_distance_t* result) {
599
723
  simsimd_size_t i = 0;
@@ -629,15 +753,14 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
629
753
  simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
630
754
  simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
631
755
  simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
632
-
633
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
634
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
635
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
636
- a2_b2 = vrsqrte_f32(a2_b2);
637
- vst1_f32(a2_b2_arr, a2_b2);
638
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
756
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
639
757
  }
640
758
 
759
+ SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
760
+ simsimd_distance_t* result) {
761
+ simsimd_l2sq_f64_sve(a, b, n, result);
762
+ *result = _simsimd_sqrt_f64_neon(*result);
763
+ }
641
764
  SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
642
765
  simsimd_distance_t* result) {
643
766
  simsimd_size_t i = 0;
@@ -673,22 +796,23 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
673
796
  simsimd_f64_t ab = svaddv_f64(svptrue_b32(), ab_vec);
674
797
  simsimd_f64_t a2 = svaddv_f64(svptrue_b32(), a2_vec);
675
798
  simsimd_f64_t b2 = svaddv_f64(svptrue_b32(), b2_vec);
676
-
677
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
678
- simsimd_f64_t a2_b2_arr[2] = {a2, b2};
679
- float64x2_t a2_b2 = vld1q_f64(a2_b2_arr);
680
- a2_b2 = vrsqrteq_f64(a2_b2);
681
- vst1q_f64(a2_b2_arr, a2_b2);
682
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
799
+ *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
683
800
  }
684
801
 
685
802
  #pragma clang attribute pop
686
803
  #pragma GCC pop_options
804
+ #endif // SIMSIMD_TARGET_SVE
687
805
 
806
+ #if SIMSIMD_TARGET_SVE_F16
688
807
  #pragma GCC push_options
689
- #pragma GCC target("+sve+fp16")
690
- #pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
808
+ #pragma GCC target("arch=armv8.2-a+sve+fp16")
809
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
691
810
 
811
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
812
+ simsimd_distance_t* result) {
813
+ simsimd_l2sq_f16_sve(a, b, n, result);
814
+ *result = _simsimd_sqrt_f32_neon(*result);
815
+ }
692
816
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
693
817
  simsimd_distance_t* result) {
694
818
  simsimd_size_t i = 0;
@@ -728,230 +852,283 @@ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
728
852
  simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
729
853
  simsimd_f16_for_arm_simd_t a2 = svaddv_f16(svptrue_b16(), a2_vec);
730
854
  simsimd_f16_for_arm_simd_t b2 = svaddv_f16(svptrue_b16(), b2_vec);
855
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
856
+ }
731
857
 
732
- // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
733
- simsimd_f32_t a2_b2_arr[2] = {a2, b2};
734
- float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
735
- a2_b2 = vrsqrte_f32(a2_b2);
736
- vst1_f32(a2_b2_arr, a2_b2);
737
- *result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
858
+ #pragma clang attribute pop
859
+ #pragma GCC pop_options
860
+ #endif // SIMSIMD_TARGET_SVE_F16
861
+
862
+ #if SIMSIMD_TARGET_SVE_BF16
863
+ #pragma GCC push_options
864
+ #pragma GCC target("arch=armv8.2-a+sve+bf16")
865
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
866
+
867
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
868
+ simsimd_distance_t* result) {
869
+ simsimd_l2sq_bf16_sve(a, b, n, result);
870
+ *result = _simsimd_sqrt_f32_neon(*result);
871
+ }
872
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_bf16_t const* b_enum, simsimd_size_t n,
873
+ simsimd_distance_t* result) {
874
+ simsimd_size_t i = 0;
875
+ svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
876
+ svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
877
+ simsimd_u16_t const* a = (simsimd_u16_t const*)(a_enum);
878
+ simsimd_u16_t const* b = (simsimd_u16_t const*)(b_enum);
879
+ do {
880
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
881
+ svuint16_t a_vec = svld1_u16(pg_vec, a + i);
882
+ svuint16_t b_vec = svld1_u16(pg_vec, b + i);
883
+
884
+ // There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
885
+ svbool_t pg_low_vec = svwhilelt_b32((unsigned int)(i), (unsigned int)n);
886
+ svbool_t pg_high_vec = svwhilelt_b32((unsigned int)(i + svcnth() / 2), (unsigned int)n);
887
+ svfloat32_t a_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(a_vec), 16));
888
+ svfloat32_t a_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(a_vec), 16));
889
+ svfloat32_t b_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(b_vec), 16));
890
+ svfloat32_t b_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(b_vec), 16));
891
+
892
+ svfloat32_t a_minus_b_low_vec = svsub_f32_x(pg_low_vec, a_low_vec, b_low_vec);
893
+ svfloat32_t a_minus_b_high_vec = svsub_f32_x(pg_high_vec, a_high_vec, b_high_vec);
894
+ d2_low_vec = svmla_f32_x(pg_vec, d2_low_vec, a_minus_b_low_vec, a_minus_b_low_vec);
895
+ d2_high_vec = svmla_f32_x(pg_vec, d2_high_vec, a_minus_b_low_vec, a_minus_b_low_vec);
896
+ i += svcnth();
897
+ } while (i < n);
898
+ simsimd_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_vec) + svaddv_f32(svptrue_b32(), d2_high_vec);
899
+ *result = d2;
900
+ }
901
+
902
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_bf16_t const* b_enum, simsimd_size_t n,
903
+ simsimd_distance_t* result) {
904
+ simsimd_size_t i = 0;
905
+ svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
906
+ svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
907
+ svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
908
+ simsimd_bf16_for_arm_simd_t const* a = (simsimd_bf16_for_arm_simd_t const*)(a_enum);
909
+ simsimd_bf16_for_arm_simd_t const* b = (simsimd_bf16_for_arm_simd_t const*)(b_enum);
910
+ do {
911
+ svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
912
+ svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
913
+ svbfloat16_t b_vec = svld1_bf16(pg_vec, b + i);
914
+ ab_vec = svbfdot_f32(ab_vec, a_vec, b_vec);
915
+ a2_vec = svbfdot_f32(a2_vec, a_vec, a_vec);
916
+ b2_vec = svbfdot_f32(b2_vec, b_vec, b_vec);
917
+ i += svcnth();
918
+ } while (i < n);
919
+
920
+ simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
921
+ simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
922
+ simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
923
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
738
924
  }
739
925
 
740
926
  #pragma clang attribute pop
741
927
  #pragma GCC pop_options
742
- #endif // SIMSIMD_TARGET_SVE
928
+ #endif // SIMSIMD_TARGET_SVE_BF16
743
929
  #endif // SIMSIMD_TARGET_ARM
744
930
 
931
+ #if SIMSIMD_TARGET_X86
932
+ #if SIMSIMD_TARGET_HASWELL
933
+ #pragma GCC push_options
934
+ #pragma GCC target("avx2")
935
+ #pragma clang attribute push(__attribute__((target("avx2"))), apply_to = function)
936
+
937
+ SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_haswell(simsimd_f32_t x) {
938
+ return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set_ss(x)));
939
+ }
940
+ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_haswell(simsimd_f64_t x) {
941
+ return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set_sd(x)));
942
+ }
943
+
944
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f64_t ab, simsimd_f64_t a2,
945
+ simsimd_f64_t b2) {
946
+
947
+ // If both vectors have magnitude 0, the distance is 0.
948
+ if (a2 == 0 && b2 == 0)
949
+ return 0;
950
+ // If any one of the vectors is 0, the square root of the product is 0,
951
+ // the division is illformed, and the result is 1.
952
+ else if (ab == 0)
953
+ return 1;
954
+ // We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
955
+ // https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
956
+ // The latency of the native instruction is 4 cycles and it's broadly supported.
957
+ // For single-precision floats it has a maximum relative error of 1.5*2^-12.
958
+ // Higher precision isn't implemented on older CPUs. See `_simsimd_cos_normalize_f64_skylake` for that.
959
+ __m128d squares = _mm_set_pd(a2, b2);
960
+ __m128d rsqrts = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(squares)));
961
+ // Newton-Raphson iteration for reciprocal square root:
962
+ // https://en.wikipedia.org/wiki/Newton%27s_method
963
+ rsqrts = _mm_add_pd( //
964
+ _mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
965
+ _mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
966
+
967
+ simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
968
+ simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
969
+ simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
970
+ return result > 0 ? result : 0;
971
+ }
972
+
973
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f32_t ab, simsimd_f32_t a2,
974
+ simsimd_f32_t b2) {
975
+
976
+ // If both vectors have magnitude 0, the distance is 0.
977
+ if (a2 == 0.0f && b2 == 0.0f)
978
+ return 0.0f;
979
+ // If any one of the vectors is 0, the square root of the product is 0,
980
+ // the division is illformed, and the result is 1.
981
+ else if (ab == 0.0f)
982
+ return 1.0f;
983
+
984
+ // Load the squares into an __m128 register for single-precision floating-point operations
985
+ __m128 squares = _mm_set_ps(a2, b2, a2, b2); // We replicate to make use of full register
986
+
987
+ // Compute the reciprocal square root of the squares using _mm_rsqrt_ps (single-precision)
988
+ __m128 rsqrts = _mm_rsqrt_ps(squares);
989
+
990
+ // Perform one iteration of Newton-Raphson refinement to improve the precision of rsqrt:
991
+ // Formula: y' = y * (1.5 - 0.5 * x * y * y)
992
+ __m128 half = _mm_set1_ps(0.5f);
993
+ __m128 three_halves = _mm_set1_ps(1.5f);
994
+ rsqrts =
995
+ _mm_mul_ps(rsqrts, _mm_sub_ps(three_halves, _mm_mul_ps(half, _mm_mul_ps(squares, _mm_mul_ps(rsqrts, rsqrts)))));
996
+
997
+ // Extract the reciprocal square roots of a2 and b2 from the __m128 register
998
+ simsimd_f32_t a2_reciprocal = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
999
+ simsimd_f32_t b2_reciprocal = _mm_cvtss_f32(rsqrts);
1000
+
1001
+ // Calculate the cosine distance: 1 - ab * a2_reciprocal * b2_reciprocal
1002
+ simsimd_distance_t result = 1.0f - ab * a2_reciprocal * b2_reciprocal;
1003
+ return result > 0 ? result : 0;
1004
+ }
1005
+
1006
+ #pragma clang attribute pop
1007
+ #pragma GCC pop_options
1008
+ #endif // SIMSIMD_TARGET_HASWELL
1009
+ #endif // SIMSIMD_TARGET_X86
1010
+
745
1011
  #if SIMSIMD_TARGET_X86
746
1012
  #if SIMSIMD_TARGET_HASWELL
747
1013
  #pragma GCC push_options
748
1014
  #pragma GCC target("avx2", "f16c", "fma")
749
1015
  #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
750
1016
 
1017
+ SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1018
+ simsimd_distance_t* result) {
1019
+ simsimd_l2sq_f16_haswell(a, b, n, result);
1020
+ *result = _simsimd_sqrt_f32_haswell(*result);
1021
+ }
751
1022
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
752
1023
  simsimd_distance_t* result) {
1024
+ __m256 a_vec, b_vec;
753
1025
  __m256 d2_vec = _mm256_setzero_ps();
754
- simsimd_size_t i = 0;
755
- for (; i + 8 <= n; i += 8) {
756
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
757
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
758
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
759
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
760
- }
761
1026
 
762
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
763
- // function will run. It is extremely slow, so even for the tail, let's combine serial
764
- // loads and stores with vectorized math.
765
- if (i < n) {
766
- union {
767
- __m128i f16_vec;
768
- simsimd_f16_t f16[8];
769
- } a_padded_tail, b_padded_tail;
770
- simsimd_size_t j = 0;
771
- for (; i < n; ++i, ++j)
772
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
773
- for (; j < 8; ++j)
774
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
775
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
776
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
777
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
778
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1027
+ simsimd_l2sq_f16_haswell_cycle:
1028
+ if (n < 8) {
1029
+ a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1030
+ b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1031
+ n = 0;
1032
+ } else {
1033
+ a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
1034
+ b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
1035
+ n -= 8, a += 8, b += 8;
779
1036
  }
1037
+ __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1038
+ d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1039
+ if (n)
1040
+ goto simsimd_l2sq_f16_haswell_cycle;
780
1041
 
781
- d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
782
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
783
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
784
-
785
- simsimd_f32_t f32_result;
786
- _mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
787
- *result = f32_result;
1042
+ *result = _simsimd_reduce_f32x8_haswell(d2_vec);
788
1043
  }
789
1044
 
790
1045
  SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
791
1046
  simsimd_distance_t* result) {
792
-
1047
+ __m256 a_vec, b_vec;
793
1048
  __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
794
- simsimd_size_t i = 0;
795
- for (; i + 8 <= n; i += 8) {
796
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
797
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
798
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
799
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
800
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
801
- }
802
1049
 
803
- // In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
804
- // function will run. It is extremely slow, so even for the tail, let's combine serial
805
- // loads and stores with vectorized math.
806
- if (i < n) {
807
- union {
808
- __m128i f16_vec;
809
- simsimd_f16_t f16[8];
810
- } a_padded_tail, b_padded_tail;
811
- simsimd_size_t j = 0;
812
- for (; i < n; ++i, ++j)
813
- a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
814
- for (; j < 8; ++j)
815
- a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
816
- __m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
817
- __m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
818
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
819
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
820
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1050
+ simsimd_cos_f16_haswell_cycle:
1051
+ if (n < 8) {
1052
+ a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1053
+ b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1054
+ n = 0;
1055
+ } else {
1056
+ a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
1057
+ b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
1058
+ n -= 8, a += 8, b += 8;
821
1059
  }
1060
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1061
+ a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1062
+ b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1063
+ if (n)
1064
+ goto simsimd_cos_f16_haswell_cycle;
822
1065
 
823
- // Horizontal reductions:
824
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
825
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
826
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
827
-
828
- a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
829
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
830
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
831
-
832
- b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
833
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
834
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
835
-
836
- simsimd_f32_t ab, a2, b2;
837
- _mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
838
- _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
839
- _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
840
-
841
- // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
842
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
843
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
844
- __m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
845
- result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
846
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1066
+ simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1067
+ simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1068
+ simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1069
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
847
1070
  }
848
1071
 
1072
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1073
+ simsimd_distance_t* result) {
1074
+ simsimd_l2sq_bf16_haswell(a, b, n, result);
1075
+ *result = _simsimd_sqrt_f32_haswell(*result);
1076
+ }
849
1077
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
850
1078
  simsimd_distance_t* result) {
1079
+ __m256 a_vec, b_vec;
851
1080
  __m256 d2_vec = _mm256_setzero_ps();
852
- simsimd_size_t i = 0;
853
- for (; i + 8 <= n; i += 8) {
854
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
855
- // x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
856
- __m256 a_vec =
857
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
858
- __m256 b_vec =
859
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
860
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
861
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
862
- }
863
1081
 
864
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
865
- // function will run. It is extremely slow, so even for the tail, let's combine serial
866
- // loads and stores with vectorized math.
867
- if (i < n) {
868
- union {
869
- __m128i bf16_vec;
870
- simsimd_bf16_t bf16[8];
871
- } a_padded_tail, b_padded_tail;
872
- simsimd_size_t j = 0;
873
- for (; i < n; ++i, ++j)
874
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
875
- for (; j < 8; ++j)
876
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
877
- __m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
878
- __m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
879
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
880
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1082
+ simsimd_l2sq_bf16_haswell_cycle:
1083
+ if (n < 8) {
1084
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1085
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1086
+ n = 0;
1087
+ } else {
1088
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)a));
1089
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)b));
1090
+ n -= 8, a += 8, b += 8;
881
1091
  }
1092
+ __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1093
+ d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1094
+ if (n)
1095
+ goto simsimd_l2sq_bf16_haswell_cycle;
882
1096
 
883
- d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
884
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
885
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
886
-
887
- simsimd_f32_t f32_result;
888
- _mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
889
- *result = f32_result;
1097
+ *result = _simsimd_reduce_f32x8_haswell(d2_vec);
890
1098
  }
891
1099
 
892
1100
  SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
893
1101
  simsimd_distance_t* result) {
894
-
1102
+ __m256 a_vec, b_vec;
895
1103
  __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
896
- simsimd_size_t i = 0;
897
- for (; i + 8 <= n; i += 8) {
898
- // Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
899
- // x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
900
- __m256 a_vec =
901
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
902
- __m256 b_vec =
903
- _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
904
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
905
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
906
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
907
- }
908
1104
 
909
- // In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
910
- // function will run. It is extremely slow, so even for the tail, let's combine serial
911
- // loads and stores with vectorized math.
912
- if (i < n) {
913
- union {
914
- __m128i bf16_vec;
915
- simsimd_bf16_t bf16[8];
916
- } a_padded_tail, b_padded_tail;
917
- simsimd_size_t j = 0;
918
- for (; i < n; ++i, ++j)
919
- a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
920
- for (; j < 8; ++j)
921
- a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
922
- __m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
923
- __m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
924
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
925
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
926
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1105
+ simsimd_cos_bf16_haswell_cycle:
1106
+ if (n < 8) {
1107
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1108
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1109
+ n = 0;
1110
+ } else {
1111
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)a));
1112
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)b));
1113
+ n -= 8, a += 8, b += 8;
927
1114
  }
1115
+ ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1116
+ a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1117
+ b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1118
+ if (n)
1119
+ goto simsimd_cos_bf16_haswell_cycle;
928
1120
 
929
- // Horizontal reductions:
930
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
931
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
932
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
933
-
934
- a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
935
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
936
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
937
-
938
- b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
939
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
940
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
941
-
942
- simsimd_f32_t ab, a2, b2;
943
- _mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
944
- _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
945
- _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
946
-
947
- // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
948
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
949
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
950
- __m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
951
- result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
952
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1121
+ simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1122
+ simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1123
+ simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1124
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
953
1125
  }
954
1126
 
1127
+ SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1128
+ simsimd_distance_t* result) {
1129
+ simsimd_l2sq_i8_haswell(a, b, n, result);
1130
+ *result = _simsimd_sqrt_f32_haswell(*result);
1131
+ }
955
1132
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
956
1133
  simsimd_distance_t* result) {
957
1134
 
@@ -1053,16 +1230,14 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
1053
1230
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1054
1231
  }
1055
1232
 
1056
- // Compute the reciprocal of the square roots
1057
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
1058
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
1059
-
1060
- // Compute cosine similarity: ab / sqrt(a2 * b2)
1061
- __m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
1062
- __m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
1063
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1233
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1064
1234
  }
1065
1235
 
1236
+ SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1237
+ simsimd_distance_t* result) {
1238
+ simsimd_l2sq_f32_haswell(a, b, n, result);
1239
+ *result = _simsimd_sqrt_f32_haswell(*result);
1240
+ }
1066
1241
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1067
1242
  simsimd_distance_t* result) {
1068
1243
 
@@ -1075,7 +1250,7 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
1075
1250
  d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1076
1251
  }
1077
1252
 
1078
- simsimd_f64_t d2 = _mm256_reduce_add_ps_dbl(d2_vec);
1253
+ simsimd_f64_t d2 = _simsimd_reduce_f32x8_haswell(d2_vec);
1079
1254
  for (; i < n; ++i) {
1080
1255
  float d = a[i] - b[i];
1081
1256
  d2 += d * d;
@@ -1098,22 +1273,15 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
1098
1273
  a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1099
1274
  b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1100
1275
  }
1101
- simsimd_f64_t ab = _mm256_reduce_add_ps_dbl(ab_vec);
1102
- simsimd_f64_t a2 = _mm256_reduce_add_ps_dbl(a2_vec);
1103
- simsimd_f64_t b2 = _mm256_reduce_add_ps_dbl(b2_vec);
1276
+
1277
+ simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1278
+ simsimd_f64_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
1279
+ simsimd_f64_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
1104
1280
  for (; i < n; ++i) {
1105
1281
  float ai = a[i], bi = b[i];
1106
1282
  ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1107
1283
  }
1108
-
1109
- // Compute the reciprocal of the square roots
1110
- __m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
1111
- __m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
1112
-
1113
- // Compute cosine similarity: ab / sqrt(a2 * b2)
1114
- __m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
1115
- __m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
1116
- *result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
1284
+ *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1117
1285
  }
1118
1286
 
1119
1287
  #pragma clang attribute pop
@@ -1122,9 +1290,14 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
1122
1290
 
1123
1291
  #if SIMSIMD_TARGET_SKYLAKE
1124
1292
  #pragma GCC push_options
1125
- #pragma GCC target("avx512f", "avx512vl", "bmi2")
1126
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2"))), apply_to = function)
1293
+ #pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
1294
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
1127
1295
 
1296
+ SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1297
+ simsimd_distance_t* result) {
1298
+ simsimd_l2sq_f32_skylake(a, b, n, result);
1299
+ *result = _simsimd_sqrt_f64_haswell(*result);
1300
+ }
1128
1301
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1129
1302
  simsimd_distance_t* result) {
1130
1303
  __m512 d2_vec = _mm512_setzero();
@@ -1146,7 +1319,51 @@ simsimd_l2sq_f32_skylake_cycle:
1146
1319
  if (n)
1147
1320
  goto simsimd_l2sq_f32_skylake_cycle;
1148
1321
 
1149
- *result = _mm512_reduce_add_ps(d2_vec);
1322
+ *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1323
+ }
1324
+
1325
+ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f64_t ab, simsimd_f64_t a2,
1326
+ simsimd_f64_t b2) {
1327
+
1328
+ // If both vectors have magnitude 0, the distance is 0.
1329
+ if (a2 == 0 && b2 == 0)
1330
+ return 0;
1331
+ // If any one of the vectors is 0, the square root of the product is 0,
1332
+ // the division is illformed, and the result is 1.
1333
+ else if (ab == 0)
1334
+ return 1;
1335
+
1336
+ // We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
1337
+ // https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
1338
+ // The maximum relative error for this approximation is less than 2^-14, which is 6x lower than
1339
+ // for single-precision floats in the `_simsimd_cos_normalize_f64_haswell` implementation.
1340
+ // Mysteriously, MSVC has no `_mm_rsqrt14_pd` intrinsic, but has it's masked variants,
1341
+ // so let's use `_mm_maskz_rsqrt14_pd(0xFF, ...)` instead.
1342
+ __m128d squares = _mm_set_pd(a2, b2);
1343
+ __m128d rsqrts = _mm_maskz_rsqrt14_pd(0xFF, squares);
1344
+
1345
+ // Let's implement a single Newton-Raphson iteration to refine the result.
1346
+ // This is how it affects downstream applications:
1347
+ //
1348
+ // +--------+------+----------+---------------------+---------------------+---------------------+
1349
+ // | Metric | NDim | DType | Baseline Error | Old SimSIMD Error | New SimSIMD Error |
1350
+ // +--------+------+----------+---------------------+---------------------+---------------------+
1351
+ // | cosine | 1536 | bfloat16 | 1.89e-08 ± 1.59e-08 | 3.07e-07 ± 3.09e-07 | 3.53e-09 ± 2.70e-09 |
1352
+ // | cosine | 1536 | float16 | 1.67e-02 ± 1.44e-02 | 2.68e-05 ± 1.95e-05 | 2.02e-05 ± 1.39e-05 |
1353
+ // | cosine | 1536 | float32 | 2.21e-08 ± 1.65e-08 | 3.47e-07 ± 3.49e-07 | 3.77e-09 ± 2.84e-09 |
1354
+ // | cosine | 1536 | float64 | 0.00e+00 ± 0.00e+00 | 3.80e-07 ± 4.50e-07 | 1.35e-11 ± 1.85e-11 |
1355
+ // | cosine | 1536 | int8 | 0.00e+00 ± 0.00e+00 | 4.60e-04 ± 3.36e-04 | 4.20e-04 ± 4.88e-04 |
1356
+ // +--------+------+----------+---------------------+---------------------+---------------------+
1357
+ //
1358
+ // https://en.wikipedia.org/wiki/Newton%27s_method
1359
+ rsqrts = _mm_add_pd( //
1360
+ _mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
1361
+ _mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
1362
+
1363
+ simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
1364
+ simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
1365
+ simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
1366
+ return result > 0 ? result : 0;
1150
1367
  }
1151
1368
 
1152
1369
  SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
@@ -1173,19 +1390,17 @@ simsimd_cos_f32_skylake_cycle:
1173
1390
  if (n)
1174
1391
  goto simsimd_cos_f32_skylake_cycle;
1175
1392
 
1176
- simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
1177
- simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
1178
- simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
1179
-
1180
- // Compute the reciprocal square roots of a2 and b2
1181
- // Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
1182
- // so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
1183
- __m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1184
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1185
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1186
- *result = 1 - ab * rsqrt_a2 * rsqrt_b2;
1393
+ simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1394
+ simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
1395
+ simsimd_f64_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
1396
+ *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1187
1397
  }
1188
1398
 
1399
+ SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1400
+ simsimd_distance_t* result) {
1401
+ simsimd_l2sq_f64_skylake(a, b, n, result);
1402
+ *result = _simsimd_sqrt_f64_haswell(*result);
1403
+ }
1189
1404
  SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1190
1405
  simsimd_distance_t* result) {
1191
1406
  __m512d d2_vec = _mm512_setzero_pd();
@@ -1234,17 +1449,10 @@ simsimd_cos_f64_skylake_cycle:
1234
1449
  if (n)
1235
1450
  goto simsimd_cos_f64_skylake_cycle;
1236
1451
 
1237
- simsimd_f32_t ab = (simsimd_f32_t)_mm512_reduce_add_pd(ab_vec);
1238
- simsimd_f32_t a2 = (simsimd_f32_t)_mm512_reduce_add_pd(a2_vec);
1239
- simsimd_f32_t b2 = (simsimd_f32_t)_mm512_reduce_add_pd(b2_vec);
1240
-
1241
- // Compute the reciprocal square roots of a2 and b2
1242
- // Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
1243
- // so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
1244
- __m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1245
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1246
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1247
- *result = 1 - ab * rsqrt_a2 * rsqrt_b2;
1452
+ simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
1453
+ simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
1454
+ simsimd_f64_t b2 = _mm512_reduce_add_pd(b2_vec);
1455
+ *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1248
1456
  }
1249
1457
 
1250
1458
  #pragma clang attribute pop
@@ -1253,15 +1461,71 @@ simsimd_cos_f64_skylake_cycle:
1253
1461
 
1254
1462
  #if SIMSIMD_TARGET_GENOA
1255
1463
  #pragma GCC push_options
1256
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1257
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), apply_to = function)
1464
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1465
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1466
+ apply_to = function)
1467
+
1468
+ SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
1469
+
1470
+ union {
1471
+ __m512 fvec;
1472
+ __m512i ivec;
1473
+ simsimd_f32_t f32[16];
1474
+ simsimd_u16_t u16[32];
1475
+ simsimd_bf16_t bf16[32];
1476
+ } d_odd, d_even, d, a_f32_even, b_f32_even, d_f32_even, a_f32_odd, b_f32_odd, d_f32_odd, a, b;
1477
+ a.ivec = a_i16;
1478
+ b.ivec = b_i16;
1479
+
1480
+ // There are several approaches to perform subtraction in `bf16`. The first one is:
1481
+ //
1482
+ // Perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
1483
+ // expand it to 32-bit integers, then shift the bits by 16 to the left.
1484
+ // Then subtract as floats, and shift back. During expansion, we will double the space,
1485
+ // and should use separate registers for top and bottom halves.
1486
+ // Some compilers don't have `_mm512_extracti32x8_epi32`, so we use `_mm512_extracti64x4_epi64`:
1487
+ //
1488
+ // a_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
1489
+ // _mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16)), 16));
1490
+ // b_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
1491
+ // _mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16)), 16));
1492
+ // a_f32_top.fvec =_mm512_castsi512_ps(
1493
+ // _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16, 1)), 16));
1494
+ // b_f32_top.fvec =_mm512_castsi512_ps(
1495
+ // _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16, 1)), 16));
1496
+ // d_f32_top.fvec = _mm512_sub_ps(a_f32_top.fvec, b_f32_top.fvec);
1497
+ // d_f32_bot.fvec = _mm512_sub_ps(a_f32_bot.fvec, b_f32_bot.fvec);
1498
+ // d.ivec = _mm512_castsi256_si512(_mm512_cvtepi32_epi16(
1499
+ // _mm512_srli_epi32(_mm512_castps_si512(d_f32_bot.fvec), 16)));
1500
+ // d.ivec = _mm512_inserti64x4(d.ivec, _mm512_cvtepi32_epi16(
1501
+ // _mm512_srli_epi32(_mm512_castps_si512(d_f32_top.fvec), 16)), 1);
1502
+ //
1503
+ // Instead of using multple shifts and an insertion, we can achieve similar result with fewer expensive
1504
+ // calls to `_mm512_permutex2var_epi16`, or a cheap `_mm512_mask_shuffle_epi8` and blend:
1505
+ //
1506
+ a_f32_odd.ivec = _mm512_and_si512(a_i16, _mm512_set1_epi32(0xFFFF0000));
1507
+ a_f32_even.ivec = _mm512_slli_epi32(a_i16, 16);
1508
+ b_f32_odd.ivec = _mm512_and_si512(b_i16, _mm512_set1_epi32(0xFFFF0000));
1509
+ b_f32_even.ivec = _mm512_slli_epi32(b_i16, 16);
1258
1510
 
1511
+ d_f32_odd.fvec = _mm512_sub_ps(a_f32_odd.fvec, b_f32_odd.fvec);
1512
+ d_f32_even.fvec = _mm512_sub_ps(a_f32_even.fvec, b_f32_even.fvec);
1513
+
1514
+ d_f32_even.ivec = _mm512_srli_epi32(d_f32_even.ivec, 16);
1515
+ d.ivec = _mm512_mask_blend_epi16(0x55555555, d_f32_odd.ivec, d_f32_even.ivec);
1516
+
1517
+ return d.ivec;
1518
+ }
1519
+
1520
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1521
+ simsimd_distance_t* result) {
1522
+ simsimd_l2sq_bf16_genoa(a, b, n, result);
1523
+ *result = _simsimd_sqrt_f32_haswell(*result);
1524
+ }
1259
1525
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1260
1526
  simsimd_distance_t* result) {
1261
- __m512 d2_top_vec = _mm512_setzero_ps(), d2_bot_vec = _mm512_setzero_ps();
1262
- __m512 d_top_vec = _mm512_setzero_ps(), d_bot_vec = _mm512_setzero_ps();
1263
- __m512 a_f32_top_vec, a_f32_bot_vec, b_f32_top_vec, b_f32_bot_vec;
1264
- __m512i a_i16_vec, b_i16_vec;
1527
+ __m512 d2_vec = _mm512_setzero_ps();
1528
+ __m512i a_i16_vec, b_i16_vec, d_i16_vec;
1265
1529
 
1266
1530
  simsimd_l2sq_bf16_genoa_cycle:
1267
1531
  if (n < 32) {
@@ -1274,35 +1538,12 @@ simsimd_l2sq_bf16_genoa_cycle:
1274
1538
  b_i16_vec = _mm512_loadu_epi16(b);
1275
1539
  a += 32, b += 32, n -= 32;
1276
1540
  }
1277
- // Let's perform the subtraction with single-precision, while the dot-product with half-precision.
1278
- // For that we need to perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
1279
- // expand it to 32-bit integers, then shift the bits by 16 to the left. Then subtract as floats,
1280
- // and shift back. During expansion, we will double the space, and should use separate registers
1281
- // for top and bottom halves.
1282
- a_f32_bot_vec =
1283
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16_vec)), 16));
1284
- b_f32_bot_vec =
1285
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16_vec)), 16));
1286
-
1287
- // Some compilers don't have `_mm512_extracti32x8_epi32`, so we need to use `_mm512_extracti64x4_epi64`
1288
- a_f32_top_vec =
1289
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16_vec, 1)), 16));
1290
- b_f32_top_vec =
1291
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16_vec, 1)), 16));
1292
-
1293
- // Subtract and cast back
1294
- d_top_vec = _mm512_sub_ps(a_f32_top_vec, b_f32_top_vec);
1295
- d_bot_vec = _mm512_sub_ps(a_f32_bot_vec, b_f32_bot_vec);
1296
- d_top_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_top_vec), 16));
1297
- d_bot_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_bot_vec), 16));
1298
-
1299
- // Square and accumulate
1300
- d2_top_vec = _mm512_dpbf16_ps(d2_top_vec, (__m512bh)(d_top_vec), (__m512bh)(d_top_vec));
1301
- d2_bot_vec = _mm512_dpbf16_ps(d2_bot_vec, (__m512bh)(d_bot_vec), (__m512bh)(d_bot_vec));
1541
+ d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
1542
+ d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
1302
1543
  if (n)
1303
1544
  goto simsimd_l2sq_bf16_genoa_cycle;
1304
1545
 
1305
- *result = _mm512_reduce_add_ps(d2_top_vec) + _mm512_reduce_add_ps(d2_bot_vec);
1546
+ *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1306
1547
  }
1307
1548
 
1308
1549
  SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
@@ -1329,15 +1570,10 @@ simsimd_cos_bf16_genoa_cycle:
1329
1570
  if (n)
1330
1571
  goto simsimd_cos_bf16_genoa_cycle;
1331
1572
 
1332
- simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
1333
- simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
1334
- simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
1335
-
1336
- // Compute the reciprocal square roots of a2 and b2
1337
- __m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1338
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1339
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1340
- *result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
1573
+ simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1574
+ simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
1575
+ simsimd_f32_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
1576
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1341
1577
  }
1342
1578
 
1343
1579
  #pragma clang attribute pop
@@ -1346,9 +1582,14 @@ simsimd_cos_bf16_genoa_cycle:
1346
1582
 
1347
1583
  #if SIMSIMD_TARGET_SAPPHIRE
1348
1584
  #pragma GCC push_options
1349
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512fp16")
1350
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
1585
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
1586
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
1351
1587
 
1588
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1589
+ simsimd_distance_t* result) {
1590
+ simsimd_l2sq_f16_sapphire(a, b, n, result);
1591
+ *result = _simsimd_sqrt_f32_haswell(*result);
1592
+ }
1352
1593
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1353
1594
  simsimd_distance_t* result) {
1354
1595
  __m512h d2_vec = _mm512_setzero_ph();
@@ -1400,12 +1641,7 @@ simsimd_cos_f16_sapphire_cycle:
1400
1641
  simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
1401
1642
  simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
1402
1643
  simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
1403
-
1404
- // Compute the reciprocal square roots of a2 and b2
1405
- __m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1406
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1407
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1408
- *result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
1644
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1409
1645
  }
1410
1646
 
1411
1647
  #pragma clang attribute pop
@@ -1414,16 +1650,22 @@ simsimd_cos_f16_sapphire_cycle:
1414
1650
 
1415
1651
  #if SIMSIMD_TARGET_ICE
1416
1652
  #pragma GCC push_options
1417
- #pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1418
- #pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), apply_to = function)
1419
-
1653
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1654
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1655
+ apply_to = function)
1656
+
1657
+ SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1658
+ simsimd_distance_t* result) {
1659
+ simsimd_l2sq_i8_ice(a, b, n, result);
1660
+ *result = _simsimd_sqrt_f32_haswell(*result);
1661
+ }
1420
1662
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1421
1663
  simsimd_distance_t* result) {
1422
1664
  __m512i d2_i32s_vec = _mm512_setzero_si512();
1423
1665
  __m512i a_vec, b_vec, d_i16s_vec;
1424
1666
 
1425
1667
  simsimd_l2sq_i8_ice_cycle:
1426
- if (n < 32) {
1668
+ if (n < 32) { // TODO: Avoid easrly i16 upcast to step through 64 values at a time
1427
1669
  __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1428
1670
  a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1429
1671
  b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
@@ -1488,14 +1730,7 @@ simsimd_cos_i8_ice_cycle:
1488
1730
  int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_low_i32s_vec, ab_high_i32s_vec));
1489
1731
  int a2 = _mm512_reduce_add_epi32(a2_i32s_vec);
1490
1732
  int b2 = _mm512_reduce_add_epi32(b2_i32s_vec);
1491
-
1492
- // Compute the reciprocal square roots of a2 and b2
1493
- // Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
1494
- // so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
1495
- __m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
1496
- simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
1497
- simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
1498
- *result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
1733
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1499
1734
  }
1500
1735
 
1501
1736
  #pragma clang attribute pop