usearch 2.16.7 → 2.16.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,10 +5,12 @@
5
5
  * @date August 27, 2024
6
6
  *
7
7
  * Contains:
8
- * - Bilinear form multiplication
9
8
  * - Mahalanobis distance
9
+ * - Bilinear form multiplication
10
+ * - Bilinear form multiplication over complex numbers
10
11
  *
11
12
  * For datatypes:
13
+ * - 64-bit floating point numbers
12
14
  * - 32-bit floating point numbers
13
15
  * - 16-bit floating point numbers
14
16
  * - 16-bit brain-floating point numbers
@@ -17,6 +19,16 @@
17
19
  * - Arm: NEON
18
20
  * - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
19
21
  *
22
+ * Most kernels in this file are designed for BLAS level 2 operations, where the operands are
23
+ * a combination of matrices and vectors, generally forming a chain of multiplications.
24
+ * Most kernels exploit the fact that matrix multiplication is associative, and the order of
25
+ * operations can be changed to minimize the number of operations: `(A * B) * C = A * (B * C)`.
26
+ * To optimize the performance, we minimize the number of memory accesses, and maximize the
27
+ * number of arithmetic operations, by using SIMD instructions.
28
+ *
29
+ * When A and C are vectors, and B is a matrix, we can load every element in B just once, and
30
+ * reuse it for every element in A and C.
31
+ *
20
32
  * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
21
33
  * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
22
34
  */
@@ -39,22 +51,29 @@ extern "C" {
39
51
  * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
40
52
  */
41
53
  SIMSIMD_PUBLIC void simsimd_bilinear_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
54
+ SIMSIMD_PUBLIC void simsimd_bilinear_f64c_serial(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
42
55
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
43
56
  SIMSIMD_PUBLIC void simsimd_bilinear_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
57
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32c_serial(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
44
58
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
45
59
  SIMSIMD_PUBLIC void simsimd_bilinear_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
60
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16c_serial(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
46
61
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
47
62
  SIMSIMD_PUBLIC void simsimd_bilinear_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
63
+ SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_serial(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
48
64
  SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
49
65
 
50
66
  /* Double-precision serial backends for all numeric types.
51
67
  * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
52
68
  */
53
69
  SIMSIMD_PUBLIC void simsimd_bilinear_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
70
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32c_accurate(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
54
71
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
55
72
  SIMSIMD_PUBLIC void simsimd_bilinear_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
73
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16c_accurate(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
56
74
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
57
75
  SIMSIMD_PUBLIC void simsimd_bilinear_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
76
+ SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_accurate(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
58
77
  SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
59
78
 
60
79
  /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
@@ -62,10 +81,13 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_accurate(simsimd_bf16_t const* a, s
62
81
  * server CPUs produced before 2023.
63
82
  */
64
83
  SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
84
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
65
85
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
66
86
  SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
87
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
67
88
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
68
89
  SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
90
+ SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
69
91
  SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
70
92
 
71
93
  /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
@@ -84,11 +106,17 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const* a, si
84
106
  * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
85
107
  * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
86
108
  */
109
+ SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
110
+ SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
111
+ SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
87
112
  SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
113
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
88
114
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
89
115
  SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
116
+ SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
90
117
  SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
91
118
  SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
119
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
92
120
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
93
121
  // clang-format on
94
122
 
@@ -98,56 +126,91 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const* a, sim
98
126
  simsimd_size_t n, simsimd_distance_t *result) { \
99
127
  simsimd_##accumulator_type##_t sum = 0; \
100
128
  for (simsimd_size_t i = 0; i != n; ++i) { \
101
- simsimd_##accumulator_type##_t partial = 0; \
129
+ simsimd_##accumulator_type##_t cb_j = 0; \
102
130
  simsimd_##accumulator_type##_t a_i = load_and_convert(a + i); \
103
131
  for (simsimd_size_t j = 0; j != n; ++j) { \
104
132
  simsimd_##accumulator_type##_t b_j = load_and_convert(b + j); \
105
133
  simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
106
- partial += c_ij * b_j; \
134
+ cb_j += c_ij * b_j; \
107
135
  } \
108
- sum += a_i * partial; \
136
+ sum += a_i * cb_j; \
109
137
  } \
110
138
  *result = (simsimd_distance_t)sum; \
111
139
  }
112
140
 
141
+ #define SIMSIMD_MAKE_COMPLEX_BILINEAR(name, input_type, accumulator_type, load_and_convert) \
142
+ SIMSIMD_PUBLIC void simsimd_bilinear_##input_type##_##name( \
143
+ simsimd_##input_type##_t const *a_pairs, simsimd_##input_type##_t const *b_pairs, \
144
+ simsimd_##input_type##_t const *c_pairs, simsimd_size_t n, simsimd_distance_t *results) { \
145
+ simsimd_##accumulator_type##_t sum_real = 0; \
146
+ simsimd_##accumulator_type##_t sum_imag = 0; \
147
+ for (simsimd_size_t i = 0; i != n; ++i) { \
148
+ simsimd_##accumulator_type##_t cb_j_real = 0; \
149
+ simsimd_##accumulator_type##_t cb_j_imag = 0; \
150
+ simsimd_##accumulator_type##_t a_i_real = load_and_convert(&(a_pairs + i)->real); \
151
+ simsimd_##accumulator_type##_t a_i_imag = load_and_convert(&(a_pairs + i)->imag); \
152
+ for (simsimd_size_t j = 0; j != n; ++j) { \
153
+ simsimd_##accumulator_type##_t b_j_real = load_and_convert(&(b_pairs + j)->real); \
154
+ simsimd_##accumulator_type##_t b_j_imag = load_and_convert(&(b_pairs + j)->imag); \
155
+ simsimd_##accumulator_type##_t c_ij_real = load_and_convert(&(c_pairs + i * n + j)->real); \
156
+ simsimd_##accumulator_type##_t c_ij_imag = load_and_convert(&(c_pairs + i * n + j)->imag); \
157
+ /* Complex multiplication: (c_ij * b_j) */ \
158
+ cb_j_real += c_ij_real * b_j_real - c_ij_imag * b_j_imag; \
159
+ cb_j_imag += c_ij_real * b_j_imag + c_ij_imag * b_j_real; \
160
+ } \
161
+ /* Complex multiplication: (a_i * cb_j) */ \
162
+ sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag; \
163
+ sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real; \
164
+ } \
165
+ results[0] = (simsimd_distance_t)sum_real; \
166
+ results[1] = (simsimd_distance_t)sum_imag; \
167
+ }
168
+
113
169
  #define SIMSIMD_MAKE_MAHALANOBIS(name, input_type, accumulator_type, load_and_convert) \
114
170
  SIMSIMD_PUBLIC void simsimd_mahalanobis_##input_type##_##name( \
115
171
  simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
116
172
  simsimd_size_t n, simsimd_distance_t *result) { \
117
173
  simsimd_##accumulator_type##_t sum = 0; \
118
174
  for (simsimd_size_t i = 0; i != n; ++i) { \
119
- simsimd_##accumulator_type##_t partial = 0; \
175
+ simsimd_##accumulator_type##_t cdiff_j = 0; \
120
176
  simsimd_##accumulator_type##_t diff_i = load_and_convert(a + i) - load_and_convert(b + i); \
121
177
  for (simsimd_size_t j = 0; j != n; ++j) { \
122
178
  simsimd_##accumulator_type##_t diff_j = load_and_convert(a + j) - load_and_convert(b + j); \
123
179
  simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
124
- partial += c_ij * diff_j; \
180
+ cdiff_j += c_ij * diff_j; \
125
181
  } \
126
- sum += diff_i * partial; \
182
+ sum += diff_i * cdiff_j; \
127
183
  } \
128
184
  *result = (simsimd_distance_t)SIMSIMD_SQRT(sum); \
129
185
  }
130
186
 
131
- SIMSIMD_MAKE_BILINEAR(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64_serial
132
- SIMSIMD_MAKE_MAHALANOBIS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f64_serial
187
+ SIMSIMD_MAKE_BILINEAR(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64_serial
188
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f64c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64c_serial
189
+ SIMSIMD_MAKE_MAHALANOBIS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f64_serial
133
190
 
134
- SIMSIMD_MAKE_BILINEAR(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_serial
135
- SIMSIMD_MAKE_MAHALANOBIS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_serial
191
+ SIMSIMD_MAKE_BILINEAR(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_serial
192
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f32c, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_serial
193
+ SIMSIMD_MAKE_MAHALANOBIS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_serial
136
194
 
137
- SIMSIMD_MAKE_BILINEAR(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_serial
138
- SIMSIMD_MAKE_MAHALANOBIS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_serial
195
+ SIMSIMD_MAKE_BILINEAR(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_serial
196
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f16c, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_serial
197
+ SIMSIMD_MAKE_MAHALANOBIS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_serial
139
198
 
140
- SIMSIMD_MAKE_BILINEAR(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_serial
141
- SIMSIMD_MAKE_MAHALANOBIS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_serial
199
+ SIMSIMD_MAKE_BILINEAR(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_serial
200
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, bf16c, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_serial
201
+ SIMSIMD_MAKE_MAHALANOBIS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_serial
142
202
 
143
- SIMSIMD_MAKE_BILINEAR(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_accurate
144
- SIMSIMD_MAKE_MAHALANOBIS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_accurate
203
+ SIMSIMD_MAKE_BILINEAR(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_accurate
204
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f32c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_accurate
205
+ SIMSIMD_MAKE_MAHALANOBIS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_accurate
145
206
 
146
- SIMSIMD_MAKE_BILINEAR(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_accurate
147
- SIMSIMD_MAKE_MAHALANOBIS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_accurate
207
+ SIMSIMD_MAKE_BILINEAR(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_accurate
208
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f16c, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_accurate
209
+ SIMSIMD_MAKE_MAHALANOBIS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_accurate
148
210
 
149
- SIMSIMD_MAKE_BILINEAR(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_accurate
150
- SIMSIMD_MAKE_MAHALANOBIS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_accurate
211
+ SIMSIMD_MAKE_BILINEAR(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_accurate
212
+ SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, bf16c, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_accurate
213
+ SIMSIMD_MAKE_MAHALANOBIS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_accurate
151
214
 
152
215
  #if _SIMSIMD_TARGET_ARM
153
216
  #if SIMSIMD_TARGET_NEON
@@ -160,25 +223,25 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const *a, simsimd_f3
160
223
  float32x4_t sum_vec = vdupq_n_f32(0);
161
224
  for (simsimd_size_t i = 0; i != n; ++i) {
162
225
  float32x4_t a_vec = vdupq_n_f32(a[i]);
163
- float32x4_t partial_sum_vec = vdupq_n_f32(0);
226
+ float32x4_t cb_j_vec = vdupq_n_f32(0);
164
227
  for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
165
228
  float32x4_t b_vec = vld1q_f32(b + j);
166
229
  float32x4_t c_vec = vld1q_f32(c + i * n + j);
167
- partial_sum_vec = vmlaq_f32(partial_sum_vec, b_vec, c_vec);
230
+ cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
168
231
  }
169
- sum_vec = vmlaq_f32(sum_vec, a_vec, partial_sum_vec);
232
+ sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
170
233
  }
171
234
 
172
235
  // Handle the tail of every row
173
236
  simsimd_f64_t sum = vaddvq_f32(sum_vec);
174
- simsimd_size_t tail_length = n % 4;
175
- simsimd_size_t tail_start = n - tail_length;
237
+ simsimd_size_t const tail_length = n % 4;
238
+ simsimd_size_t const tail_start = n - tail_length;
176
239
  if (tail_length) {
177
240
  for (simsimd_size_t i = 0; i != n; ++i) {
178
241
  simsimd_f32_t a_i = a[i];
179
- simsimd_f32_t partial_sum = 0;
180
- for (simsimd_size_t j = tail_start; j != n; ++j) partial_sum += b[j] * c[i * n + j];
181
- sum += a[i] * partial_sum;
242
+ simsimd_f32_t cb_j = 0;
243
+ for (simsimd_size_t j = tail_start; j != n; ++j) cb_j += b[j] * c[i * n + j];
244
+ sum += a[i] * cb_j;
182
245
  }
183
246
  }
184
247
 
@@ -190,35 +253,87 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const *a, simsimd
190
253
  float32x4_t sum_vec = vdupq_n_f32(0);
191
254
  for (simsimd_size_t i = 0; i != n; ++i) {
192
255
  float32x4_t diff_i_vec = vdupq_n_f32(a[i] - b[i]);
193
- float32x4_t partial_sum_vec = vdupq_n_f32(0);
256
+ float32x4_t cdiff_j_vec = vdupq_n_f32(0);
194
257
  for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
195
258
  float32x4_t diff_j_vec = vsubq_f32(vld1q_f32(a + j), vld1q_f32(b + j));
196
259
  float32x4_t c_vec = vld1q_f32(c + i * n + j);
197
- partial_sum_vec = vmlaq_f32(partial_sum_vec, diff_j_vec, c_vec);
260
+ cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
198
261
  }
199
262
 
200
- sum_vec = vmlaq_f32(sum_vec, diff_i_vec, partial_sum_vec);
263
+ sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
201
264
  }
202
265
 
203
266
  // Handle the tail of every row
204
267
  simsimd_f64_t sum = vaddvq_f32(sum_vec);
205
- simsimd_size_t tail_length = n % 4;
206
- simsimd_size_t tail_start = n - tail_length;
268
+ simsimd_size_t const tail_length = n % 4;
269
+ simsimd_size_t const tail_start = n - tail_length;
207
270
  if (tail_length) {
208
271
  for (simsimd_size_t i = 0; i != n; ++i) {
209
272
  simsimd_f32_t diff_i = a[i] - b[i];
210
- simsimd_f32_t partial_sum = 0;
273
+ simsimd_f32_t cdiff_j = 0;
211
274
  for (simsimd_size_t j = tail_start; j != n; ++j) {
212
275
  simsimd_f32_t diff_j = a[j] - b[j];
213
- partial_sum += diff_j * c[i * n + j];
276
+ cdiff_j += diff_j * c[i * n + j];
214
277
  }
215
- sum += diff_i * partial_sum;
278
+ sum += diff_i * cdiff_j;
216
279
  }
217
280
  }
218
281
 
219
282
  *result = _simsimd_sqrt_f64_neon(sum);
220
283
  }
221
284
 
285
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
286
+ simsimd_f32c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
287
+ simsimd_f32_t sum_real = 0;
288
+ simsimd_f32_t sum_imag = 0;
289
+ for (simsimd_size_t i = 0; i != n; ++i) {
290
+ simsimd_f32c_t a_i = a[i];
291
+ simsimd_f32c_t cb_j;
292
+ float32x4_t cb_j_real_vec = vdupq_n_f32(0);
293
+ float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
294
+ for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
295
+ // Unpack the input arrays into real and imaginary parts:
296
+ float32x4x2_t b_vec = vld2q_f32((simsimd_f32_t const *)(b + j));
297
+ float32x4x2_t c_vec = vld2q_f32((simsimd_f32_t const *)(c + i * n + j));
298
+ float32x4_t b_real_vec = b_vec.val[0];
299
+ float32x4_t b_imag_vec = b_vec.val[1];
300
+ float32x4_t c_real_vec = c_vec.val[0];
301
+ float32x4_t c_imag_vec = c_vec.val[1];
302
+
303
+ // Compute the dot product:
304
+ cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
305
+ cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
306
+ cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
307
+ cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
308
+ }
309
+ cb_j.real = vaddvq_f32(cb_j_real_vec);
310
+ cb_j.imag = vaddvq_f32(cb_j_imag_vec);
311
+ sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
312
+ sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
313
+ }
314
+
315
+ // Handle the tail of every row
316
+ simsimd_size_t const tail_length = n % 4;
317
+ simsimd_size_t const tail_start = n - tail_length;
318
+ if (tail_length) {
319
+ for (simsimd_size_t i = 0; i != n; ++i) {
320
+ simsimd_f32c_t a_i = a[i];
321
+ simsimd_f32c_t cb_j = {0, 0};
322
+ for (simsimd_size_t j = tail_start; j != n; ++j) {
323
+ simsimd_f32c_t b_j = b[j];
324
+ simsimd_f32c_t c_ij = c[i * n + j];
325
+ cb_j.real += b_j.real * c_ij.real - b_j.imag * c_ij.imag;
326
+ cb_j.imag += b_j.real * c_ij.imag + b_j.imag * c_ij.real;
327
+ }
328
+ sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
329
+ sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
330
+ }
331
+ }
332
+
333
+ results[0] = sum_real;
334
+ results[1] = sum_imag;
335
+ }
336
+
222
337
  #pragma clang attribute pop
223
338
  #pragma GCC pop_options
224
339
  #endif // SIMSIMD_TARGET_NEON
@@ -234,26 +349,26 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const *a, simsimd_f1
234
349
  for (simsimd_size_t i = 0; i != n; ++i) {
235
350
  // MSVC doesn't recognize `vdup_n_f16` as a valid intrinsic
236
351
  float32x4_t a_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
237
- float32x4_t partial_sum_vec = vdupq_n_f32(0);
352
+ float32x4_t cb_j_vec = vdupq_n_f32(0);
238
353
  for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
239
354
  float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
240
355
  float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
241
- partial_sum_vec = vmlaq_f32(partial_sum_vec, b_vec, c_vec);
356
+ cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
242
357
  }
243
- sum_vec = vmlaq_f32(sum_vec, a_vec, partial_sum_vec);
358
+ sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
244
359
  }
245
360
 
246
361
  // Handle the tail of every row
247
362
  simsimd_f64_t sum = vaddvq_f32(sum_vec);
248
- simsimd_size_t tail_length = n % 4;
249
- simsimd_size_t tail_start = n - tail_length;
363
+ simsimd_size_t const tail_length = n % 4;
364
+ simsimd_size_t const tail_start = n - tail_length;
250
365
  if (tail_length) {
251
366
  for (simsimd_size_t i = 0; i != n; ++i) {
252
367
  simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
253
368
  float32x4_t b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
254
369
  float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
255
- simsimd_f32_t partial_sum = vaddvq_f32(vmulq_f32(b_vec, c_vec));
256
- sum += a_i * partial_sum;
370
+ simsimd_f32_t cb_j = vaddvq_f32(vmulq_f32(b_vec, c_vec));
371
+ sum += a_i * cb_j;
257
372
  }
258
373
  }
259
374
 
@@ -268,21 +383,21 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const *a, simsimd
268
383
  float32x4_t a_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
269
384
  float32x4_t b_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(b + i))));
270
385
  float32x4_t diff_i_vec = vsubq_f32(a_i_vec, b_i_vec);
271
- float32x4_t partial_sum_vec = vdupq_n_f32(0);
386
+ float32x4_t cdiff_j_vec = vdupq_n_f32(0);
272
387
  for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
273
388
  float32x4_t a_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(a + j)));
274
389
  float32x4_t b_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
275
390
  float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
276
391
  float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
277
- partial_sum_vec = vmlaq_f32(partial_sum_vec, diff_j_vec, c_vec);
392
+ cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
278
393
  }
279
- sum_vec = vmlaq_f32(sum_vec, diff_i_vec, partial_sum_vec);
394
+ sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
280
395
  }
281
396
 
282
397
  // Handle the tail of every row
283
398
  simsimd_f32_t sum = vaddvq_f32(sum_vec);
284
- simsimd_size_t tail_length = n % 4;
285
- simsimd_size_t tail_start = n - tail_length;
399
+ simsimd_size_t const tail_length = n % 4;
400
+ simsimd_size_t const tail_start = n - tail_length;
286
401
  if (tail_length) {
287
402
  for (simsimd_size_t i = 0; i != n; ++i) {
288
403
  simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
@@ -292,14 +407,94 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const *a, simsimd
292
407
  float32x4_t b_j_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
293
408
  float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
294
409
  float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
295
- simsimd_f32_t partial_sum = vaddvq_f32(vmulq_f32(diff_j_vec, c_vec));
296
- sum += diff_i * partial_sum;
410
+ simsimd_f32_t cdiff_j = vaddvq_f32(vmulq_f32(diff_j_vec, c_vec));
411
+ sum += diff_i * cdiff_j;
297
412
  }
298
413
  }
299
414
 
300
415
  *result = _simsimd_sqrt_f32_neon(sum);
301
416
  }
302
417
 
418
+ SIMSIMD_INTERNAL simsimd_f32_t _simsimd_reduce_f16x8_neon(float16x8_t vec) {
419
+ // Split the 8-element vector into two 4-element vectors
420
+ float16x4_t low = vget_low_f16(vec); // Lower 4 elements
421
+ float16x4_t high = vget_high_f16(vec); // Upper 4 elements
422
+
423
+ // Add the lower and upper parts
424
+ float16x4_t sum = vadd_f16(low, high);
425
+
426
+ // Perform pairwise addition to reduce 4 elements to 2, then to 1
427
+ sum = vpadd_f16(sum, sum); // First reduction: 4 -> 2
428
+ sum = vpadd_f16(sum, sum); // Second reduction: 2 -> 1
429
+
430
+ // Convert the remaining half-precision value to single-precision and return
431
+ return vgetq_lane_f32(vcvt_f32_f16(sum), 0);
432
+ }
433
+
434
+ SIMSIMD_INTERNAL float16x8x2_t _simsimd_partial_load_f16x8x2_neon(simsimd_f16c_t const *x, simsimd_size_t n) {
435
+ union {
436
+ float16x8x2_t vecs;
437
+ simsimd_f16_t scalars[2][8];
438
+ } result;
439
+ simsimd_size_t i = 0;
440
+ for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
441
+ for (; i < 8; ++i) result.scalars[0][i] = 0, result.scalars[1][i] = 0;
442
+ return result.vecs;
443
+ }
444
+
445
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
446
+ simsimd_f16c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
447
+ simsimd_f32_t sum_real = 0;
448
+ simsimd_f32_t sum_imag = 0;
449
+ simsimd_size_t const tail_length = n % 8;
450
+ simsimd_size_t const tail_start = n - tail_length;
451
+ for (simsimd_size_t i = 0; i != n; ++i) {
452
+ simsimd_f32c_t a_i = {simsimd_f16_to_f32(&a[i].real), simsimd_f16_to_f32(&a[i].imag)};
453
+ float16x8_t cb_j_real_vec = vdupq_n_f16(0);
454
+ float16x8_t cb_j_imag_vec = vdupq_n_f16(0);
455
+ for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
456
+ // Unpack the input arrays into real and imaginary parts:
457
+ float16x8x2_t b_vec = vld2q_f16((float16_t const *)(b + j));
458
+ float16x8x2_t c_vec = vld2q_f16((float16_t const *)(c + i * n + j));
459
+ float16x8_t b_real_vec = b_vec.val[0];
460
+ float16x8_t b_imag_vec = b_vec.val[1];
461
+ float16x8_t c_real_vec = c_vec.val[0];
462
+ float16x8_t c_imag_vec = c_vec.val[1];
463
+
464
+ // Compute the dot product:
465
+ cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
466
+ cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
467
+ cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
468
+ cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
469
+ }
470
+ // Handle row tails
471
+ if (tail_length) {
472
+ // Unpack the input arrays into real and imaginary parts:
473
+ float16x8x2_t b_vec = _simsimd_partial_load_f16x8x2_neon(b + tail_start, tail_length);
474
+ float16x8x2_t c_vec = _simsimd_partial_load_f16x8x2_neon(c + i * n + tail_start, tail_length);
475
+ float16x8_t b_real_vec = b_vec.val[0];
476
+ float16x8_t b_imag_vec = b_vec.val[1];
477
+ float16x8_t c_real_vec = c_vec.val[0];
478
+ float16x8_t c_imag_vec = c_vec.val[1];
479
+
480
+ // Compute the dot product:
481
+ cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
482
+ cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
483
+ cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
484
+ cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
485
+ }
486
+
487
+ simsimd_f32c_t cb_j;
488
+ cb_j.real = _simsimd_reduce_f16x8_neon(cb_j_real_vec);
489
+ cb_j.imag = _simsimd_reduce_f16x8_neon(cb_j_imag_vec);
490
+ sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
491
+ sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
492
+ }
493
+
494
+ results[0] = sum_real;
495
+ results[1] = sum_imag;
496
+ }
497
+
303
498
  #pragma clang attribute pop
304
499
  #pragma GCC pop_options
305
500
  #endif // SIMSIMD_TARGET_NEON_F16
@@ -314,26 +509,26 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const *a, simsimd_
314
509
  float32x4_t sum_vec = vdupq_n_f32(0);
315
510
  for (simsimd_size_t i = 0; i != n; ++i) {
316
511
  float32x4_t a_vec = vdupq_n_f32(simsimd_bf16_to_f32(a + i));
317
- float32x4_t partial_sum_vec = vdupq_n_f32(0);
512
+ float32x4_t cb_j_vec = vdupq_n_f32(0);
318
513
  for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
319
514
  bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
320
515
  bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
321
- partial_sum_vec = vbfdotq_f32(partial_sum_vec, b_vec, c_vec);
516
+ cb_j_vec = vbfdotq_f32(cb_j_vec, b_vec, c_vec);
322
517
  }
323
- sum_vec = vmlaq_f32(sum_vec, a_vec, partial_sum_vec);
518
+ sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
324
519
  }
325
520
 
326
521
  // Handle the tail of every row
327
522
  simsimd_f64_t sum = vaddvq_f32(sum_vec);
328
- simsimd_size_t tail_length = n % 8;
329
- simsimd_size_t tail_start = n - tail_length;
523
+ simsimd_size_t const tail_length = n % 8;
524
+ simsimd_size_t const tail_start = n - tail_length;
330
525
  if (tail_length) {
331
526
  for (simsimd_size_t i = 0; i != n; ++i) {
332
527
  simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
333
528
  bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b + tail_start, tail_length);
334
529
  bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
335
- simsimd_f32_t partial_sum = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), b_vec, c_vec));
336
- sum += a_i * partial_sum;
530
+ simsimd_f32_t cb_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), b_vec, c_vec));
531
+ sum += a_i * cb_j;
337
532
  }
338
533
  }
339
534
 
@@ -348,7 +543,7 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsi
348
543
  simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
349
544
  simsimd_f32_t b_i = simsimd_bf16_to_f32(b + i);
350
545
  float32x4_t diff_i_vec = vdupq_n_f32(a_i - b_i);
351
- float32x4_t partial_sum_vec = vdupq_n_f32(0);
546
+ float32x4_t cdiff_j_vec = vdupq_n_f32(0);
352
547
  for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
353
548
  bfloat16x8_t a_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(a + j));
354
549
  bfloat16x8_t b_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
@@ -365,15 +560,15 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsi
365
560
  bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
366
561
 
367
562
  bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
368
- partial_sum_vec = vbfdotq_f32(partial_sum_vec, diff_j_vec, c_vec);
563
+ cdiff_j_vec = vbfdotq_f32(cdiff_j_vec, diff_j_vec, c_vec);
369
564
  }
370
- sum_vec = vmlaq_f32(sum_vec, diff_i_vec, partial_sum_vec);
565
+ sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
371
566
  }
372
567
 
373
568
  // Handle the tail of every row
374
569
  simsimd_f32_t sum = vaddvq_f32(sum_vec);
375
- simsimd_size_t tail_length = n % 8;
376
- simsimd_size_t tail_start = n - tail_length;
570
+ simsimd_size_t const tail_length = n % 8;
571
+ simsimd_size_t const tail_start = n - tail_length;
377
572
  if (tail_length) {
378
573
  for (simsimd_size_t i = 0; i != n; ++i) {
379
574
  simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
@@ -392,14 +587,84 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsi
392
587
  bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
393
588
 
394
589
  bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
395
- simsimd_f32_t partial_sum = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), diff_j_vec, c_vec));
396
- sum += diff_i * partial_sum;
590
+ simsimd_f32_t cdiff_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), diff_j_vec, c_vec));
591
+ sum += diff_i * cdiff_j;
397
592
  }
398
593
  }
399
594
 
400
595
  *result = _simsimd_sqrt_f32_neon(sum);
401
596
  }
402
597
 
598
+ SIMSIMD_INTERNAL int16x4x2_t _simsimd_partial_load_bf16x4x2_neon(simsimd_bf16c_t const *x, simsimd_size_t n) {
599
+ union {
600
+ int16x4x2_t vec;
601
+ simsimd_bf16_t scalars[2][4];
602
+ } result;
603
+ simsimd_size_t i = 0;
604
+ for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
605
+ for (; i < 4; ++i) result.scalars[1][i] = 0, result.scalars[1][i] = 0;
606
+ return result.vec;
607
+ }
608
+
609
+ SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
610
+ simsimd_bf16c_t const *c, simsimd_size_t n,
611
+ simsimd_distance_t *results) {
612
+ simsimd_f32_t sum_real = 0;
613
+ simsimd_f32_t sum_imag = 0;
614
+ simsimd_size_t const tail_length = n % 4;
615
+ simsimd_size_t const tail_start = n - tail_length;
616
+ for (simsimd_size_t i = 0; i != n; ++i) {
617
+ simsimd_f32c_t a_i = {simsimd_bf16_to_f32(&a[i].real), simsimd_bf16_to_f32(&a[i].imag)};
618
+ // A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
619
+ // FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
620
+ // at once. That's how the original implementation worked, but compiling it was a nightmare :)
621
+ float32x4_t cb_j_real_vec = vdupq_n_f32(0);
622
+ float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
623
+ for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
624
+ // Unpack the input arrays into real and imaginary parts.
625
+ // MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
626
+ // integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
627
+ int16x4x2_t b_vec = vld2_s16((short const *)(b + j));
628
+ int16x4x2_t c_vec = vld2_s16((short const *)(c + i * n + j));
629
+ float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
630
+ float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
631
+ float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
632
+ float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
633
+
634
+ // Compute the dot product:
635
+ cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
636
+ cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
637
+ cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
638
+ cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
639
+ }
640
+ // Handle row tails
641
+ if (tail_length) {
642
+ // Unpack the input arrays into real and imaginary parts:
643
+ int16x4x2_t b_vec = _simsimd_partial_load_bf16x4x2_neon(b + tail_start, tail_length);
644
+ int16x4x2_t c_vec = _simsimd_partial_load_bf16x4x2_neon(c + i * n + tail_start, tail_length);
645
+ float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
646
+ float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
647
+ float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
648
+ float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
649
+
650
+ // Compute the dot product:
651
+ cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
652
+ cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
653
+ cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
654
+ cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
655
+ }
656
+
657
+ simsimd_f32c_t cb_j;
658
+ cb_j.real = vaddvq_f32(cb_j_real_vec);
659
+ cb_j.imag = vaddvq_f32(cb_j_imag_vec);
660
+ sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
661
+ sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
662
+ }
663
+
664
+ results[0] = sum_real;
665
+ results[1] = sum_imag;
666
+ }
667
+
403
668
  #pragma clang attribute pop
404
669
  #pragma GCC pop_options
405
670
  #endif // SIMSIMD_TARGET_NEON_BF16
@@ -417,26 +682,26 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_haswell(simsimd_f16_t const *a, simsimd
417
682
  __m256 sum_vec = _mm256_setzero_ps();
418
683
  for (simsimd_size_t i = 0; i != n; ++i) {
419
684
  __m256 a_vec = _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i)));
420
- __m256 partial_sum_vec = _mm256_setzero_ps();
685
+ __m256 cb_j_vec = _mm256_setzero_ps();
421
686
  for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
422
687
  __m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j)));
423
688
  __m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
424
- partial_sum_vec = _mm256_fmadd_ps(b_vec, c_vec, partial_sum_vec);
689
+ cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
425
690
  }
426
- sum_vec = _mm256_fmadd_ps(a_vec, partial_sum_vec, sum_vec);
691
+ sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
427
692
  }
428
693
 
429
694
  // Handle the tail of every row
430
695
  simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
431
- simsimd_size_t tail_length = n % 8;
432
- simsimd_size_t tail_start = n - tail_length;
696
+ simsimd_size_t const tail_length = n % 8;
697
+ simsimd_size_t const tail_start = n - tail_length;
433
698
  if (tail_length) {
434
699
  for (simsimd_size_t i = 0; i != n; ++i) {
435
700
  simsimd_f32_t a_i = _mm256_cvtss_f32(_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))));
436
701
  __m256 b_vec = _simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length);
437
702
  __m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
438
- simsimd_f32_t partial_sum = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
439
- sum += a_i * partial_sum;
703
+ simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
704
+ sum += a_i * cb_j;
440
705
  }
441
706
  }
442
707
 
@@ -451,21 +716,21 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const *a, sims
451
716
  __m256 diff_i_vec = _mm256_sub_ps( //
452
717
  _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))), //
453
718
  _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(b + i))));
454
- __m256 partial_sum_vec = _mm256_setzero_ps();
719
+ __m256 cdiff_j_vec = _mm256_setzero_ps();
455
720
  for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
456
721
  __m256 diff_j_vec = _mm256_sub_ps( //
457
722
  _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(a + j))),
458
723
  _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j))));
459
724
  __m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
460
- partial_sum_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, partial_sum_vec);
725
+ cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
461
726
  }
462
- sum_vec = _mm256_fmadd_ps(diff_i_vec, partial_sum_vec, sum_vec);
727
+ sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
463
728
  }
464
729
 
465
730
  // Handle the tail of every row
466
731
  simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
467
- simsimd_size_t tail_length = n % 8;
468
- simsimd_size_t tail_start = n - tail_length;
732
+ simsimd_size_t const tail_length = n % 8;
733
+ simsimd_size_t const tail_start = n - tail_length;
469
734
  if (tail_length) {
470
735
  for (simsimd_size_t i = 0; i != n; ++i) {
471
736
  simsimd_f32_t diff_i = _mm256_cvtss_f32(_mm256_sub_ps( //
@@ -475,8 +740,8 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const *a, sims
475
740
  _simsimd_partial_load_f16x8_haswell(a + tail_start, tail_length),
476
741
  _simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length));
477
742
  __m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
478
- simsimd_f32_t partial_sum = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
479
- sum += diff_i * partial_sum;
743
+ simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
744
+ sum += diff_i * cdiff_j;
480
745
  }
481
746
  }
482
747
 
@@ -490,19 +755,19 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const *a, simsi
490
755
  for (simsimd_size_t i = 0; i != n; ++i) {
491
756
  // The `simsimd_bf16_to_f32` is cheaper than `_simsimd_bf16x8_to_f32x8_haswell`
492
757
  __m256 a_vec = _mm256_set1_ps(simsimd_bf16_to_f32(a + i));
493
- __m256 partial_sum_vec = _mm256_setzero_ps();
758
+ __m256 cb_j_vec = _mm256_setzero_ps();
494
759
  for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
495
760
  __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j)));
496
761
  __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
497
- partial_sum_vec = _mm256_fmadd_ps(b_vec, c_vec, partial_sum_vec);
762
+ cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
498
763
  }
499
- sum_vec = _mm256_fmadd_ps(a_vec, partial_sum_vec, sum_vec);
764
+ sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
500
765
  }
501
766
 
502
767
  // Handle the tail of every row
503
768
  simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
504
- simsimd_size_t tail_length = n % 8;
505
- simsimd_size_t tail_start = n - tail_length;
769
+ simsimd_size_t const tail_length = n % 8;
770
+ simsimd_size_t const tail_start = n - tail_length;
506
771
  if (tail_length) {
507
772
  for (simsimd_size_t i = 0; i != n; ++i) {
508
773
  simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
@@ -510,8 +775,8 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const *a, simsi
510
775
  _simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length));
511
776
  __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell( //
512
777
  _simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
513
- simsimd_f32_t partial_sum = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
514
- sum += a_i * partial_sum;
778
+ simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
779
+ sum += a_i * cb_j;
515
780
  }
516
781
  }
517
782
 
@@ -526,21 +791,21 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, si
526
791
  __m256 diff_i_vec = _mm256_sub_ps( //
527
792
  _mm256_set1_ps(simsimd_bf16_to_f32(a + i)), //
528
793
  _mm256_set1_ps(simsimd_bf16_to_f32(b + i)));
529
- __m256 partial_sum_vec = _mm256_setzero_ps();
794
+ __m256 cdiff_j_vec = _mm256_setzero_ps();
530
795
  for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
531
796
  __m256 diff_j_vec = _mm256_sub_ps( //
532
797
  _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(a + j))), //
533
798
  _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j))));
534
799
  __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
535
- partial_sum_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, partial_sum_vec);
800
+ cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
536
801
  }
537
- sum_vec = _mm256_fmadd_ps(diff_i_vec, partial_sum_vec, sum_vec);
802
+ sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
538
803
  }
539
804
 
540
805
  // Handle the tail of every row
541
806
  simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
542
- simsimd_size_t tail_length = n % 8;
543
- simsimd_size_t tail_start = n - tail_length;
807
+ simsimd_size_t const tail_length = n % 8;
808
+ simsimd_size_t const tail_start = n - tail_length;
544
809
  if (tail_length) {
545
810
  for (simsimd_size_t i = 0; i != n; ++i) {
546
811
  simsimd_f32_t diff_i = simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i);
@@ -549,8 +814,8 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, si
549
814
  _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length)));
550
815
  __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(
551
816
  _simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
552
- simsimd_f32_t partial_sum = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
553
- sum += diff_i * partial_sum;
817
+ simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
818
+ sum += diff_i * cdiff_j;
554
819
  }
555
820
  }
556
821
 
@@ -566,16 +831,71 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, si
566
831
  #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2")
567
832
  #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2"))), apply_to = function)
568
833
 
834
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake_under16unrolled(simsimd_f32_t const *a, simsimd_f32_t const *b,
835
+ simsimd_f32_t const *c, simsimd_size_t n,
836
+ simsimd_distance_t *result) {
837
+ // The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
838
+ // until the very end of the computation.
839
+ __mmask16 const mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
840
+ __m512 const b_vec = _mm512_maskz_loadu_ps(mask, b);
841
+
842
+ __m512 cb_j1 = _mm512_setzero_ps();
843
+ __m512 cb_j2 = _mm512_setzero_ps();
844
+ __m512 cb_j3 = _mm512_setzero_ps();
845
+ __m512 cb_j4 = _mm512_setzero_ps();
846
+
847
+ // Unroll the loop to process 4x ZMM registers at a time.
848
+ simsimd_size_t i = 0;
849
+ for (; i + 4 <= n; i += 4) {
850
+ cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
851
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
852
+ cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
853
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
854
+ cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
855
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
856
+ cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
857
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
858
+ }
859
+
860
+ if (i + 0 < n)
861
+ cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
862
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
863
+ if (i + 1 < n)
864
+ cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
865
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
866
+ if (i + 2 < n)
867
+ cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
868
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
869
+ if (i + 3 < n)
870
+ cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
871
+ _mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
872
+
873
+ // Combine cb_j sums
874
+ __m512 sum_vec = _mm512_add_ps( //
875
+ _mm512_add_ps(cb_j1, cb_j2), //
876
+ _mm512_add_ps(cb_j3, cb_j4));
877
+ *result = _mm512_reduce_add_ps(sum_vec);
878
+ }
879
+
569
880
  SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
570
881
  simsimd_size_t n, simsimd_distance_t *result) {
571
- simsimd_size_t tail_length = n % 16;
572
- simsimd_size_t tail_start = n - tail_length;
882
+
883
+ // On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
884
+ // per row and 16 rows at a time, still keeping enough register space for temporaries.
885
+ if (n <= 16) {
886
+ simsimd_bilinear_f32_skylake_under16unrolled(a, b, c, n, result);
887
+ return;
888
+ }
889
+
890
+ // Default case for arbitrary size `n`
891
+ simsimd_size_t const tail_length = n % 16;
892
+ simsimd_size_t const tail_start = n - tail_length;
573
893
  __m512 sum_vec = _mm512_setzero_ps();
574
- __mmask16 tail_mask = (__mmask16)_bzhi_u32(0xFFFF, tail_length);
894
+ __mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
575
895
 
576
896
  for (simsimd_size_t i = 0; i != n; ++i) {
577
897
  __m512 a_vec = _mm512_set1_ps(a[i]);
578
- __m512 partial_sum_vec = _mm512_setzero_ps();
898
+ __m512 cb_j_vec = _mm512_setzero_ps();
579
899
  __m512 b_vec, c_vec;
580
900
  simsimd_size_t j = 0;
581
901
 
@@ -588,10 +908,10 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd
588
908
  b_vec = _mm512_maskz_loadu_ps(tail_mask, b + tail_start);
589
909
  c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
590
910
  }
591
- partial_sum_vec = _mm512_fmadd_ps(b_vec, c_vec, partial_sum_vec);
911
+ cb_j_vec = _mm512_fmadd_ps(b_vec, c_vec, cb_j_vec);
592
912
  j += 16;
593
913
  if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
594
- sum_vec = _mm512_fmadd_ps(a_vec, partial_sum_vec, sum_vec);
914
+ sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
595
915
  }
596
916
 
597
917
  *result = _mm512_reduce_add_ps(sum_vec);
@@ -600,14 +920,14 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd
600
920
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b,
601
921
  simsimd_f32_t const *c, simsimd_size_t n,
602
922
  simsimd_distance_t *result) {
603
- simsimd_size_t tail_length = n % 16;
604
- simsimd_size_t tail_start = n - tail_length;
923
+ simsimd_size_t const tail_length = n % 16;
924
+ simsimd_size_t const tail_start = n - tail_length;
605
925
  __m512 sum_vec = _mm512_setzero_ps();
606
- __mmask16 tail_mask = (__mmask16)_bzhi_u32(0xFFFF, tail_length);
926
+ __mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
607
927
 
608
928
  for (simsimd_size_t i = 0; i != n; ++i) {
609
929
  __m512 diff_i_vec = _mm512_set1_ps(a[i] - b[i]);
610
- __m512 partial_sum_vec = _mm512_setzero_ps(), partial_sum_bot_vec = _mm512_setzero_ps();
930
+ __m512 cdiff_j_vec = _mm512_setzero_ps(), cdiff_j_bot_vec = _mm512_setzero_ps();
611
931
  __m512 a_j_vec, b_j_vec, diff_j_vec, c_vec;
612
932
  simsimd_size_t j = 0;
613
933
 
@@ -624,15 +944,243 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, sims
624
944
  c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
625
945
  }
626
946
  diff_j_vec = _mm512_sub_ps(a_j_vec, b_j_vec);
627
- partial_sum_vec = _mm512_fmadd_ps(diff_j_vec, c_vec, partial_sum_vec);
947
+ cdiff_j_vec = _mm512_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
628
948
  j += 16;
629
949
  if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
630
- sum_vec = _mm512_fmadd_ps(diff_i_vec, partial_sum_vec, sum_vec);
950
+ sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
631
951
  }
632
952
 
633
953
  *result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_ps(sum_vec));
634
954
  }
635
955
 
956
+ SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
957
+ simsimd_f32c_t const *c, simsimd_size_t n,
958
+ simsimd_distance_t *results) {
959
+
960
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
961
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
962
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
963
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
964
+ // one of the vectors.
965
+ __m512i const sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
966
+
967
+ // Default case for arbitrary size `n`
968
+ simsimd_size_t const tail_length = n % 8;
969
+ simsimd_size_t const tail_start = n - tail_length;
970
+ __mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
971
+ simsimd_f32_t sum_real = 0;
972
+ simsimd_f32_t sum_imag = 0;
973
+
974
+ for (simsimd_size_t i = 0; i != n; ++i) {
975
+ simsimd_f32_t const a_i_real = a[i].real;
976
+ simsimd_f32_t const a_i_imag = a[i].imag;
977
+ __m512 cb_j_real_vec = _mm512_setzero_ps();
978
+ __m512 cb_j_imag_vec = _mm512_setzero_ps();
979
+ __m512 b_vec, c_vec;
980
+ simsimd_size_t j = 0;
981
+
982
+ simsimd_bilinear_f32c_skylake_cycle:
983
+ if (j + 8 <= n) {
984
+ b_vec = _mm512_loadu_ps((simsimd_f32_t const *)(b + j));
985
+ c_vec = _mm512_loadu_ps((simsimd_f32_t const *)(c + i * n + j));
986
+ }
987
+ else {
988
+ b_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(b + tail_start));
989
+ c_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(c + i * n + tail_start));
990
+ }
991
+ // The real part of the product: b.real * c.real - b.imag * c.imag.
992
+ // The subtraction will be performed later with a sign flip.
993
+ cb_j_real_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_real_vec);
994
+ // The imaginary part of the product: b.real * c.imag + b.imag * c.real.
995
+ // Swap the imaginary and real parts of `c` before multiplication:
996
+ c_vec = _mm512_permute_ps(c_vec, 0xB1); //? Swap adjacent entries within each pair
997
+ cb_j_imag_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_imag_vec);
998
+ j += 8;
999
+ if (j < n) goto simsimd_bilinear_f32c_skylake_cycle;
1000
+ // Flip the sign bit in every second scalar before accumulation:
1001
+ cb_j_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(cb_j_real_vec), sign_flip_vec));
1002
+ // Horizontal sums are the expensive part of the computation:
1003
+ simsimd_f32_t const cb_j_real = _mm512_reduce_add_ps(cb_j_real_vec);
1004
+ simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ps(cb_j_imag_vec);
1005
+ sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1006
+ sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1007
+ }
1008
+
1009
+ // Reduce horizontal sums:
1010
+ results[0] = sum_real;
1011
+ results[1] = sum_imag;
1012
+ }
1013
+
1014
+ SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake_under8unrolled(simsimd_f64_t const *a, simsimd_f64_t const *b,
1015
+ simsimd_f64_t const *c, simsimd_size_t n,
1016
+ simsimd_distance_t *result) {
1017
+
1018
+ // The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
1019
+ // until the very end of the computation.
1020
+ __mmask8 const row_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
1021
+ __m512d const b_vec = _mm512_maskz_loadu_pd(row_mask, b);
1022
+
1023
+ __m512d cb_j1 = _mm512_setzero_pd();
1024
+ __m512d cb_j2 = _mm512_setzero_pd();
1025
+ __m512d cb_j3 = _mm512_setzero_pd();
1026
+ __m512d cb_j4 = _mm512_setzero_pd();
1027
+
1028
+ // clang-format off
1029
+ if (n > 0) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 0), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[0])), cb_j1);
1030
+ if (n > 1) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 1), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[1])), cb_j2);
1031
+ if (n > 2) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 2), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[2])), cb_j3);
1032
+ if (n > 3) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 3), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[3])), cb_j4);
1033
+
1034
+ if (n > 4) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 4), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[4])), cb_j1);
1035
+ if (n > 5) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 5), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[5])), cb_j2);
1036
+ if (n > 6) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 6), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[6])), cb_j3);
1037
+ if (n > 7) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 7), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[7])), cb_j4);
1038
+ // clang-format on
1039
+
1040
+ // Combine cb_j sums
1041
+ __m512d sum_vec = _mm512_add_pd( //
1042
+ _mm512_add_pd(cb_j1, cb_j2), //
1043
+ _mm512_add_pd(cb_j3, cb_j4));
1044
+ *result = _mm512_reduce_add_pd(sum_vec);
1045
+ }
1046
+
1047
+ SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c,
1048
+ simsimd_size_t n, simsimd_distance_t *result) {
1049
+
1050
+ // On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
1051
+ // per row and 8 rows at a time, still keeping enough register space for temporaries.
1052
+ if (n <= 8) {
1053
+ simsimd_bilinear_f64_skylake_under8unrolled(a, b, c, n, result);
1054
+ return;
1055
+ }
1056
+
1057
+ // Default case for arbitrary size `n`
1058
+ simsimd_size_t const tail_length = n % 8;
1059
+ simsimd_size_t const tail_start = n - tail_length;
1060
+ __m512d sum_vec = _mm512_setzero_pd();
1061
+ __mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
1062
+
1063
+ for (simsimd_size_t i = 0; i != n; ++i) {
1064
+ __m512d a_vec = _mm512_set1_pd(a[i]);
1065
+ __m512d cb_j_vec = _mm512_setzero_pd();
1066
+ __m512d b_vec, c_vec;
1067
+ simsimd_size_t j = 0;
1068
+
1069
+ simsimd_bilinear_f64_skylake_cycle:
1070
+ if (j + 8 <= n) {
1071
+ b_vec = _mm512_loadu_pd(b + j);
1072
+ c_vec = _mm512_loadu_pd(c + i * n + j);
1073
+ }
1074
+ else {
1075
+ b_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
1076
+ c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
1077
+ }
1078
+ cb_j_vec = _mm512_fmadd_pd(b_vec, c_vec, cb_j_vec);
1079
+ j += 8;
1080
+ if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
1081
+ sum_vec = _mm512_fmadd_pd(a_vec, cb_j_vec, sum_vec);
1082
+ }
1083
+
1084
+ *result = _mm512_reduce_add_pd(sum_vec);
1085
+ }
1086
+
1087
+ SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b,
1088
+ simsimd_f64_t const *c, simsimd_size_t n,
1089
+ simsimd_distance_t *result) {
1090
+ simsimd_size_t const tail_length = n % 8;
1091
+ simsimd_size_t const tail_start = n - tail_length;
1092
+ __mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
1093
+ __m512d sum_vec = _mm512_setzero_pd();
1094
+
1095
+ for (simsimd_size_t i = 0; i != n; ++i) {
1096
+ __m512d diff_i_vec = _mm512_set1_pd(a[i] - b[i]);
1097
+ __m512d cdiff_j_vec = _mm512_setzero_pd();
1098
+ __m512d a_j_vec, b_j_vec, diff_j_vec, c_vec;
1099
+ simsimd_size_t j = 0;
1100
+
1101
+ // The nested loop is cleaner to implement with a `goto` in this case:
1102
+ simsimd_bilinear_f64_skylake_cycle:
1103
+ if (j + 8 <= n) {
1104
+ a_j_vec = _mm512_loadu_pd(a + j);
1105
+ b_j_vec = _mm512_loadu_pd(b + j);
1106
+ c_vec = _mm512_loadu_pd(c + i * n + j);
1107
+ }
1108
+ else {
1109
+ a_j_vec = _mm512_maskz_loadu_pd(tail_mask, a + tail_start);
1110
+ b_j_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
1111
+ c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
1112
+ }
1113
+ diff_j_vec = _mm512_sub_pd(a_j_vec, b_j_vec);
1114
+ cdiff_j_vec = _mm512_fmadd_pd(diff_j_vec, c_vec, cdiff_j_vec);
1115
+ j += 8;
1116
+ if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
1117
+ sum_vec = _mm512_fmadd_pd(diff_i_vec, cdiff_j_vec, sum_vec);
1118
+ }
1119
+
1120
+ *result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_pd(sum_vec));
1121
+ }
1122
+
1123
+ SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const *a, simsimd_f64c_t const *b,
1124
+ simsimd_f64c_t const *c, simsimd_size_t n,
1125
+ simsimd_distance_t *results) {
1126
+
1127
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1128
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1129
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1130
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1131
+ // one of the vectors.
1132
+ __m512i const sign_flip_vec = _mm512_set_epi64( //
1133
+ 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
1134
+ 0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
1135
+ );
1136
+
1137
+ // Default case for arbitrary size `n`
1138
+ simsimd_size_t const tail_length = n % 4;
1139
+ simsimd_size_t const tail_start = n - tail_length;
1140
+ __mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
1141
+ simsimd_f64_t sum_real = 0;
1142
+ simsimd_f64_t sum_imag = 0;
1143
+
1144
+ for (simsimd_size_t i = 0; i != n; ++i) {
1145
+ simsimd_f64_t const a_i_real = a[i].real;
1146
+ simsimd_f64_t const a_i_imag = a[i].imag;
1147
+ __m512d cb_j_real_vec = _mm512_setzero_pd();
1148
+ __m512d cb_j_imag_vec = _mm512_setzero_pd();
1149
+ __m512d b_vec, c_vec;
1150
+ simsimd_size_t j = 0;
1151
+
1152
+ simsimd_bilinear_f64c_skylake_cycle:
1153
+ if (j + 4 <= n) {
1154
+ b_vec = _mm512_loadu_pd((simsimd_f64_t const *)(b + j));
1155
+ c_vec = _mm512_loadu_pd((simsimd_f64_t const *)(c + i * n + j));
1156
+ }
1157
+ else {
1158
+ b_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(b + tail_start));
1159
+ c_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(c + i * n + tail_start));
1160
+ }
1161
+ // The real part of the product: b.real * c.real - b.imag * c.imag.
1162
+ // The subtraction will be performed later with a sign flip.
1163
+ cb_j_real_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_real_vec);
1164
+ // The imaginary part of the product: b.real * c.imag + b.imag * c.real.
1165
+ // Swap the imaginary and real parts of `c` before multiplication:
1166
+ c_vec = _mm512_permute_pd(c_vec, 0x55); //? Same as 0b01010101.
1167
+ cb_j_imag_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_imag_vec);
1168
+ j += 4;
1169
+ if (j < n) goto simsimd_bilinear_f64c_skylake_cycle;
1170
+ // Flip the sign bit in every second scalar before accumulation:
1171
+ cb_j_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(cb_j_real_vec), sign_flip_vec));
1172
+ // Horizontal sums are the expensive part of the computation:
1173
+ simsimd_f64_t const cb_j_real = _mm512_reduce_add_pd(cb_j_real_vec);
1174
+ simsimd_f64_t const cb_j_imag = _mm512_reduce_add_pd(cb_j_imag_vec);
1175
+ sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1176
+ sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1177
+ }
1178
+
1179
+ // Reduce horizontal sums:
1180
+ results[0] = sum_real;
1181
+ results[1] = sum_imag;
1182
+ }
1183
+
636
1184
  #pragma clang attribute pop
637
1185
  #pragma GCC pop_options
638
1186
  #endif // SIMSIMD_TARGET_SKYLAKE
@@ -645,14 +1193,14 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, sims
645
1193
 
646
1194
  SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
647
1195
  simsimd_bf16_t const *c, simsimd_size_t n, simsimd_distance_t *result) {
648
- simsimd_size_t tail_length = n % 32;
649
- simsimd_size_t tail_start = n - tail_length;
1196
+ simsimd_size_t const tail_length = n % 32;
1197
+ simsimd_size_t const tail_start = n - tail_length;
1198
+ __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
650
1199
  __m512 sum_vec = _mm512_setzero_ps();
651
- __mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
652
1200
 
653
1201
  for (simsimd_size_t i = 0; i != n; ++i) {
654
1202
  __m512 a_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i));
655
- __m512 partial_sum_vec = _mm512_setzero_ps();
1203
+ __m512 cb_j_vec = _mm512_setzero_ps();
656
1204
  __m512i b_vec, c_vec;
657
1205
  simsimd_size_t j = 0;
658
1206
 
@@ -665,10 +1213,10 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd
665
1213
  b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
666
1214
  c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
667
1215
  }
668
- partial_sum_vec = _mm512_dpbf16_ps(partial_sum_vec, (__m512bh)(b_vec), (__m512bh)(c_vec));
1216
+ cb_j_vec = _mm512_dpbf16_ps(cb_j_vec, (__m512bh)(b_vec), (__m512bh)(c_vec));
669
1217
  j += 32;
670
1218
  if (j < n) goto simsimd_bilinear_bf16_genoa_cycle;
671
- sum_vec = _mm512_fmadd_ps(a_vec, partial_sum_vec, sum_vec);
1219
+ sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
672
1220
  }
673
1221
 
674
1222
  *result = _mm512_reduce_add_ps(sum_vec);
@@ -677,14 +1225,14 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd
677
1225
  SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
678
1226
  simsimd_bf16_t const *c, simsimd_size_t n,
679
1227
  simsimd_distance_t *result) {
680
- simsimd_size_t tail_length = n % 32;
681
- simsimd_size_t tail_start = n - tail_length;
1228
+ simsimd_size_t const tail_length = n % 32;
1229
+ simsimd_size_t const tail_start = n - tail_length;
1230
+ __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
682
1231
  __m512 sum_vec = _mm512_setzero_ps();
683
- __mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
684
1232
 
685
1233
  for (simsimd_size_t i = 0; i != n; ++i) {
686
1234
  __m512 diff_i_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i));
687
- __m512 partial_sum_vec = _mm512_setzero_ps();
1235
+ __m512 cdiff_j_vec = _mm512_setzero_ps();
688
1236
  __m512i a_j_vec, b_j_vec, diff_j_vec, c_vec;
689
1237
  simsimd_size_t j = 0;
690
1238
 
@@ -701,15 +1249,78 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, sims
701
1249
  c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
702
1250
  }
703
1251
  diff_j_vec = _simsimd_substract_bf16x32_genoa(a_j_vec, b_j_vec);
704
- partial_sum_vec = _mm512_dpbf16_ps(partial_sum_vec, (__m512bh)(diff_j_vec), (__m512bh)(c_vec));
1252
+ cdiff_j_vec = _mm512_dpbf16_ps(cdiff_j_vec, (__m512bh)(diff_j_vec), (__m512bh)(c_vec));
705
1253
  j += 32;
706
1254
  if (j < n) goto simsimd_mahalanobis_bf16_genoa_cycle;
707
- sum_vec = _mm512_fmadd_ps(diff_i_vec, partial_sum_vec, sum_vec);
1255
+ sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
708
1256
  }
709
1257
 
710
1258
  *result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ps(sum_vec));
711
1259
  }
712
1260
 
1261
+ SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
1262
+ simsimd_bf16c_t const *c, simsimd_size_t n,
1263
+ simsimd_distance_t *results) {
1264
+
1265
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1266
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1267
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1268
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1269
+ // one of the vectors.
1270
+ __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1271
+ __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1272
+ 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1273
+ 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1274
+ 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1275
+ 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1276
+ );
1277
+
1278
+ // Default case for arbitrary size `n`
1279
+ simsimd_size_t const tail_length = n % 16;
1280
+ simsimd_size_t const tail_start = n - tail_length;
1281
+ __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
1282
+ simsimd_f64_t sum_real = 0;
1283
+ simsimd_f64_t sum_imag = 0;
1284
+
1285
+ for (simsimd_size_t i = 0; i != n; ++i) {
1286
+ simsimd_f32_t const a_i_real = a[i].real;
1287
+ simsimd_f32_t const a_i_imag = a[i].imag;
1288
+ __m512 cb_j_real_vec = _mm512_setzero_ps();
1289
+ __m512 cb_j_imag_vec = _mm512_setzero_ps();
1290
+ __m512i b_vec, c_vec;
1291
+ simsimd_size_t j = 0;
1292
+
1293
+ simsimd_bilinear_bf16c_skylake_cycle:
1294
+ if (j + 16 <= n) {
1295
+ b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
1296
+ c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
1297
+ }
1298
+ else {
1299
+ b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
1300
+ c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
1301
+ }
1302
+ cb_j_real_vec = _mm512_dpbf16_ps( //
1303
+ cb_j_real_vec, //
1304
+ (__m512bh)(_mm512_xor_si512(c_vec, sign_flip_vec)), //
1305
+ (__m512bh)b_vec);
1306
+ cb_j_imag_vec = _mm512_dpbf16_ps( //
1307
+ cb_j_imag_vec, //
1308
+ (__m512bh)(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
1309
+ (__m512bh)b_vec);
1310
+ j += 16;
1311
+ if (j < n) goto simsimd_bilinear_bf16c_skylake_cycle;
1312
+ // Horizontal sums are the expensive part of the computation:
1313
+ simsimd_f64_t const cb_j_real = _simsimd_reduce_f32x16_skylake(cb_j_real_vec);
1314
+ simsimd_f64_t const cb_j_imag = _simsimd_reduce_f32x16_skylake(cb_j_imag_vec);
1315
+ sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1316
+ sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1317
+ }
1318
+
1319
+ // Reduce horizontal sums:
1320
+ results[0] = sum_real;
1321
+ results[1] = sum_imag;
1322
+ }
1323
+
713
1324
  #pragma clang attribute pop
714
1325
  #pragma GCC pop_options
715
1326
  #endif // SIMSIMD_TARGET_GENOA
@@ -720,17 +1331,83 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, sims
720
1331
  #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
721
1332
  apply_to = function)
722
1333
 
1334
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire_under32unrolled(simsimd_f16_t const *a, simsimd_f16_t const *b,
1335
+ simsimd_f16_t const *c, simsimd_size_t const n,
1336
+ simsimd_distance_t *result) {
1337
+ // The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
1338
+ // until the very end of the computation.
1339
+ __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1340
+ __m512h const b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
1341
+
1342
+ // Independently accumulate the partial sums into separate variables to avoid data-dependencies.
1343
+ __m512h cb_j1 = _mm512_setzero_ph();
1344
+ __m512h cb_j2 = _mm512_setzero_ph();
1345
+ __m512h cb_j3 = _mm512_setzero_ph();
1346
+ __m512h cb_j4 = _mm512_setzero_ph();
1347
+
1348
+ // Unroll the loop to process 4x ZMM registers at a time.
1349
+ simsimd_size_t i = 0;
1350
+ for (; i + 4 <= n; i += 4) {
1351
+ // If the code is compiled without native support for `_Float16`, we need a workaround
1352
+ // to avoid implicit casts from out `simsimd_f16_t` to `_Float16`.
1353
+ cb_j1 = _mm512_fmadd_ph(
1354
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
1355
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
1356
+ cb_j2 = _mm512_fmadd_ph(
1357
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
1358
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
1359
+ cb_j3 = _mm512_fmadd_ph(
1360
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
1361
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
1362
+ cb_j4 = _mm512_fmadd_ph(
1363
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
1364
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
1365
+ }
1366
+
1367
+ // Handle the tail of the loop:
1368
+ if (i + 0 < n)
1369
+ cb_j1 = _mm512_fmadd_ph(
1370
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
1371
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
1372
+ if (i + 1 < n)
1373
+ cb_j2 = _mm512_fmadd_ph(
1374
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
1375
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
1376
+ if (i + 2 < n)
1377
+ cb_j3 = _mm512_fmadd_ph(
1378
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
1379
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
1380
+ if (i + 3 < n)
1381
+ cb_j4 = _mm512_fmadd_ph(
1382
+ _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
1383
+ _mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
1384
+
1385
+ // Combine cb_j sums
1386
+ __m512h sum_vec = _mm512_add_ph( //
1387
+ _mm512_add_ph(cb_j1, cb_j2), //
1388
+ _mm512_add_ph(cb_j3, cb_j4));
1389
+ *result = _mm512_reduce_add_ph(sum_vec);
1390
+ }
1391
+
723
1392
  SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
724
1393
  simsimd_f16_t const *c, simsimd_size_t n,
725
1394
  simsimd_distance_t *result) {
726
- simsimd_size_t tail_length = n % 32;
727
- simsimd_size_t tail_start = n - tail_length;
1395
+
1396
+ // On modern x86 CPUs we have enough register space to load fairly large matrices with up to 32 cells
1397
+ // per row and 32 rows at a time, still keeping enough register space for temporaries.
1398
+ if (n <= 32) {
1399
+ simsimd_bilinear_f16_sapphire_under32unrolled(a, b, c, n, result);
1400
+ return;
1401
+ }
1402
+
1403
+ simsimd_size_t const tail_length = n % 32;
1404
+ simsimd_size_t const tail_start = n - tail_length;
1405
+ __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
728
1406
  __m512h sum_vec = _mm512_setzero_ph();
729
- __mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
730
1407
 
731
1408
  for (simsimd_size_t i = 0; i != n; ++i) {
732
1409
  __m512h a_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
733
- __m512h partial_sum_vec = _mm512_setzero_ph();
1410
+ __m512h cb_j_vec = _mm512_setzero_ph();
734
1411
  __m512i b_vec, c_vec;
735
1412
  simsimd_size_t j = 0;
736
1413
 
@@ -743,10 +1420,10 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsim
743
1420
  b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
744
1421
  c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
745
1422
  }
746
- partial_sum_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(c_vec), partial_sum_vec);
1423
+ cb_j_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(c_vec), cb_j_vec);
747
1424
  j += 32;
748
1425
  if (j < n) goto simsimd_bilinear_f16_sapphire_cycle;
749
- sum_vec = _mm512_fmadd_ph(a_vec, partial_sum_vec, sum_vec);
1426
+ sum_vec = _mm512_fmadd_ph(a_vec, cb_j_vec, sum_vec);
750
1427
  }
751
1428
 
752
1429
  *result = _mm512_reduce_add_ph(sum_vec);
@@ -755,16 +1432,16 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsim
755
1432
  SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
756
1433
  simsimd_f16_t const *c, simsimd_size_t n,
757
1434
  simsimd_distance_t *result) {
758
- simsimd_size_t tail_length = n % 32;
759
- simsimd_size_t tail_start = n - tail_length;
1435
+ simsimd_size_t const tail_length = n % 32;
1436
+ simsimd_size_t const tail_start = n - tail_length;
1437
+ __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
760
1438
  __m512h sum_vec = _mm512_setzero_ph();
761
- __mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
762
1439
 
763
1440
  for (simsimd_size_t i = 0; i != n; ++i) {
764
1441
  __m512h a_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
765
1442
  __m512h b_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(b + i)));
766
1443
  __m512h diff_i_vec = _mm512_sub_ph(a_i_vec, b_i_vec);
767
- __m512h partial_sum_vec = _mm512_setzero_ph();
1444
+ __m512h cdiff_j_vec = _mm512_setzero_ph();
768
1445
  __m512h diff_j_vec;
769
1446
  __m512i a_j_vec, b_j_vec, c_vec;
770
1447
  simsimd_size_t j = 0;
@@ -782,15 +1459,76 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const *a, sim
782
1459
  c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
783
1460
  }
784
1461
  diff_j_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_j_vec), _mm512_castsi512_ph(b_j_vec));
785
- partial_sum_vec = _mm512_fmadd_ph(diff_j_vec, _mm512_castsi512_ph(c_vec), partial_sum_vec);
1462
+ cdiff_j_vec = _mm512_fmadd_ph(diff_j_vec, _mm512_castsi512_ph(c_vec), cdiff_j_vec);
786
1463
  j += 32;
787
1464
  if (j < n) goto simsimd_mahalanobis_f16_sapphire_cycle;
788
- sum_vec = _mm512_fmadd_ph(diff_i_vec, partial_sum_vec, sum_vec);
1465
+ sum_vec = _mm512_fmadd_ph(diff_i_vec, cdiff_j_vec, sum_vec);
789
1466
  }
790
1467
 
791
1468
  *result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ph(sum_vec));
792
1469
  }
793
1470
 
1471
+ SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
1472
+ simsimd_f16c_t const *c, simsimd_size_t n,
1473
+ simsimd_distance_t *results) {
1474
+
1475
+ // We take into account, that FMS is the same as FMA with a negative multiplier.
1476
+ // To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
1477
+ // This way we can avoid the shuffling and the need for separate real and imaginary parts.
1478
+ // For the imaginary part of the product, we would need to swap the real and imaginary parts of
1479
+ // one of the vectors.
1480
+ __m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
1481
+ __m512i const swap_adjacent_vec = _mm512_set_epi8( //
1482
+ 61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
1483
+ 45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
1484
+ 29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
1485
+ 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
1486
+ );
1487
+
1488
+ // Default case for arbitrary size `n`
1489
+ simsimd_size_t const tail_length = n % 16;
1490
+ simsimd_size_t const tail_start = n - tail_length;
1491
+ __mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
1492
+ simsimd_f32_t sum_real = 0;
1493
+ simsimd_f32_t sum_imag = 0;
1494
+
1495
+ for (simsimd_size_t i = 0; i != n; ++i) {
1496
+ simsimd_f32_t const a_i_real = a[i].real;
1497
+ simsimd_f32_t const a_i_imag = a[i].imag;
1498
+ __m512h cb_j_real_vec = _mm512_setzero_ph();
1499
+ __m512h cb_j_imag_vec = _mm512_setzero_ph();
1500
+ __m512i b_vec, c_vec;
1501
+ simsimd_size_t j = 0;
1502
+
1503
+ simsimd_bilinear_f16c_skylake_cycle:
1504
+ if (j + 16 <= n) {
1505
+ b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
1506
+ c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
1507
+ }
1508
+ else {
1509
+ b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
1510
+ c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
1511
+ }
1512
+ cb_j_real_vec = _mm512_fmadd_ph( //
1513
+ _mm512_castsi512_ph(_mm512_xor_si512(c_vec, sign_flip_vec)), //
1514
+ _mm512_castsi512_ph(b_vec), cb_j_real_vec);
1515
+ cb_j_imag_vec = _mm512_fmadd_ph( //
1516
+ _mm512_castsi512_ph(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
1517
+ _mm512_castsi512_ph(b_vec), cb_j_imag_vec);
1518
+ j += 16;
1519
+ if (j < n) goto simsimd_bilinear_f16c_skylake_cycle;
1520
+ // Horizontal sums are the expensive part of the computation:
1521
+ simsimd_f32_t const cb_j_real = _mm512_reduce_add_ph(cb_j_real_vec);
1522
+ simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ph(cb_j_imag_vec);
1523
+ sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
1524
+ sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
1525
+ }
1526
+
1527
+ // Reduce horizontal sums:
1528
+ results[0] = sum_real;
1529
+ results[1] = sum_imag;
1530
+ }
1531
+
794
1532
  #pragma clang attribute pop
795
1533
  #pragma GCC pop_options
796
1534
  #endif // SIMSIMD_TARGET_SAPPHIRE