usearch 2.16.7 → 2.16.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/include/usearch/index.hpp +1 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/CONTRIBUTING.md +69 -12
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +1 -1
- package/simsimd/README.md +253 -40
- package/simsimd/c/lib.c +114 -109
- package/simsimd/include/simsimd/binary.h +33 -17
- package/simsimd/include/simsimd/curved.h +867 -129
- package/simsimd/include/simsimd/dot.h +562 -569
- package/simsimd/include/simsimd/probability.h +14 -9
- package/simsimd/include/simsimd/simsimd.h +102 -46
- package/simsimd/include/simsimd/spatial.h +28 -18
- package/simsimd/include/simsimd/types.h +24 -0
- package/simsimd/javascript/fallback.ts +26 -15
- package/simsimd/javascript/simsimd.ts +1 -1
- package/simsimd/package-lock.json +10 -9
- package/simsimd/package.json +2 -2
- package/simsimd/rust/lib.rs +218 -5
- package/simsimd/scripts/bench.cxx +156 -97
- package/simsimd/scripts/test.c +14 -8
- package/simsimd/scripts/test.mjs +158 -145
- package/simsimd/scripts/test.py +71 -52
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
* @date August 27, 2024
|
|
6
6
|
*
|
|
7
7
|
* Contains:
|
|
8
|
-
* - Bilinear form multiplication
|
|
9
8
|
* - Mahalanobis distance
|
|
9
|
+
* - Bilinear form multiplication
|
|
10
|
+
* - Bilinear form multiplication over complex numbers
|
|
10
11
|
*
|
|
11
12
|
* For datatypes:
|
|
13
|
+
* - 64-bit floating point numbers
|
|
12
14
|
* - 32-bit floating point numbers
|
|
13
15
|
* - 16-bit floating point numbers
|
|
14
16
|
* - 16-bit brain-floating point numbers
|
|
@@ -17,6 +19,16 @@
|
|
|
17
19
|
* - Arm: NEON
|
|
18
20
|
* - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
|
|
19
21
|
*
|
|
22
|
+
* Most kernels in this file are designed for BLAS level 2 operations, where the operands are
|
|
23
|
+
* a combination of matrices and vectors, generally forming a chain of multiplications.
|
|
24
|
+
* Most kernels exploit the fact that matrix multiplication is associative, and the order of
|
|
25
|
+
* operations can be changed to minimize the number of operations: `(A * B) * C = A * (B * C)`.
|
|
26
|
+
* To optimize the performance, we minimize the number of memory accesses, and maximize the
|
|
27
|
+
* number of arithmetic operations, by using SIMD instructions.
|
|
28
|
+
*
|
|
29
|
+
* When A and C are vectors, and B is a matrix, we can load every element in B just once, and
|
|
30
|
+
* reuse it for every element in A and C.
|
|
31
|
+
*
|
|
20
32
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
21
33
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
22
34
|
*/
|
|
@@ -39,22 +51,29 @@ extern "C" {
|
|
|
39
51
|
* For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
|
|
40
52
|
*/
|
|
41
53
|
SIMSIMD_PUBLIC void simsimd_bilinear_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
54
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f64c_serial(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
42
55
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
43
56
|
SIMSIMD_PUBLIC void simsimd_bilinear_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
57
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_serial(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
44
58
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
45
59
|
SIMSIMD_PUBLIC void simsimd_bilinear_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
60
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_serial(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
46
61
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
47
62
|
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
63
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_serial(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
48
64
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
49
65
|
|
|
50
66
|
/* Double-precision serial backends for all numeric types.
|
|
51
67
|
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
52
68
|
*/
|
|
53
69
|
SIMSIMD_PUBLIC void simsimd_bilinear_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
70
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_accurate(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
54
71
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
55
72
|
SIMSIMD_PUBLIC void simsimd_bilinear_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
73
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_accurate(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
56
74
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
57
75
|
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
76
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_accurate(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
58
77
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
59
78
|
|
|
60
79
|
/* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
|
|
@@ -62,10 +81,13 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_accurate(simsimd_bf16_t const* a, s
|
|
|
62
81
|
* server CPUs produced before 2023.
|
|
63
82
|
*/
|
|
64
83
|
SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
84
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
65
85
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
66
86
|
SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
87
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
67
88
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
68
89
|
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
90
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
69
91
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
70
92
|
|
|
71
93
|
/* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
|
|
@@ -84,11 +106,17 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const* a, si
|
|
|
84
106
|
* Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
|
|
85
107
|
* Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
|
|
86
108
|
*/
|
|
109
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
110
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const* a, simsimd_f64c_t const* b, simsimd_f64c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
111
|
+
SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_f64_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
87
112
|
SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
113
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const* a, simsimd_f32c_t const* b, simsimd_f32c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
88
114
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_f32_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
89
115
|
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
116
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const* a, simsimd_bf16c_t const* b, simsimd_bf16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
90
117
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_bf16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
91
118
|
SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
119
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const* a, simsimd_f16c_t const* b, simsimd_f16c_t const* c, simsimd_size_t n, simsimd_distance_t* results);
|
|
92
120
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_f16_t const* c, simsimd_size_t n, simsimd_distance_t* result);
|
|
93
121
|
// clang-format on
|
|
94
122
|
|
|
@@ -98,56 +126,91 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const* a, sim
|
|
|
98
126
|
simsimd_size_t n, simsimd_distance_t *result) { \
|
|
99
127
|
simsimd_##accumulator_type##_t sum = 0; \
|
|
100
128
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
101
|
-
simsimd_##accumulator_type##_t
|
|
129
|
+
simsimd_##accumulator_type##_t cb_j = 0; \
|
|
102
130
|
simsimd_##accumulator_type##_t a_i = load_and_convert(a + i); \
|
|
103
131
|
for (simsimd_size_t j = 0; j != n; ++j) { \
|
|
104
132
|
simsimd_##accumulator_type##_t b_j = load_and_convert(b + j); \
|
|
105
133
|
simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
|
|
106
|
-
|
|
134
|
+
cb_j += c_ij * b_j; \
|
|
107
135
|
} \
|
|
108
|
-
sum += a_i *
|
|
136
|
+
sum += a_i * cb_j; \
|
|
109
137
|
} \
|
|
110
138
|
*result = (simsimd_distance_t)sum; \
|
|
111
139
|
}
|
|
112
140
|
|
|
141
|
+
#define SIMSIMD_MAKE_COMPLEX_BILINEAR(name, input_type, accumulator_type, load_and_convert) \
|
|
142
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_##input_type##_##name( \
|
|
143
|
+
simsimd_##input_type##_t const *a_pairs, simsimd_##input_type##_t const *b_pairs, \
|
|
144
|
+
simsimd_##input_type##_t const *c_pairs, simsimd_size_t n, simsimd_distance_t *results) { \
|
|
145
|
+
simsimd_##accumulator_type##_t sum_real = 0; \
|
|
146
|
+
simsimd_##accumulator_type##_t sum_imag = 0; \
|
|
147
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
148
|
+
simsimd_##accumulator_type##_t cb_j_real = 0; \
|
|
149
|
+
simsimd_##accumulator_type##_t cb_j_imag = 0; \
|
|
150
|
+
simsimd_##accumulator_type##_t a_i_real = load_and_convert(&(a_pairs + i)->real); \
|
|
151
|
+
simsimd_##accumulator_type##_t a_i_imag = load_and_convert(&(a_pairs + i)->imag); \
|
|
152
|
+
for (simsimd_size_t j = 0; j != n; ++j) { \
|
|
153
|
+
simsimd_##accumulator_type##_t b_j_real = load_and_convert(&(b_pairs + j)->real); \
|
|
154
|
+
simsimd_##accumulator_type##_t b_j_imag = load_and_convert(&(b_pairs + j)->imag); \
|
|
155
|
+
simsimd_##accumulator_type##_t c_ij_real = load_and_convert(&(c_pairs + i * n + j)->real); \
|
|
156
|
+
simsimd_##accumulator_type##_t c_ij_imag = load_and_convert(&(c_pairs + i * n + j)->imag); \
|
|
157
|
+
/* Complex multiplication: (c_ij * b_j) */ \
|
|
158
|
+
cb_j_real += c_ij_real * b_j_real - c_ij_imag * b_j_imag; \
|
|
159
|
+
cb_j_imag += c_ij_real * b_j_imag + c_ij_imag * b_j_real; \
|
|
160
|
+
} \
|
|
161
|
+
/* Complex multiplication: (a_i * cb_j) */ \
|
|
162
|
+
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag; \
|
|
163
|
+
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real; \
|
|
164
|
+
} \
|
|
165
|
+
results[0] = (simsimd_distance_t)sum_real; \
|
|
166
|
+
results[1] = (simsimd_distance_t)sum_imag; \
|
|
167
|
+
}
|
|
168
|
+
|
|
113
169
|
#define SIMSIMD_MAKE_MAHALANOBIS(name, input_type, accumulator_type, load_and_convert) \
|
|
114
170
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_##input_type##_##name( \
|
|
115
171
|
simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
|
|
116
172
|
simsimd_size_t n, simsimd_distance_t *result) { \
|
|
117
173
|
simsimd_##accumulator_type##_t sum = 0; \
|
|
118
174
|
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
119
|
-
simsimd_##accumulator_type##_t
|
|
175
|
+
simsimd_##accumulator_type##_t cdiff_j = 0; \
|
|
120
176
|
simsimd_##accumulator_type##_t diff_i = load_and_convert(a + i) - load_and_convert(b + i); \
|
|
121
177
|
for (simsimd_size_t j = 0; j != n; ++j) { \
|
|
122
178
|
simsimd_##accumulator_type##_t diff_j = load_and_convert(a + j) - load_and_convert(b + j); \
|
|
123
179
|
simsimd_##accumulator_type##_t c_ij = load_and_convert(c + i * n + j); \
|
|
124
|
-
|
|
180
|
+
cdiff_j += c_ij * diff_j; \
|
|
125
181
|
} \
|
|
126
|
-
sum += diff_i *
|
|
182
|
+
sum += diff_i * cdiff_j; \
|
|
127
183
|
} \
|
|
128
184
|
*result = (simsimd_distance_t)SIMSIMD_SQRT(sum); \
|
|
129
185
|
}
|
|
130
186
|
|
|
131
|
-
SIMSIMD_MAKE_BILINEAR(serial, f64, f64, SIMSIMD_DEREFERENCE)
|
|
132
|
-
|
|
187
|
+
SIMSIMD_MAKE_BILINEAR(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64_serial
|
|
188
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f64c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f64c_serial
|
|
189
|
+
SIMSIMD_MAKE_MAHALANOBIS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f64_serial
|
|
133
190
|
|
|
134
|
-
SIMSIMD_MAKE_BILINEAR(serial, f32, f32, SIMSIMD_DEREFERENCE)
|
|
135
|
-
|
|
191
|
+
SIMSIMD_MAKE_BILINEAR(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_serial
|
|
192
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f32c, f32, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_serial
|
|
193
|
+
SIMSIMD_MAKE_MAHALANOBIS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_serial
|
|
136
194
|
|
|
137
|
-
SIMSIMD_MAKE_BILINEAR(serial, f16, f32, SIMSIMD_F16_TO_F32)
|
|
138
|
-
|
|
195
|
+
SIMSIMD_MAKE_BILINEAR(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_serial
|
|
196
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, f16c, f32, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_serial
|
|
197
|
+
SIMSIMD_MAKE_MAHALANOBIS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_serial
|
|
139
198
|
|
|
140
|
-
SIMSIMD_MAKE_BILINEAR(serial, bf16, f32, SIMSIMD_BF16_TO_F32)
|
|
141
|
-
|
|
199
|
+
SIMSIMD_MAKE_BILINEAR(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_serial
|
|
200
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(serial, bf16c, f32, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_serial
|
|
201
|
+
SIMSIMD_MAKE_MAHALANOBIS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_serial
|
|
142
202
|
|
|
143
|
-
SIMSIMD_MAKE_BILINEAR(accurate, f32, f64, SIMSIMD_DEREFERENCE)
|
|
144
|
-
|
|
203
|
+
SIMSIMD_MAKE_BILINEAR(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32_accurate
|
|
204
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f32c, f64, SIMSIMD_DEREFERENCE) // simsimd_bilinear_f32c_accurate
|
|
205
|
+
SIMSIMD_MAKE_MAHALANOBIS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_mahalanobis_f32_accurate
|
|
145
206
|
|
|
146
|
-
SIMSIMD_MAKE_BILINEAR(accurate, f16, f64, SIMSIMD_F16_TO_F32)
|
|
147
|
-
|
|
207
|
+
SIMSIMD_MAKE_BILINEAR(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16_accurate
|
|
208
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, f16c, f64, SIMSIMD_F16_TO_F32) // simsimd_bilinear_f16c_accurate
|
|
209
|
+
SIMSIMD_MAKE_MAHALANOBIS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_mahalanobis_f16_accurate
|
|
148
210
|
|
|
149
|
-
SIMSIMD_MAKE_BILINEAR(accurate, bf16, f64, SIMSIMD_BF16_TO_F32)
|
|
150
|
-
|
|
211
|
+
SIMSIMD_MAKE_BILINEAR(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16_accurate
|
|
212
|
+
SIMSIMD_MAKE_COMPLEX_BILINEAR(accurate, bf16c, f64, SIMSIMD_BF16_TO_F32) // simsimd_bilinear_bf16c_accurate
|
|
213
|
+
SIMSIMD_MAKE_MAHALANOBIS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_mahalanobis_bf16_accurate
|
|
151
214
|
|
|
152
215
|
#if _SIMSIMD_TARGET_ARM
|
|
153
216
|
#if SIMSIMD_TARGET_NEON
|
|
@@ -160,25 +223,25 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f32_neon(simsimd_f32_t const *a, simsimd_f3
|
|
|
160
223
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
161
224
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
162
225
|
float32x4_t a_vec = vdupq_n_f32(a[i]);
|
|
163
|
-
float32x4_t
|
|
226
|
+
float32x4_t cb_j_vec = vdupq_n_f32(0);
|
|
164
227
|
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
165
228
|
float32x4_t b_vec = vld1q_f32(b + j);
|
|
166
229
|
float32x4_t c_vec = vld1q_f32(c + i * n + j);
|
|
167
|
-
|
|
230
|
+
cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
|
|
168
231
|
}
|
|
169
|
-
sum_vec = vmlaq_f32(sum_vec, a_vec,
|
|
232
|
+
sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
|
|
170
233
|
}
|
|
171
234
|
|
|
172
235
|
// Handle the tail of every row
|
|
173
236
|
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
174
|
-
simsimd_size_t tail_length = n % 4;
|
|
175
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
237
|
+
simsimd_size_t const tail_length = n % 4;
|
|
238
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
176
239
|
if (tail_length) {
|
|
177
240
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
178
241
|
simsimd_f32_t a_i = a[i];
|
|
179
|
-
simsimd_f32_t
|
|
180
|
-
for (simsimd_size_t j = tail_start; j != n; ++j)
|
|
181
|
-
sum += a[i] *
|
|
242
|
+
simsimd_f32_t cb_j = 0;
|
|
243
|
+
for (simsimd_size_t j = tail_start; j != n; ++j) cb_j += b[j] * c[i * n + j];
|
|
244
|
+
sum += a[i] * cb_j;
|
|
182
245
|
}
|
|
183
246
|
}
|
|
184
247
|
|
|
@@ -190,35 +253,87 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_neon(simsimd_f32_t const *a, simsimd
|
|
|
190
253
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
191
254
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
192
255
|
float32x4_t diff_i_vec = vdupq_n_f32(a[i] - b[i]);
|
|
193
|
-
float32x4_t
|
|
256
|
+
float32x4_t cdiff_j_vec = vdupq_n_f32(0);
|
|
194
257
|
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
195
258
|
float32x4_t diff_j_vec = vsubq_f32(vld1q_f32(a + j), vld1q_f32(b + j));
|
|
196
259
|
float32x4_t c_vec = vld1q_f32(c + i * n + j);
|
|
197
|
-
|
|
260
|
+
cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
|
|
198
261
|
}
|
|
199
262
|
|
|
200
|
-
sum_vec = vmlaq_f32(sum_vec, diff_i_vec,
|
|
263
|
+
sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
|
|
201
264
|
}
|
|
202
265
|
|
|
203
266
|
// Handle the tail of every row
|
|
204
267
|
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
205
|
-
simsimd_size_t tail_length = n % 4;
|
|
206
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
268
|
+
simsimd_size_t const tail_length = n % 4;
|
|
269
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
207
270
|
if (tail_length) {
|
|
208
271
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
209
272
|
simsimd_f32_t diff_i = a[i] - b[i];
|
|
210
|
-
simsimd_f32_t
|
|
273
|
+
simsimd_f32_t cdiff_j = 0;
|
|
211
274
|
for (simsimd_size_t j = tail_start; j != n; ++j) {
|
|
212
275
|
simsimd_f32_t diff_j = a[j] - b[j];
|
|
213
|
-
|
|
276
|
+
cdiff_j += diff_j * c[i * n + j];
|
|
214
277
|
}
|
|
215
|
-
sum += diff_i *
|
|
278
|
+
sum += diff_i * cdiff_j;
|
|
216
279
|
}
|
|
217
280
|
}
|
|
218
281
|
|
|
219
282
|
*result = _simsimd_sqrt_f64_neon(sum);
|
|
220
283
|
}
|
|
221
284
|
|
|
285
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_neon(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
|
|
286
|
+
simsimd_f32c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
|
|
287
|
+
simsimd_f32_t sum_real = 0;
|
|
288
|
+
simsimd_f32_t sum_imag = 0;
|
|
289
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
290
|
+
simsimd_f32c_t a_i = a[i];
|
|
291
|
+
simsimd_f32c_t cb_j;
|
|
292
|
+
float32x4_t cb_j_real_vec = vdupq_n_f32(0);
|
|
293
|
+
float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
|
|
294
|
+
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
295
|
+
// Unpack the input arrays into real and imaginary parts:
|
|
296
|
+
float32x4x2_t b_vec = vld2q_f32((simsimd_f32_t const *)(b + j));
|
|
297
|
+
float32x4x2_t c_vec = vld2q_f32((simsimd_f32_t const *)(c + i * n + j));
|
|
298
|
+
float32x4_t b_real_vec = b_vec.val[0];
|
|
299
|
+
float32x4_t b_imag_vec = b_vec.val[1];
|
|
300
|
+
float32x4_t c_real_vec = c_vec.val[0];
|
|
301
|
+
float32x4_t c_imag_vec = c_vec.val[1];
|
|
302
|
+
|
|
303
|
+
// Compute the dot product:
|
|
304
|
+
cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
305
|
+
cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
306
|
+
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
307
|
+
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
308
|
+
}
|
|
309
|
+
cb_j.real = vaddvq_f32(cb_j_real_vec);
|
|
310
|
+
cb_j.imag = vaddvq_f32(cb_j_imag_vec);
|
|
311
|
+
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
312
|
+
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Handle the tail of every row
|
|
316
|
+
simsimd_size_t const tail_length = n % 4;
|
|
317
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
318
|
+
if (tail_length) {
|
|
319
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
320
|
+
simsimd_f32c_t a_i = a[i];
|
|
321
|
+
simsimd_f32c_t cb_j = {0, 0};
|
|
322
|
+
for (simsimd_size_t j = tail_start; j != n; ++j) {
|
|
323
|
+
simsimd_f32c_t b_j = b[j];
|
|
324
|
+
simsimd_f32c_t c_ij = c[i * n + j];
|
|
325
|
+
cb_j.real += b_j.real * c_ij.real - b_j.imag * c_ij.imag;
|
|
326
|
+
cb_j.imag += b_j.real * c_ij.imag + b_j.imag * c_ij.real;
|
|
327
|
+
}
|
|
328
|
+
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
329
|
+
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
results[0] = sum_real;
|
|
334
|
+
results[1] = sum_imag;
|
|
335
|
+
}
|
|
336
|
+
|
|
222
337
|
#pragma clang attribute pop
|
|
223
338
|
#pragma GCC pop_options
|
|
224
339
|
#endif // SIMSIMD_TARGET_NEON
|
|
@@ -234,26 +349,26 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_neon(simsimd_f16_t const *a, simsimd_f1
|
|
|
234
349
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
235
350
|
// MSVC doesn't recognize `vdup_n_f16` as a valid intrinsic
|
|
236
351
|
float32x4_t a_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
|
|
237
|
-
float32x4_t
|
|
352
|
+
float32x4_t cb_j_vec = vdupq_n_f32(0);
|
|
238
353
|
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
239
354
|
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
|
|
240
355
|
float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
|
|
241
|
-
|
|
356
|
+
cb_j_vec = vmlaq_f32(cb_j_vec, b_vec, c_vec);
|
|
242
357
|
}
|
|
243
|
-
sum_vec = vmlaq_f32(sum_vec, a_vec,
|
|
358
|
+
sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
|
|
244
359
|
}
|
|
245
360
|
|
|
246
361
|
// Handle the tail of every row
|
|
247
362
|
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
248
|
-
simsimd_size_t tail_length = n % 4;
|
|
249
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
363
|
+
simsimd_size_t const tail_length = n % 4;
|
|
364
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
250
365
|
if (tail_length) {
|
|
251
366
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
252
367
|
simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
|
|
253
368
|
float32x4_t b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
|
|
254
369
|
float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
|
|
255
|
-
simsimd_f32_t
|
|
256
|
-
sum += a_i *
|
|
370
|
+
simsimd_f32_t cb_j = vaddvq_f32(vmulq_f32(b_vec, c_vec));
|
|
371
|
+
sum += a_i * cb_j;
|
|
257
372
|
}
|
|
258
373
|
}
|
|
259
374
|
|
|
@@ -268,21 +383,21 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const *a, simsimd
|
|
|
268
383
|
float32x4_t a_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(a + i))));
|
|
269
384
|
float32x4_t b_i_vec = vcvt_f32_f16(vreinterpret_f16_s16(vdup_n_s16(*(short const *)(b + i))));
|
|
270
385
|
float32x4_t diff_i_vec = vsubq_f32(a_i_vec, b_i_vec);
|
|
271
|
-
float32x4_t
|
|
386
|
+
float32x4_t cdiff_j_vec = vdupq_n_f32(0);
|
|
272
387
|
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
273
388
|
float32x4_t a_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(a + j)));
|
|
274
389
|
float32x4_t b_j_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(b + j)));
|
|
275
390
|
float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
|
|
276
391
|
float32x4_t c_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)(c + i * n + j)));
|
|
277
|
-
|
|
392
|
+
cdiff_j_vec = vmlaq_f32(cdiff_j_vec, diff_j_vec, c_vec);
|
|
278
393
|
}
|
|
279
|
-
sum_vec = vmlaq_f32(sum_vec, diff_i_vec,
|
|
394
|
+
sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
|
|
280
395
|
}
|
|
281
396
|
|
|
282
397
|
// Handle the tail of every row
|
|
283
398
|
simsimd_f32_t sum = vaddvq_f32(sum_vec);
|
|
284
|
-
simsimd_size_t tail_length = n % 4;
|
|
285
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
399
|
+
simsimd_size_t const tail_length = n % 4;
|
|
400
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
286
401
|
if (tail_length) {
|
|
287
402
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
288
403
|
simsimd_f32_t a_i = vaddvq_f32(vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a + i, 1)));
|
|
@@ -292,14 +407,94 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_neon(simsimd_f16_t const *a, simsimd
|
|
|
292
407
|
float32x4_t b_j_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b + tail_start, tail_length));
|
|
293
408
|
float32x4_t diff_j_vec = vsubq_f32(a_j_vec, b_j_vec);
|
|
294
409
|
float32x4_t c_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(c + i * n + tail_start, tail_length));
|
|
295
|
-
simsimd_f32_t
|
|
296
|
-
sum += diff_i *
|
|
410
|
+
simsimd_f32_t cdiff_j = vaddvq_f32(vmulq_f32(diff_j_vec, c_vec));
|
|
411
|
+
sum += diff_i * cdiff_j;
|
|
297
412
|
}
|
|
298
413
|
}
|
|
299
414
|
|
|
300
415
|
*result = _simsimd_sqrt_f32_neon(sum);
|
|
301
416
|
}
|
|
302
417
|
|
|
418
|
+
SIMSIMD_INTERNAL simsimd_f32_t _simsimd_reduce_f16x8_neon(float16x8_t vec) {
|
|
419
|
+
// Split the 8-element vector into two 4-element vectors
|
|
420
|
+
float16x4_t low = vget_low_f16(vec); // Lower 4 elements
|
|
421
|
+
float16x4_t high = vget_high_f16(vec); // Upper 4 elements
|
|
422
|
+
|
|
423
|
+
// Add the lower and upper parts
|
|
424
|
+
float16x4_t sum = vadd_f16(low, high);
|
|
425
|
+
|
|
426
|
+
// Perform pairwise addition to reduce 4 elements to 2, then to 1
|
|
427
|
+
sum = vpadd_f16(sum, sum); // First reduction: 4 -> 2
|
|
428
|
+
sum = vpadd_f16(sum, sum); // Second reduction: 2 -> 1
|
|
429
|
+
|
|
430
|
+
// Convert the remaining half-precision value to single-precision and return
|
|
431
|
+
return vgetq_lane_f32(vcvt_f32_f16(sum), 0);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
SIMSIMD_INTERNAL float16x8x2_t _simsimd_partial_load_f16x8x2_neon(simsimd_f16c_t const *x, simsimd_size_t n) {
|
|
435
|
+
union {
|
|
436
|
+
float16x8x2_t vecs;
|
|
437
|
+
simsimd_f16_t scalars[2][8];
|
|
438
|
+
} result;
|
|
439
|
+
simsimd_size_t i = 0;
|
|
440
|
+
for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
|
|
441
|
+
for (; i < 8; ++i) result.scalars[0][i] = 0, result.scalars[1][i] = 0;
|
|
442
|
+
return result.vecs;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_neon(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
|
|
446
|
+
simsimd_f16c_t const *c, simsimd_size_t n, simsimd_distance_t *results) {
|
|
447
|
+
simsimd_f32_t sum_real = 0;
|
|
448
|
+
simsimd_f32_t sum_imag = 0;
|
|
449
|
+
simsimd_size_t const tail_length = n % 8;
|
|
450
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
451
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
452
|
+
simsimd_f32c_t a_i = {simsimd_f16_to_f32(&a[i].real), simsimd_f16_to_f32(&a[i].imag)};
|
|
453
|
+
float16x8_t cb_j_real_vec = vdupq_n_f16(0);
|
|
454
|
+
float16x8_t cb_j_imag_vec = vdupq_n_f16(0);
|
|
455
|
+
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
456
|
+
// Unpack the input arrays into real and imaginary parts:
|
|
457
|
+
float16x8x2_t b_vec = vld2q_f16((float16_t const *)(b + j));
|
|
458
|
+
float16x8x2_t c_vec = vld2q_f16((float16_t const *)(c + i * n + j));
|
|
459
|
+
float16x8_t b_real_vec = b_vec.val[0];
|
|
460
|
+
float16x8_t b_imag_vec = b_vec.val[1];
|
|
461
|
+
float16x8_t c_real_vec = c_vec.val[0];
|
|
462
|
+
float16x8_t c_imag_vec = c_vec.val[1];
|
|
463
|
+
|
|
464
|
+
// Compute the dot product:
|
|
465
|
+
cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
466
|
+
cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
467
|
+
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
468
|
+
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
469
|
+
}
|
|
470
|
+
// Handle row tails
|
|
471
|
+
if (tail_length) {
|
|
472
|
+
// Unpack the input arrays into real and imaginary parts:
|
|
473
|
+
float16x8x2_t b_vec = _simsimd_partial_load_f16x8x2_neon(b + tail_start, tail_length);
|
|
474
|
+
float16x8x2_t c_vec = _simsimd_partial_load_f16x8x2_neon(c + i * n + tail_start, tail_length);
|
|
475
|
+
float16x8_t b_real_vec = b_vec.val[0];
|
|
476
|
+
float16x8_t b_imag_vec = b_vec.val[1];
|
|
477
|
+
float16x8_t c_real_vec = c_vec.val[0];
|
|
478
|
+
float16x8_t c_imag_vec = c_vec.val[1];
|
|
479
|
+
|
|
480
|
+
// Compute the dot product:
|
|
481
|
+
cb_j_real_vec = vfmaq_f16(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
482
|
+
cb_j_real_vec = vfmsq_f16(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
483
|
+
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
484
|
+
cb_j_imag_vec = vfmaq_f16(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
simsimd_f32c_t cb_j;
|
|
488
|
+
cb_j.real = _simsimd_reduce_f16x8_neon(cb_j_real_vec);
|
|
489
|
+
cb_j.imag = _simsimd_reduce_f16x8_neon(cb_j_imag_vec);
|
|
490
|
+
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
491
|
+
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
results[0] = sum_real;
|
|
495
|
+
results[1] = sum_imag;
|
|
496
|
+
}
|
|
497
|
+
|
|
303
498
|
#pragma clang attribute pop
|
|
304
499
|
#pragma GCC pop_options
|
|
305
500
|
#endif // SIMSIMD_TARGET_NEON_F16
|
|
@@ -314,26 +509,26 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_neon(simsimd_bf16_t const *a, simsimd_
|
|
|
314
509
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
315
510
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
316
511
|
float32x4_t a_vec = vdupq_n_f32(simsimd_bf16_to_f32(a + i));
|
|
317
|
-
float32x4_t
|
|
512
|
+
float32x4_t cb_j_vec = vdupq_n_f32(0);
|
|
318
513
|
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
319
514
|
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
|
|
320
515
|
bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
|
|
321
|
-
|
|
516
|
+
cb_j_vec = vbfdotq_f32(cb_j_vec, b_vec, c_vec);
|
|
322
517
|
}
|
|
323
|
-
sum_vec = vmlaq_f32(sum_vec, a_vec,
|
|
518
|
+
sum_vec = vmlaq_f32(sum_vec, a_vec, cb_j_vec);
|
|
324
519
|
}
|
|
325
520
|
|
|
326
521
|
// Handle the tail of every row
|
|
327
522
|
simsimd_f64_t sum = vaddvq_f32(sum_vec);
|
|
328
|
-
simsimd_size_t tail_length = n % 8;
|
|
329
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
523
|
+
simsimd_size_t const tail_length = n % 8;
|
|
524
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
330
525
|
if (tail_length) {
|
|
331
526
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
332
527
|
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
333
528
|
bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b + tail_start, tail_length);
|
|
334
529
|
bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
|
|
335
|
-
simsimd_f32_t
|
|
336
|
-
sum += a_i *
|
|
530
|
+
simsimd_f32_t cb_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), b_vec, c_vec));
|
|
531
|
+
sum += a_i * cb_j;
|
|
337
532
|
}
|
|
338
533
|
}
|
|
339
534
|
|
|
@@ -348,7 +543,7 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsi
|
|
|
348
543
|
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
349
544
|
simsimd_f32_t b_i = simsimd_bf16_to_f32(b + i);
|
|
350
545
|
float32x4_t diff_i_vec = vdupq_n_f32(a_i - b_i);
|
|
351
|
-
float32x4_t
|
|
546
|
+
float32x4_t cdiff_j_vec = vdupq_n_f32(0);
|
|
352
547
|
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
353
548
|
bfloat16x8_t a_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(a + j));
|
|
354
549
|
bfloat16x8_t b_j_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(b + j));
|
|
@@ -365,15 +560,15 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsi
|
|
|
365
560
|
bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
|
|
366
561
|
|
|
367
562
|
bfloat16x8_t c_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)(c + i * n + j));
|
|
368
|
-
|
|
563
|
+
cdiff_j_vec = vbfdotq_f32(cdiff_j_vec, diff_j_vec, c_vec);
|
|
369
564
|
}
|
|
370
|
-
sum_vec = vmlaq_f32(sum_vec, diff_i_vec,
|
|
565
|
+
sum_vec = vmlaq_f32(sum_vec, diff_i_vec, cdiff_j_vec);
|
|
371
566
|
}
|
|
372
567
|
|
|
373
568
|
// Handle the tail of every row
|
|
374
569
|
simsimd_f32_t sum = vaddvq_f32(sum_vec);
|
|
375
|
-
simsimd_size_t tail_length = n % 8;
|
|
376
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
570
|
+
simsimd_size_t const tail_length = n % 8;
|
|
571
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
377
572
|
if (tail_length) {
|
|
378
573
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
379
574
|
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
@@ -392,14 +587,84 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_neon(simsimd_bf16_t const *a, simsi
|
|
|
392
587
|
bfloat16x8_t diff_j_vec = vcombine_bf16(vcvt_bf16_f32(diff_j_vec_low), vcvt_bf16_f32(diff_j_vec_high));
|
|
393
588
|
|
|
394
589
|
bfloat16x8_t c_vec = _simsimd_partial_load_bf16x8_neon(c + i * n + tail_start, tail_length);
|
|
395
|
-
simsimd_f32_t
|
|
396
|
-
sum += diff_i *
|
|
590
|
+
simsimd_f32_t cdiff_j = vaddvq_f32(vbfdotq_f32(vdupq_n_f32(0), diff_j_vec, c_vec));
|
|
591
|
+
sum += diff_i * cdiff_j;
|
|
397
592
|
}
|
|
398
593
|
}
|
|
399
594
|
|
|
400
595
|
*result = _simsimd_sqrt_f32_neon(sum);
|
|
401
596
|
}
|
|
402
597
|
|
|
598
|
+
SIMSIMD_INTERNAL int16x4x2_t _simsimd_partial_load_bf16x4x2_neon(simsimd_bf16c_t const *x, simsimd_size_t n) {
|
|
599
|
+
union {
|
|
600
|
+
int16x4x2_t vec;
|
|
601
|
+
simsimd_bf16_t scalars[2][4];
|
|
602
|
+
} result;
|
|
603
|
+
simsimd_size_t i = 0;
|
|
604
|
+
for (; i < n; ++i) result.scalars[0][i] = x[i].real, result.scalars[1][i] = x[i].imag;
|
|
605
|
+
for (; i < 4; ++i) result.scalars[1][i] = 0, result.scalars[1][i] = 0;
|
|
606
|
+
return result.vec;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_neon(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
|
|
610
|
+
simsimd_bf16c_t const *c, simsimd_size_t n,
|
|
611
|
+
simsimd_distance_t *results) {
|
|
612
|
+
simsimd_f32_t sum_real = 0;
|
|
613
|
+
simsimd_f32_t sum_imag = 0;
|
|
614
|
+
simsimd_size_t const tail_length = n % 4;
|
|
615
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
616
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
617
|
+
simsimd_f32c_t a_i = {simsimd_bf16_to_f32(&a[i].real), simsimd_bf16_to_f32(&a[i].imag)};
|
|
618
|
+
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
619
|
+
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
620
|
+
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
621
|
+
float32x4_t cb_j_real_vec = vdupq_n_f32(0);
|
|
622
|
+
float32x4_t cb_j_imag_vec = vdupq_n_f32(0);
|
|
623
|
+
for (simsimd_size_t j = 0; j + 4 <= n; j += 4) {
|
|
624
|
+
// Unpack the input arrays into real and imaginary parts.
|
|
625
|
+
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
626
|
+
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
627
|
+
int16x4x2_t b_vec = vld2_s16((short const *)(b + j));
|
|
628
|
+
int16x4x2_t c_vec = vld2_s16((short const *)(c + i * n + j));
|
|
629
|
+
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
630
|
+
float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
|
|
631
|
+
float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
|
|
632
|
+
float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
|
|
633
|
+
|
|
634
|
+
// Compute the dot product:
|
|
635
|
+
cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
636
|
+
cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
637
|
+
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
638
|
+
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
639
|
+
}
|
|
640
|
+
// Handle row tails
|
|
641
|
+
if (tail_length) {
|
|
642
|
+
// Unpack the input arrays into real and imaginary parts:
|
|
643
|
+
int16x4x2_t b_vec = _simsimd_partial_load_bf16x4x2_neon(b + tail_start, tail_length);
|
|
644
|
+
int16x4x2_t c_vec = _simsimd_partial_load_bf16x4x2_neon(c + i * n + tail_start, tail_length);
|
|
645
|
+
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
646
|
+
float32x4_t b_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[1]));
|
|
647
|
+
float32x4_t c_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[0]));
|
|
648
|
+
float32x4_t c_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(c_vec.val[1]));
|
|
649
|
+
|
|
650
|
+
// Compute the dot product:
|
|
651
|
+
cb_j_real_vec = vfmaq_f32(cb_j_real_vec, c_real_vec, b_real_vec);
|
|
652
|
+
cb_j_real_vec = vfmsq_f32(cb_j_real_vec, c_imag_vec, b_imag_vec);
|
|
653
|
+
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_real_vec, b_imag_vec);
|
|
654
|
+
cb_j_imag_vec = vfmaq_f32(cb_j_imag_vec, c_imag_vec, b_real_vec);
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
simsimd_f32c_t cb_j;
|
|
658
|
+
cb_j.real = vaddvq_f32(cb_j_real_vec);
|
|
659
|
+
cb_j.imag = vaddvq_f32(cb_j_imag_vec);
|
|
660
|
+
sum_real += a_i.real * cb_j.real - a_i.imag * cb_j.imag;
|
|
661
|
+
sum_imag += a_i.real * cb_j.imag + a_i.imag * cb_j.real;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
results[0] = sum_real;
|
|
665
|
+
results[1] = sum_imag;
|
|
666
|
+
}
|
|
667
|
+
|
|
403
668
|
#pragma clang attribute pop
|
|
404
669
|
#pragma GCC pop_options
|
|
405
670
|
#endif // SIMSIMD_TARGET_NEON_BF16
|
|
@@ -417,26 +682,26 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_haswell(simsimd_f16_t const *a, simsimd
|
|
|
417
682
|
__m256 sum_vec = _mm256_setzero_ps();
|
|
418
683
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
419
684
|
__m256 a_vec = _mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i)));
|
|
420
|
-
__m256
|
|
685
|
+
__m256 cb_j_vec = _mm256_setzero_ps();
|
|
421
686
|
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
422
687
|
__m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j)));
|
|
423
688
|
__m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
424
|
-
|
|
689
|
+
cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
|
|
425
690
|
}
|
|
426
|
-
sum_vec = _mm256_fmadd_ps(a_vec,
|
|
691
|
+
sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
427
692
|
}
|
|
428
693
|
|
|
429
694
|
// Handle the tail of every row
|
|
430
695
|
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
431
|
-
simsimd_size_t tail_length = n % 8;
|
|
432
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
696
|
+
simsimd_size_t const tail_length = n % 8;
|
|
697
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
433
698
|
if (tail_length) {
|
|
434
699
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
435
700
|
simsimd_f32_t a_i = _mm256_cvtss_f32(_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))));
|
|
436
701
|
__m256 b_vec = _simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length);
|
|
437
702
|
__m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
|
|
438
|
-
simsimd_f32_t
|
|
439
|
-
sum += a_i *
|
|
703
|
+
simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
|
|
704
|
+
sum += a_i * cb_j;
|
|
440
705
|
}
|
|
441
706
|
}
|
|
442
707
|
|
|
@@ -451,21 +716,21 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const *a, sims
|
|
|
451
716
|
__m256 diff_i_vec = _mm256_sub_ps( //
|
|
452
717
|
_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(a + i))), //
|
|
453
718
|
_mm256_cvtph_ps(_mm_set1_epi16(*(short const *)(b + i))));
|
|
454
|
-
__m256
|
|
719
|
+
__m256 cdiff_j_vec = _mm256_setzero_ps();
|
|
455
720
|
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
456
721
|
__m256 diff_j_vec = _mm256_sub_ps( //
|
|
457
722
|
_mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(a + j))),
|
|
458
723
|
_mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(b + j))));
|
|
459
724
|
__m256 c_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
460
|
-
|
|
725
|
+
cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
|
|
461
726
|
}
|
|
462
|
-
sum_vec = _mm256_fmadd_ps(diff_i_vec,
|
|
727
|
+
sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
463
728
|
}
|
|
464
729
|
|
|
465
730
|
// Handle the tail of every row
|
|
466
731
|
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
467
|
-
simsimd_size_t tail_length = n % 8;
|
|
468
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
732
|
+
simsimd_size_t const tail_length = n % 8;
|
|
733
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
469
734
|
if (tail_length) {
|
|
470
735
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
471
736
|
simsimd_f32_t diff_i = _mm256_cvtss_f32(_mm256_sub_ps( //
|
|
@@ -475,8 +740,8 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_haswell(simsimd_f16_t const *a, sims
|
|
|
475
740
|
_simsimd_partial_load_f16x8_haswell(a + tail_start, tail_length),
|
|
476
741
|
_simsimd_partial_load_f16x8_haswell(b + tail_start, tail_length));
|
|
477
742
|
__m256 c_vec = _simsimd_partial_load_f16x8_haswell(c + i * n + tail_start, tail_length);
|
|
478
|
-
simsimd_f32_t
|
|
479
|
-
sum += diff_i *
|
|
743
|
+
simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
|
|
744
|
+
sum += diff_i * cdiff_j;
|
|
480
745
|
}
|
|
481
746
|
}
|
|
482
747
|
|
|
@@ -490,19 +755,19 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const *a, simsi
|
|
|
490
755
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
491
756
|
// The `simsimd_bf16_to_f32` is cheaper than `_simsimd_bf16x8_to_f32x8_haswell`
|
|
492
757
|
__m256 a_vec = _mm256_set1_ps(simsimd_bf16_to_f32(a + i));
|
|
493
|
-
__m256
|
|
758
|
+
__m256 cb_j_vec = _mm256_setzero_ps();
|
|
494
759
|
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
495
760
|
__m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j)));
|
|
496
761
|
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
497
|
-
|
|
762
|
+
cb_j_vec = _mm256_fmadd_ps(b_vec, c_vec, cb_j_vec);
|
|
498
763
|
}
|
|
499
|
-
sum_vec = _mm256_fmadd_ps(a_vec,
|
|
764
|
+
sum_vec = _mm256_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
500
765
|
}
|
|
501
766
|
|
|
502
767
|
// Handle the tail of every row
|
|
503
768
|
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
504
|
-
simsimd_size_t tail_length = n % 8;
|
|
505
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
769
|
+
simsimd_size_t const tail_length = n % 8;
|
|
770
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
506
771
|
if (tail_length) {
|
|
507
772
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
508
773
|
simsimd_f32_t a_i = simsimd_bf16_to_f32(a + i);
|
|
@@ -510,8 +775,8 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_haswell(simsimd_bf16_t const *a, simsi
|
|
|
510
775
|
_simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length));
|
|
511
776
|
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell( //
|
|
512
777
|
_simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
|
|
513
|
-
simsimd_f32_t
|
|
514
|
-
sum += a_i *
|
|
778
|
+
simsimd_f32_t cb_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(b_vec, c_vec));
|
|
779
|
+
sum += a_i * cb_j;
|
|
515
780
|
}
|
|
516
781
|
}
|
|
517
782
|
|
|
@@ -526,21 +791,21 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, si
|
|
|
526
791
|
__m256 diff_i_vec = _mm256_sub_ps( //
|
|
527
792
|
_mm256_set1_ps(simsimd_bf16_to_f32(a + i)), //
|
|
528
793
|
_mm256_set1_ps(simsimd_bf16_to_f32(b + i)));
|
|
529
|
-
__m256
|
|
794
|
+
__m256 cdiff_j_vec = _mm256_setzero_ps();
|
|
530
795
|
for (simsimd_size_t j = 0; j + 8 <= n; j += 8) {
|
|
531
796
|
__m256 diff_j_vec = _mm256_sub_ps( //
|
|
532
797
|
_simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(a + j))), //
|
|
533
798
|
_simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(b + j))));
|
|
534
799
|
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)(c + i * n + j)));
|
|
535
|
-
|
|
800
|
+
cdiff_j_vec = _mm256_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
|
|
536
801
|
}
|
|
537
|
-
sum_vec = _mm256_fmadd_ps(diff_i_vec,
|
|
802
|
+
sum_vec = _mm256_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
538
803
|
}
|
|
539
804
|
|
|
540
805
|
// Handle the tail of every row
|
|
541
806
|
simsimd_f32_t sum = _simsimd_reduce_f32x8_haswell(sum_vec);
|
|
542
|
-
simsimd_size_t tail_length = n % 8;
|
|
543
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
807
|
+
simsimd_size_t const tail_length = n % 8;
|
|
808
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
544
809
|
if (tail_length) {
|
|
545
810
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
546
811
|
simsimd_f32_t diff_i = simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i);
|
|
@@ -549,8 +814,8 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, si
|
|
|
549
814
|
_simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b + tail_start, tail_length)));
|
|
550
815
|
__m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(
|
|
551
816
|
_simsimd_partial_load_bf16x8_haswell(c + i * n + tail_start, tail_length));
|
|
552
|
-
simsimd_f32_t
|
|
553
|
-
sum += diff_i *
|
|
817
|
+
simsimd_f32_t cdiff_j = _simsimd_reduce_f32x8_haswell(_mm256_mul_ps(diff_j_vec, c_vec));
|
|
818
|
+
sum += diff_i * cdiff_j;
|
|
554
819
|
}
|
|
555
820
|
}
|
|
556
821
|
|
|
@@ -566,16 +831,71 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_haswell(simsimd_bf16_t const *a, si
|
|
|
566
831
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2")
|
|
567
832
|
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
568
833
|
|
|
834
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake_under16unrolled(simsimd_f32_t const *a, simsimd_f32_t const *b,
|
|
835
|
+
simsimd_f32_t const *c, simsimd_size_t n,
|
|
836
|
+
simsimd_distance_t *result) {
|
|
837
|
+
// The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
|
|
838
|
+
// until the very end of the computation.
|
|
839
|
+
__mmask16 const mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
840
|
+
__m512 const b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
841
|
+
|
|
842
|
+
__m512 cb_j1 = _mm512_setzero_ps();
|
|
843
|
+
__m512 cb_j2 = _mm512_setzero_ps();
|
|
844
|
+
__m512 cb_j3 = _mm512_setzero_ps();
|
|
845
|
+
__m512 cb_j4 = _mm512_setzero_ps();
|
|
846
|
+
|
|
847
|
+
// Unroll the loop to process 4x ZMM registers at a time.
|
|
848
|
+
simsimd_size_t i = 0;
|
|
849
|
+
for (; i + 4 <= n; i += 4) {
|
|
850
|
+
cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
|
|
851
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
|
|
852
|
+
cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
|
|
853
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
|
|
854
|
+
cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
|
|
855
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
|
|
856
|
+
cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
|
|
857
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
if (i + 0 < n)
|
|
861
|
+
cb_j1 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 0)),
|
|
862
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 0])), cb_j1);
|
|
863
|
+
if (i + 1 < n)
|
|
864
|
+
cb_j2 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 1)),
|
|
865
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 1])), cb_j2);
|
|
866
|
+
if (i + 2 < n)
|
|
867
|
+
cb_j3 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 2)),
|
|
868
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 2])), cb_j3);
|
|
869
|
+
if (i + 3 < n)
|
|
870
|
+
cb_j4 = _mm512_fmadd_ps(_mm512_maskz_loadu_ps(mask, c + n * (i + 3)),
|
|
871
|
+
_mm512_mul_ps(b_vec, _mm512_set1_ps(a[i + 3])), cb_j4);
|
|
872
|
+
|
|
873
|
+
// Combine cb_j sums
|
|
874
|
+
__m512 sum_vec = _mm512_add_ps( //
|
|
875
|
+
_mm512_add_ps(cb_j1, cb_j2), //
|
|
876
|
+
_mm512_add_ps(cb_j3, cb_j4));
|
|
877
|
+
*result = _mm512_reduce_add_ps(sum_vec);
|
|
878
|
+
}
|
|
879
|
+
|
|
569
880
|
SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c,
|
|
570
881
|
simsimd_size_t n, simsimd_distance_t *result) {
|
|
571
|
-
|
|
572
|
-
|
|
882
|
+
|
|
883
|
+
// On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
|
|
884
|
+
// per row and 16 rows at a time, still keeping enough register space for temporaries.
|
|
885
|
+
if (n <= 16) {
|
|
886
|
+
simsimd_bilinear_f32_skylake_under16unrolled(a, b, c, n, result);
|
|
887
|
+
return;
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
// Default case for arbitrary size `n`
|
|
891
|
+
simsimd_size_t const tail_length = n % 16;
|
|
892
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
573
893
|
__m512 sum_vec = _mm512_setzero_ps();
|
|
574
|
-
__mmask16 tail_mask = (__mmask16)_bzhi_u32(
|
|
894
|
+
__mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
575
895
|
|
|
576
896
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
577
897
|
__m512 a_vec = _mm512_set1_ps(a[i]);
|
|
578
|
-
__m512
|
|
898
|
+
__m512 cb_j_vec = _mm512_setzero_ps();
|
|
579
899
|
__m512 b_vec, c_vec;
|
|
580
900
|
simsimd_size_t j = 0;
|
|
581
901
|
|
|
@@ -588,10 +908,10 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd
|
|
|
588
908
|
b_vec = _mm512_maskz_loadu_ps(tail_mask, b + tail_start);
|
|
589
909
|
c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
|
|
590
910
|
}
|
|
591
|
-
|
|
911
|
+
cb_j_vec = _mm512_fmadd_ps(b_vec, c_vec, cb_j_vec);
|
|
592
912
|
j += 16;
|
|
593
913
|
if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
|
|
594
|
-
sum_vec = _mm512_fmadd_ps(a_vec,
|
|
914
|
+
sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
595
915
|
}
|
|
596
916
|
|
|
597
917
|
*result = _mm512_reduce_add_ps(sum_vec);
|
|
@@ -600,14 +920,14 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f32_skylake(simsimd_f32_t const *a, simsimd
|
|
|
600
920
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b,
|
|
601
921
|
simsimd_f32_t const *c, simsimd_size_t n,
|
|
602
922
|
simsimd_distance_t *result) {
|
|
603
|
-
simsimd_size_t tail_length = n % 16;
|
|
604
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
923
|
+
simsimd_size_t const tail_length = n % 16;
|
|
924
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
605
925
|
__m512 sum_vec = _mm512_setzero_ps();
|
|
606
|
-
__mmask16 tail_mask = (__mmask16)_bzhi_u32(
|
|
926
|
+
__mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
607
927
|
|
|
608
928
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
609
929
|
__m512 diff_i_vec = _mm512_set1_ps(a[i] - b[i]);
|
|
610
|
-
__m512
|
|
930
|
+
__m512 cdiff_j_vec = _mm512_setzero_ps(), cdiff_j_bot_vec = _mm512_setzero_ps();
|
|
611
931
|
__m512 a_j_vec, b_j_vec, diff_j_vec, c_vec;
|
|
612
932
|
simsimd_size_t j = 0;
|
|
613
933
|
|
|
@@ -624,15 +944,243 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, sims
|
|
|
624
944
|
c_vec = _mm512_maskz_loadu_ps(tail_mask, c + i * n + tail_start);
|
|
625
945
|
}
|
|
626
946
|
diff_j_vec = _mm512_sub_ps(a_j_vec, b_j_vec);
|
|
627
|
-
|
|
947
|
+
cdiff_j_vec = _mm512_fmadd_ps(diff_j_vec, c_vec, cdiff_j_vec);
|
|
628
948
|
j += 16;
|
|
629
949
|
if (j < n) goto simsimd_bilinear_f32_skylake_cycle;
|
|
630
|
-
sum_vec = _mm512_fmadd_ps(diff_i_vec,
|
|
950
|
+
sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
631
951
|
}
|
|
632
952
|
|
|
633
953
|
*result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_ps(sum_vec));
|
|
634
954
|
}
|
|
635
955
|
|
|
956
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f32c_skylake(simsimd_f32c_t const *a, simsimd_f32c_t const *b,
|
|
957
|
+
simsimd_f32c_t const *c, simsimd_size_t n,
|
|
958
|
+
simsimd_distance_t *results) {
|
|
959
|
+
|
|
960
|
+
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
961
|
+
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
962
|
+
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
963
|
+
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
964
|
+
// one of the vectors.
|
|
965
|
+
__m512i const sign_flip_vec = _mm512_set1_epi64(0x8000000000000000);
|
|
966
|
+
|
|
967
|
+
// Default case for arbitrary size `n`
|
|
968
|
+
simsimd_size_t const tail_length = n % 8;
|
|
969
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
970
|
+
__mmask16 const tail_mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
971
|
+
simsimd_f32_t sum_real = 0;
|
|
972
|
+
simsimd_f32_t sum_imag = 0;
|
|
973
|
+
|
|
974
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
975
|
+
simsimd_f32_t const a_i_real = a[i].real;
|
|
976
|
+
simsimd_f32_t const a_i_imag = a[i].imag;
|
|
977
|
+
__m512 cb_j_real_vec = _mm512_setzero_ps();
|
|
978
|
+
__m512 cb_j_imag_vec = _mm512_setzero_ps();
|
|
979
|
+
__m512 b_vec, c_vec;
|
|
980
|
+
simsimd_size_t j = 0;
|
|
981
|
+
|
|
982
|
+
simsimd_bilinear_f32c_skylake_cycle:
|
|
983
|
+
if (j + 8 <= n) {
|
|
984
|
+
b_vec = _mm512_loadu_ps((simsimd_f32_t const *)(b + j));
|
|
985
|
+
c_vec = _mm512_loadu_ps((simsimd_f32_t const *)(c + i * n + j));
|
|
986
|
+
}
|
|
987
|
+
else {
|
|
988
|
+
b_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(b + tail_start));
|
|
989
|
+
c_vec = _mm512_maskz_loadu_ps(tail_mask, (simsimd_f32_t const *)(c + i * n + tail_start));
|
|
990
|
+
}
|
|
991
|
+
// The real part of the product: b.real * c.real - b.imag * c.imag.
|
|
992
|
+
// The subtraction will be performed later with a sign flip.
|
|
993
|
+
cb_j_real_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_real_vec);
|
|
994
|
+
// The imaginary part of the product: b.real * c.imag + b.imag * c.real.
|
|
995
|
+
// Swap the imaginary and real parts of `c` before multiplication:
|
|
996
|
+
c_vec = _mm512_permute_ps(c_vec, 0xB1); //? Swap adjacent entries within each pair
|
|
997
|
+
cb_j_imag_vec = _mm512_fmadd_ps(c_vec, b_vec, cb_j_imag_vec);
|
|
998
|
+
j += 8;
|
|
999
|
+
if (j < n) goto simsimd_bilinear_f32c_skylake_cycle;
|
|
1000
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1001
|
+
cb_j_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(cb_j_real_vec), sign_flip_vec));
|
|
1002
|
+
// Horizontal sums are the expensive part of the computation:
|
|
1003
|
+
simsimd_f32_t const cb_j_real = _mm512_reduce_add_ps(cb_j_real_vec);
|
|
1004
|
+
simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ps(cb_j_imag_vec);
|
|
1005
|
+
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1006
|
+
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
// Reduce horizontal sums:
|
|
1010
|
+
results[0] = sum_real;
|
|
1011
|
+
results[1] = sum_imag;
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake_under8unrolled(simsimd_f64_t const *a, simsimd_f64_t const *b,
|
|
1015
|
+
simsimd_f64_t const *c, simsimd_size_t n,
|
|
1016
|
+
simsimd_distance_t *result) {
|
|
1017
|
+
|
|
1018
|
+
// The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
|
|
1019
|
+
// until the very end of the computation.
|
|
1020
|
+
__mmask8 const row_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1021
|
+
__m512d const b_vec = _mm512_maskz_loadu_pd(row_mask, b);
|
|
1022
|
+
|
|
1023
|
+
__m512d cb_j1 = _mm512_setzero_pd();
|
|
1024
|
+
__m512d cb_j2 = _mm512_setzero_pd();
|
|
1025
|
+
__m512d cb_j3 = _mm512_setzero_pd();
|
|
1026
|
+
__m512d cb_j4 = _mm512_setzero_pd();
|
|
1027
|
+
|
|
1028
|
+
// clang-format off
|
|
1029
|
+
if (n > 0) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 0), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[0])), cb_j1);
|
|
1030
|
+
if (n > 1) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 1), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[1])), cb_j2);
|
|
1031
|
+
if (n > 2) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 2), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[2])), cb_j3);
|
|
1032
|
+
if (n > 3) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 3), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[3])), cb_j4);
|
|
1033
|
+
|
|
1034
|
+
if (n > 4) cb_j1 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 4), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[4])), cb_j1);
|
|
1035
|
+
if (n > 5) cb_j2 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 5), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[5])), cb_j2);
|
|
1036
|
+
if (n > 6) cb_j3 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 6), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[6])), cb_j3);
|
|
1037
|
+
if (n > 7) cb_j4 = _mm512_fmadd_pd(_mm512_maskz_loadu_pd(row_mask, c + n * 7), _mm512_mul_pd(b_vec, _mm512_set1_pd(a[7])), cb_j4);
|
|
1038
|
+
// clang-format on
|
|
1039
|
+
|
|
1040
|
+
// Combine cb_j sums
|
|
1041
|
+
__m512d sum_vec = _mm512_add_pd( //
|
|
1042
|
+
_mm512_add_pd(cb_j1, cb_j2), //
|
|
1043
|
+
_mm512_add_pd(cb_j3, cb_j4));
|
|
1044
|
+
*result = _mm512_reduce_add_pd(sum_vec);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c,
|
|
1048
|
+
simsimd_size_t n, simsimd_distance_t *result) {
|
|
1049
|
+
|
|
1050
|
+
// On modern x86 CPUs we have enough register space to load fairly large matrices with up to 16 cells
|
|
1051
|
+
// per row and 8 rows at a time, still keeping enough register space for temporaries.
|
|
1052
|
+
if (n <= 8) {
|
|
1053
|
+
simsimd_bilinear_f64_skylake_under8unrolled(a, b, c, n, result);
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
// Default case for arbitrary size `n`
|
|
1058
|
+
simsimd_size_t const tail_length = n % 8;
|
|
1059
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1060
|
+
__m512d sum_vec = _mm512_setzero_pd();
|
|
1061
|
+
__mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1062
|
+
|
|
1063
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1064
|
+
__m512d a_vec = _mm512_set1_pd(a[i]);
|
|
1065
|
+
__m512d cb_j_vec = _mm512_setzero_pd();
|
|
1066
|
+
__m512d b_vec, c_vec;
|
|
1067
|
+
simsimd_size_t j = 0;
|
|
1068
|
+
|
|
1069
|
+
simsimd_bilinear_f64_skylake_cycle:
|
|
1070
|
+
if (j + 8 <= n) {
|
|
1071
|
+
b_vec = _mm512_loadu_pd(b + j);
|
|
1072
|
+
c_vec = _mm512_loadu_pd(c + i * n + j);
|
|
1073
|
+
}
|
|
1074
|
+
else {
|
|
1075
|
+
b_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
|
|
1076
|
+
c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
|
|
1077
|
+
}
|
|
1078
|
+
cb_j_vec = _mm512_fmadd_pd(b_vec, c_vec, cb_j_vec);
|
|
1079
|
+
j += 8;
|
|
1080
|
+
if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
|
|
1081
|
+
sum_vec = _mm512_fmadd_pd(a_vec, cb_j_vec, sum_vec);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
*result = _mm512_reduce_add_pd(sum_vec);
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
SIMSIMD_PUBLIC void simsimd_mahalanobis_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b,
|
|
1088
|
+
simsimd_f64_t const *c, simsimd_size_t n,
|
|
1089
|
+
simsimd_distance_t *result) {
|
|
1090
|
+
simsimd_size_t const tail_length = n % 8;
|
|
1091
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1092
|
+
__mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
1093
|
+
__m512d sum_vec = _mm512_setzero_pd();
|
|
1094
|
+
|
|
1095
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1096
|
+
__m512d diff_i_vec = _mm512_set1_pd(a[i] - b[i]);
|
|
1097
|
+
__m512d cdiff_j_vec = _mm512_setzero_pd();
|
|
1098
|
+
__m512d a_j_vec, b_j_vec, diff_j_vec, c_vec;
|
|
1099
|
+
simsimd_size_t j = 0;
|
|
1100
|
+
|
|
1101
|
+
// The nested loop is cleaner to implement with a `goto` in this case:
|
|
1102
|
+
simsimd_bilinear_f64_skylake_cycle:
|
|
1103
|
+
if (j + 8 <= n) {
|
|
1104
|
+
a_j_vec = _mm512_loadu_pd(a + j);
|
|
1105
|
+
b_j_vec = _mm512_loadu_pd(b + j);
|
|
1106
|
+
c_vec = _mm512_loadu_pd(c + i * n + j);
|
|
1107
|
+
}
|
|
1108
|
+
else {
|
|
1109
|
+
a_j_vec = _mm512_maskz_loadu_pd(tail_mask, a + tail_start);
|
|
1110
|
+
b_j_vec = _mm512_maskz_loadu_pd(tail_mask, b + tail_start);
|
|
1111
|
+
c_vec = _mm512_maskz_loadu_pd(tail_mask, c + i * n + tail_start);
|
|
1112
|
+
}
|
|
1113
|
+
diff_j_vec = _mm512_sub_pd(a_j_vec, b_j_vec);
|
|
1114
|
+
cdiff_j_vec = _mm512_fmadd_pd(diff_j_vec, c_vec, cdiff_j_vec);
|
|
1115
|
+
j += 8;
|
|
1116
|
+
if (j < n) goto simsimd_bilinear_f64_skylake_cycle;
|
|
1117
|
+
sum_vec = _mm512_fmadd_pd(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
*result = _simsimd_sqrt_f64_haswell(_mm512_reduce_add_pd(sum_vec));
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f64c_skylake(simsimd_f64c_t const *a, simsimd_f64c_t const *b,
|
|
1124
|
+
simsimd_f64c_t const *c, simsimd_size_t n,
|
|
1125
|
+
simsimd_distance_t *results) {
|
|
1126
|
+
|
|
1127
|
+
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1128
|
+
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
1129
|
+
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
1130
|
+
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
1131
|
+
// one of the vectors.
|
|
1132
|
+
__m512i const sign_flip_vec = _mm512_set_epi64( //
|
|
1133
|
+
0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000, //
|
|
1134
|
+
0x8000000000000000, 0x0000000000000000, 0x8000000000000000, 0x0000000000000000 //
|
|
1135
|
+
);
|
|
1136
|
+
|
|
1137
|
+
// Default case for arbitrary size `n`
|
|
1138
|
+
simsimd_size_t const tail_length = n % 4;
|
|
1139
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1140
|
+
__mmask8 const tail_mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
1141
|
+
simsimd_f64_t sum_real = 0;
|
|
1142
|
+
simsimd_f64_t sum_imag = 0;
|
|
1143
|
+
|
|
1144
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1145
|
+
simsimd_f64_t const a_i_real = a[i].real;
|
|
1146
|
+
simsimd_f64_t const a_i_imag = a[i].imag;
|
|
1147
|
+
__m512d cb_j_real_vec = _mm512_setzero_pd();
|
|
1148
|
+
__m512d cb_j_imag_vec = _mm512_setzero_pd();
|
|
1149
|
+
__m512d b_vec, c_vec;
|
|
1150
|
+
simsimd_size_t j = 0;
|
|
1151
|
+
|
|
1152
|
+
simsimd_bilinear_f64c_skylake_cycle:
|
|
1153
|
+
if (j + 4 <= n) {
|
|
1154
|
+
b_vec = _mm512_loadu_pd((simsimd_f64_t const *)(b + j));
|
|
1155
|
+
c_vec = _mm512_loadu_pd((simsimd_f64_t const *)(c + i * n + j));
|
|
1156
|
+
}
|
|
1157
|
+
else {
|
|
1158
|
+
b_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(b + tail_start));
|
|
1159
|
+
c_vec = _mm512_maskz_loadu_pd(tail_mask, (simsimd_f64_t const *)(c + i * n + tail_start));
|
|
1160
|
+
}
|
|
1161
|
+
// The real part of the product: b.real * c.real - b.imag * c.imag.
|
|
1162
|
+
// The subtraction will be performed later with a sign flip.
|
|
1163
|
+
cb_j_real_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_real_vec);
|
|
1164
|
+
// The imaginary part of the product: b.real * c.imag + b.imag * c.real.
|
|
1165
|
+
// Swap the imaginary and real parts of `c` before multiplication:
|
|
1166
|
+
c_vec = _mm512_permute_pd(c_vec, 0x55); //? Same as 0b01010101.
|
|
1167
|
+
cb_j_imag_vec = _mm512_fmadd_pd(c_vec, b_vec, cb_j_imag_vec);
|
|
1168
|
+
j += 4;
|
|
1169
|
+
if (j < n) goto simsimd_bilinear_f64c_skylake_cycle;
|
|
1170
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1171
|
+
cb_j_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(cb_j_real_vec), sign_flip_vec));
|
|
1172
|
+
// Horizontal sums are the expensive part of the computation:
|
|
1173
|
+
simsimd_f64_t const cb_j_real = _mm512_reduce_add_pd(cb_j_real_vec);
|
|
1174
|
+
simsimd_f64_t const cb_j_imag = _mm512_reduce_add_pd(cb_j_imag_vec);
|
|
1175
|
+
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1176
|
+
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
// Reduce horizontal sums:
|
|
1180
|
+
results[0] = sum_real;
|
|
1181
|
+
results[1] = sum_imag;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
636
1184
|
#pragma clang attribute pop
|
|
637
1185
|
#pragma GCC pop_options
|
|
638
1186
|
#endif // SIMSIMD_TARGET_SKYLAKE
|
|
@@ -645,14 +1193,14 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f32_skylake(simsimd_f32_t const *a, sims
|
|
|
645
1193
|
|
|
646
1194
|
SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
647
1195
|
simsimd_bf16_t const *c, simsimd_size_t n, simsimd_distance_t *result) {
|
|
648
|
-
simsimd_size_t tail_length = n % 32;
|
|
649
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
1196
|
+
simsimd_size_t const tail_length = n % 32;
|
|
1197
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1198
|
+
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
650
1199
|
__m512 sum_vec = _mm512_setzero_ps();
|
|
651
|
-
__mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
652
1200
|
|
|
653
1201
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
654
1202
|
__m512 a_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i));
|
|
655
|
-
__m512
|
|
1203
|
+
__m512 cb_j_vec = _mm512_setzero_ps();
|
|
656
1204
|
__m512i b_vec, c_vec;
|
|
657
1205
|
simsimd_size_t j = 0;
|
|
658
1206
|
|
|
@@ -665,10 +1213,10 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd
|
|
|
665
1213
|
b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
|
|
666
1214
|
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
667
1215
|
}
|
|
668
|
-
|
|
1216
|
+
cb_j_vec = _mm512_dpbf16_ps(cb_j_vec, (__m512bh)(b_vec), (__m512bh)(c_vec));
|
|
669
1217
|
j += 32;
|
|
670
1218
|
if (j < n) goto simsimd_bilinear_bf16_genoa_cycle;
|
|
671
|
-
sum_vec = _mm512_fmadd_ps(a_vec,
|
|
1219
|
+
sum_vec = _mm512_fmadd_ps(a_vec, cb_j_vec, sum_vec);
|
|
672
1220
|
}
|
|
673
1221
|
|
|
674
1222
|
*result = _mm512_reduce_add_ps(sum_vec);
|
|
@@ -677,14 +1225,14 @@ SIMSIMD_PUBLIC void simsimd_bilinear_bf16_genoa(simsimd_bf16_t const *a, simsimd
|
|
|
677
1225
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b,
|
|
678
1226
|
simsimd_bf16_t const *c, simsimd_size_t n,
|
|
679
1227
|
simsimd_distance_t *result) {
|
|
680
|
-
simsimd_size_t tail_length = n % 32;
|
|
681
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
1228
|
+
simsimd_size_t const tail_length = n % 32;
|
|
1229
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1230
|
+
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
682
1231
|
__m512 sum_vec = _mm512_setzero_ps();
|
|
683
|
-
__mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
684
1232
|
|
|
685
1233
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
686
1234
|
__m512 diff_i_vec = _mm512_set1_ps(simsimd_bf16_to_f32(a + i) - simsimd_bf16_to_f32(b + i));
|
|
687
|
-
__m512
|
|
1235
|
+
__m512 cdiff_j_vec = _mm512_setzero_ps();
|
|
688
1236
|
__m512i a_j_vec, b_j_vec, diff_j_vec, c_vec;
|
|
689
1237
|
simsimd_size_t j = 0;
|
|
690
1238
|
|
|
@@ -701,15 +1249,78 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, sims
|
|
|
701
1249
|
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
702
1250
|
}
|
|
703
1251
|
diff_j_vec = _simsimd_substract_bf16x32_genoa(a_j_vec, b_j_vec);
|
|
704
|
-
|
|
1252
|
+
cdiff_j_vec = _mm512_dpbf16_ps(cdiff_j_vec, (__m512bh)(diff_j_vec), (__m512bh)(c_vec));
|
|
705
1253
|
j += 32;
|
|
706
1254
|
if (j < n) goto simsimd_mahalanobis_bf16_genoa_cycle;
|
|
707
|
-
sum_vec = _mm512_fmadd_ps(diff_i_vec,
|
|
1255
|
+
sum_vec = _mm512_fmadd_ps(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
708
1256
|
}
|
|
709
1257
|
|
|
710
1258
|
*result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ps(sum_vec));
|
|
711
1259
|
}
|
|
712
1260
|
|
|
1261
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_bf16c_genoa(simsimd_bf16c_t const *a, simsimd_bf16c_t const *b,
|
|
1262
|
+
simsimd_bf16c_t const *c, simsimd_size_t n,
|
|
1263
|
+
simsimd_distance_t *results) {
|
|
1264
|
+
|
|
1265
|
+
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1266
|
+
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
1267
|
+
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
1268
|
+
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
1269
|
+
// one of the vectors.
|
|
1270
|
+
__m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
|
|
1271
|
+
__m512i const swap_adjacent_vec = _mm512_set_epi8( //
|
|
1272
|
+
61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
|
|
1273
|
+
45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
|
|
1274
|
+
29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
|
|
1275
|
+
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
|
|
1276
|
+
);
|
|
1277
|
+
|
|
1278
|
+
// Default case for arbitrary size `n`
|
|
1279
|
+
simsimd_size_t const tail_length = n % 16;
|
|
1280
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1281
|
+
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
1282
|
+
simsimd_f64_t sum_real = 0;
|
|
1283
|
+
simsimd_f64_t sum_imag = 0;
|
|
1284
|
+
|
|
1285
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1286
|
+
simsimd_f32_t const a_i_real = a[i].real;
|
|
1287
|
+
simsimd_f32_t const a_i_imag = a[i].imag;
|
|
1288
|
+
__m512 cb_j_real_vec = _mm512_setzero_ps();
|
|
1289
|
+
__m512 cb_j_imag_vec = _mm512_setzero_ps();
|
|
1290
|
+
__m512i b_vec, c_vec;
|
|
1291
|
+
simsimd_size_t j = 0;
|
|
1292
|
+
|
|
1293
|
+
simsimd_bilinear_bf16c_skylake_cycle:
|
|
1294
|
+
if (j + 16 <= n) {
|
|
1295
|
+
b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
|
|
1296
|
+
c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
|
|
1297
|
+
}
|
|
1298
|
+
else {
|
|
1299
|
+
b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
|
|
1300
|
+
c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
|
|
1301
|
+
}
|
|
1302
|
+
cb_j_real_vec = _mm512_dpbf16_ps( //
|
|
1303
|
+
cb_j_real_vec, //
|
|
1304
|
+
(__m512bh)(_mm512_xor_si512(c_vec, sign_flip_vec)), //
|
|
1305
|
+
(__m512bh)b_vec);
|
|
1306
|
+
cb_j_imag_vec = _mm512_dpbf16_ps( //
|
|
1307
|
+
cb_j_imag_vec, //
|
|
1308
|
+
(__m512bh)(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
|
|
1309
|
+
(__m512bh)b_vec);
|
|
1310
|
+
j += 16;
|
|
1311
|
+
if (j < n) goto simsimd_bilinear_bf16c_skylake_cycle;
|
|
1312
|
+
// Horizontal sums are the expensive part of the computation:
|
|
1313
|
+
simsimd_f64_t const cb_j_real = _simsimd_reduce_f32x16_skylake(cb_j_real_vec);
|
|
1314
|
+
simsimd_f64_t const cb_j_imag = _simsimd_reduce_f32x16_skylake(cb_j_imag_vec);
|
|
1315
|
+
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1316
|
+
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
// Reduce horizontal sums:
|
|
1320
|
+
results[0] = sum_real;
|
|
1321
|
+
results[1] = sum_imag;
|
|
1322
|
+
}
|
|
1323
|
+
|
|
713
1324
|
#pragma clang attribute pop
|
|
714
1325
|
#pragma GCC pop_options
|
|
715
1326
|
#endif // SIMSIMD_TARGET_GENOA
|
|
@@ -720,17 +1331,83 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_bf16_genoa(simsimd_bf16_t const *a, sims
|
|
|
720
1331
|
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
|
|
721
1332
|
apply_to = function)
|
|
722
1333
|
|
|
1334
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire_under32unrolled(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
1335
|
+
simsimd_f16_t const *c, simsimd_size_t const n,
|
|
1336
|
+
simsimd_distance_t *result) {
|
|
1337
|
+
// The goal of this optimization is to avoid horizontal accumulation of the cb_j sums
|
|
1338
|
+
// until the very end of the computation.
|
|
1339
|
+
__mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1340
|
+
__m512h const b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
|
|
1341
|
+
|
|
1342
|
+
// Independently accumulate the partial sums into separate variables to avoid data-dependencies.
|
|
1343
|
+
__m512h cb_j1 = _mm512_setzero_ph();
|
|
1344
|
+
__m512h cb_j2 = _mm512_setzero_ph();
|
|
1345
|
+
__m512h cb_j3 = _mm512_setzero_ph();
|
|
1346
|
+
__m512h cb_j4 = _mm512_setzero_ph();
|
|
1347
|
+
|
|
1348
|
+
// Unroll the loop to process 4x ZMM registers at a time.
|
|
1349
|
+
simsimd_size_t i = 0;
|
|
1350
|
+
for (; i + 4 <= n; i += 4) {
|
|
1351
|
+
// If the code is compiled without native support for `_Float16`, we need a workaround
|
|
1352
|
+
// to avoid implicit casts from out `simsimd_f16_t` to `_Float16`.
|
|
1353
|
+
cb_j1 = _mm512_fmadd_ph(
|
|
1354
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
|
|
1355
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
|
|
1356
|
+
cb_j2 = _mm512_fmadd_ph(
|
|
1357
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
|
|
1358
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
|
|
1359
|
+
cb_j3 = _mm512_fmadd_ph(
|
|
1360
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
|
|
1361
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
|
|
1362
|
+
cb_j4 = _mm512_fmadd_ph(
|
|
1363
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
|
|
1364
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// Handle the tail of the loop:
|
|
1368
|
+
if (i + 0 < n)
|
|
1369
|
+
cb_j1 = _mm512_fmadd_ph(
|
|
1370
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 0))),
|
|
1371
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 0]))), cb_j1);
|
|
1372
|
+
if (i + 1 < n)
|
|
1373
|
+
cb_j2 = _mm512_fmadd_ph(
|
|
1374
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 1))),
|
|
1375
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 1]))), cb_j2);
|
|
1376
|
+
if (i + 2 < n)
|
|
1377
|
+
cb_j3 = _mm512_fmadd_ph(
|
|
1378
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 2))),
|
|
1379
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 2]))), cb_j3);
|
|
1380
|
+
if (i + 3 < n)
|
|
1381
|
+
cb_j4 = _mm512_fmadd_ph(
|
|
1382
|
+
_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c + n * (i + 3))),
|
|
1383
|
+
_mm512_mul_ph(b_vec, _mm512_castsi512_ph(_mm512_set1_epi16(((simsimd_i16_t const *)a)[i + 3]))), cb_j4);
|
|
1384
|
+
|
|
1385
|
+
// Combine cb_j sums
|
|
1386
|
+
__m512h sum_vec = _mm512_add_ph( //
|
|
1387
|
+
_mm512_add_ph(cb_j1, cb_j2), //
|
|
1388
|
+
_mm512_add_ph(cb_j3, cb_j4));
|
|
1389
|
+
*result = _mm512_reduce_add_ph(sum_vec);
|
|
1390
|
+
}
|
|
1391
|
+
|
|
723
1392
|
SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
724
1393
|
simsimd_f16_t const *c, simsimd_size_t n,
|
|
725
1394
|
simsimd_distance_t *result) {
|
|
726
|
-
|
|
727
|
-
|
|
1395
|
+
|
|
1396
|
+
// On modern x86 CPUs we have enough register space to load fairly large matrices with up to 32 cells
|
|
1397
|
+
// per row and 32 rows at a time, still keeping enough register space for temporaries.
|
|
1398
|
+
if (n <= 32) {
|
|
1399
|
+
simsimd_bilinear_f16_sapphire_under32unrolled(a, b, c, n, result);
|
|
1400
|
+
return;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
simsimd_size_t const tail_length = n % 32;
|
|
1404
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1405
|
+
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
728
1406
|
__m512h sum_vec = _mm512_setzero_ph();
|
|
729
|
-
__mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
730
1407
|
|
|
731
1408
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
732
1409
|
__m512h a_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
|
|
733
|
-
__m512h
|
|
1410
|
+
__m512h cb_j_vec = _mm512_setzero_ph();
|
|
734
1411
|
__m512i b_vec, c_vec;
|
|
735
1412
|
simsimd_size_t j = 0;
|
|
736
1413
|
|
|
@@ -743,10 +1420,10 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsim
|
|
|
743
1420
|
b_vec = _mm512_maskz_loadu_epi16(tail_mask, b + tail_start);
|
|
744
1421
|
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
745
1422
|
}
|
|
746
|
-
|
|
1423
|
+
cb_j_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(c_vec), cb_j_vec);
|
|
747
1424
|
j += 32;
|
|
748
1425
|
if (j < n) goto simsimd_bilinear_f16_sapphire_cycle;
|
|
749
|
-
sum_vec = _mm512_fmadd_ph(a_vec,
|
|
1426
|
+
sum_vec = _mm512_fmadd_ph(a_vec, cb_j_vec, sum_vec);
|
|
750
1427
|
}
|
|
751
1428
|
|
|
752
1429
|
*result = _mm512_reduce_add_ph(sum_vec);
|
|
@@ -755,16 +1432,16 @@ SIMSIMD_PUBLIC void simsimd_bilinear_f16_sapphire(simsimd_f16_t const *a, simsim
|
|
|
755
1432
|
SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b,
|
|
756
1433
|
simsimd_f16_t const *c, simsimd_size_t n,
|
|
757
1434
|
simsimd_distance_t *result) {
|
|
758
|
-
simsimd_size_t tail_length = n % 32;
|
|
759
|
-
simsimd_size_t tail_start = n - tail_length;
|
|
1435
|
+
simsimd_size_t const tail_length = n % 32;
|
|
1436
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1437
|
+
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
760
1438
|
__m512h sum_vec = _mm512_setzero_ph();
|
|
761
|
-
__mmask32 tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length);
|
|
762
1439
|
|
|
763
1440
|
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
764
1441
|
__m512h a_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(a + i)));
|
|
765
1442
|
__m512h b_i_vec = _mm512_castsi512_ph(_mm512_set1_epi16(*(short const *)(b + i)));
|
|
766
1443
|
__m512h diff_i_vec = _mm512_sub_ph(a_i_vec, b_i_vec);
|
|
767
|
-
__m512h
|
|
1444
|
+
__m512h cdiff_j_vec = _mm512_setzero_ph();
|
|
768
1445
|
__m512h diff_j_vec;
|
|
769
1446
|
__m512i a_j_vec, b_j_vec, c_vec;
|
|
770
1447
|
simsimd_size_t j = 0;
|
|
@@ -782,15 +1459,76 @@ SIMSIMD_PUBLIC void simsimd_mahalanobis_f16_sapphire(simsimd_f16_t const *a, sim
|
|
|
782
1459
|
c_vec = _mm512_maskz_loadu_epi16(tail_mask, c + i * n + tail_start);
|
|
783
1460
|
}
|
|
784
1461
|
diff_j_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_j_vec), _mm512_castsi512_ph(b_j_vec));
|
|
785
|
-
|
|
1462
|
+
cdiff_j_vec = _mm512_fmadd_ph(diff_j_vec, _mm512_castsi512_ph(c_vec), cdiff_j_vec);
|
|
786
1463
|
j += 32;
|
|
787
1464
|
if (j < n) goto simsimd_mahalanobis_f16_sapphire_cycle;
|
|
788
|
-
sum_vec = _mm512_fmadd_ph(diff_i_vec,
|
|
1465
|
+
sum_vec = _mm512_fmadd_ph(diff_i_vec, cdiff_j_vec, sum_vec);
|
|
789
1466
|
}
|
|
790
1467
|
|
|
791
1468
|
*result = _simsimd_sqrt_f32_haswell(_mm512_reduce_add_ph(sum_vec));
|
|
792
1469
|
}
|
|
793
1470
|
|
|
1471
|
+
SIMSIMD_PUBLIC void simsimd_bilinear_f16c_sapphire(simsimd_f16c_t const *a, simsimd_f16c_t const *b,
|
|
1472
|
+
simsimd_f16c_t const *c, simsimd_size_t n,
|
|
1473
|
+
simsimd_distance_t *results) {
|
|
1474
|
+
|
|
1475
|
+
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1476
|
+
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
1477
|
+
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
1478
|
+
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
1479
|
+
// one of the vectors.
|
|
1480
|
+
__m512i const sign_flip_vec = _mm512_set1_epi32(0x80000000);
|
|
1481
|
+
__m512i const swap_adjacent_vec = _mm512_set_epi8( //
|
|
1482
|
+
61, 60, 63, 62, 57, 56, 59, 58, 53, 52, 55, 54, 49, 48, 51, 50, // 4th 128-bit lane
|
|
1483
|
+
45, 44, 47, 46, 41, 40, 43, 42, 37, 36, 39, 38, 33, 32, 35, 34, // 3rd 128-bit lane
|
|
1484
|
+
29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, // 2nd 128-bit lane
|
|
1485
|
+
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 // 1st 128-bit lane
|
|
1486
|
+
);
|
|
1487
|
+
|
|
1488
|
+
// Default case for arbitrary size `n`
|
|
1489
|
+
simsimd_size_t const tail_length = n % 16;
|
|
1490
|
+
simsimd_size_t const tail_start = n - tail_length;
|
|
1491
|
+
__mmask32 const tail_mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, tail_length * 2);
|
|
1492
|
+
simsimd_f32_t sum_real = 0;
|
|
1493
|
+
simsimd_f32_t sum_imag = 0;
|
|
1494
|
+
|
|
1495
|
+
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
1496
|
+
simsimd_f32_t const a_i_real = a[i].real;
|
|
1497
|
+
simsimd_f32_t const a_i_imag = a[i].imag;
|
|
1498
|
+
__m512h cb_j_real_vec = _mm512_setzero_ph();
|
|
1499
|
+
__m512h cb_j_imag_vec = _mm512_setzero_ph();
|
|
1500
|
+
__m512i b_vec, c_vec;
|
|
1501
|
+
simsimd_size_t j = 0;
|
|
1502
|
+
|
|
1503
|
+
simsimd_bilinear_f16c_skylake_cycle:
|
|
1504
|
+
if (j + 16 <= n) {
|
|
1505
|
+
b_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(b + j));
|
|
1506
|
+
c_vec = _mm512_loadu_epi16((simsimd_i16_t const *)(c + i * n + j));
|
|
1507
|
+
}
|
|
1508
|
+
else {
|
|
1509
|
+
b_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(b + tail_start));
|
|
1510
|
+
c_vec = _mm512_maskz_loadu_epi16(tail_mask, (simsimd_i16_t const *)(c + i * n + tail_start));
|
|
1511
|
+
}
|
|
1512
|
+
cb_j_real_vec = _mm512_fmadd_ph( //
|
|
1513
|
+
_mm512_castsi512_ph(_mm512_xor_si512(c_vec, sign_flip_vec)), //
|
|
1514
|
+
_mm512_castsi512_ph(b_vec), cb_j_real_vec);
|
|
1515
|
+
cb_j_imag_vec = _mm512_fmadd_ph( //
|
|
1516
|
+
_mm512_castsi512_ph(_mm512_shuffle_epi8(c_vec, swap_adjacent_vec)), //
|
|
1517
|
+
_mm512_castsi512_ph(b_vec), cb_j_imag_vec);
|
|
1518
|
+
j += 16;
|
|
1519
|
+
if (j < n) goto simsimd_bilinear_f16c_skylake_cycle;
|
|
1520
|
+
// Horizontal sums are the expensive part of the computation:
|
|
1521
|
+
simsimd_f32_t const cb_j_real = _mm512_reduce_add_ph(cb_j_real_vec);
|
|
1522
|
+
simsimd_f32_t const cb_j_imag = _mm512_reduce_add_ph(cb_j_imag_vec);
|
|
1523
|
+
sum_real += a_i_real * cb_j_real - a_i_imag * cb_j_imag;
|
|
1524
|
+
sum_imag += a_i_real * cb_j_imag + a_i_imag * cb_j_real;
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
// Reduce horizontal sums:
|
|
1528
|
+
results[0] = sum_real;
|
|
1529
|
+
results[1] = sum_imag;
|
|
1530
|
+
}
|
|
1531
|
+
|
|
794
1532
|
#pragma clang attribute pop
|
|
795
1533
|
#pragma GCC pop_options
|
|
796
1534
|
#endif // SIMSIMD_TARGET_SAPPHIRE
|