usearch 2.15.2 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.gyp +1 -1
- package/c/CMakeLists.txt +10 -3
- package/cpp/README.md +1 -1
- package/cpp/bench.cpp +2 -2
- package/cpp/test.cpp +16 -0
- package/include/usearch/index.hpp +44 -17
- package/include/usearch/index_dense.hpp +51 -89
- package/include/usearch/index_plugins.hpp +468 -43
- package/javascript/lib.cpp +17 -5
- package/javascript/usearch.test.js +205 -34
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.git-blame-ignore-revs +2 -0
- package/simsimd/CODE_OF_CONDUCT.md +128 -0
- package/simsimd/CONTRIBUTING.md +84 -26
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +4 -4
- package/simsimd/MANIFEST.in +3 -0
- package/simsimd/Package.swift +15 -3
- package/simsimd/README.md +534 -177
- package/simsimd/binding.gyp +6 -2
- package/simsimd/build.rs +13 -1
- package/simsimd/c/lib.c +266 -53
- package/simsimd/include/simsimd/binary.h +321 -112
- package/simsimd/include/simsimd/curved.h +803 -0
- package/simsimd/include/simsimd/dot.h +694 -521
- package/simsimd/include/simsimd/elementwise.h +1437 -0
- package/simsimd/include/simsimd/geospatial.h +9 -3
- package/simsimd/include/simsimd/mesh.h +69 -0
- package/simsimd/include/simsimd/probability.h +234 -206
- package/simsimd/include/simsimd/simsimd.h +1476 -617
- package/simsimd/include/simsimd/sparse.h +1428 -0
- package/simsimd/include/simsimd/spatial.h +1556 -742
- package/simsimd/include/simsimd/types.h +276 -132
- package/simsimd/javascript/lib.c +1 -2
- package/simsimd/javascript/simsimd.ts +4 -4
- package/simsimd/package.json +4 -4
- package/simsimd/rust/lib.rs +179 -1
- package/simsimd/scripts/bench.cxx +1070 -0
- package/simsimd/scripts/bench.ipynb +187 -0
- package/simsimd/{javascript → scripts}/bench.js +1 -1
- package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
- package/simsimd/scripts/bench_vectors.py +700 -0
- package/simsimd/scripts/bench_vectors_live.py +213 -0
- package/simsimd/scripts/test.c +155 -0
- package/simsimd/{javascript → scripts}/test.mjs +2 -2
- package/simsimd/scripts/test.py +1410 -0
- package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
- package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
- /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
- /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
* - 32-bit IEEE floating point numbers
|
|
14
14
|
* - 16-bit IEEE floating point numbers
|
|
15
15
|
* - 16-bit brain floating point numbers
|
|
16
|
+
* - 8-bit unsigned integers
|
|
16
17
|
* - 8-bit signed integers
|
|
17
18
|
*
|
|
18
19
|
* For hardware architectures:
|
|
19
|
-
* - Arm
|
|
20
|
-
* - x86
|
|
20
|
+
* - Arm: NEON, SVE
|
|
21
|
+
* - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
|
|
21
22
|
*
|
|
22
23
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
23
24
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -54,6 +55,7 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_serial(simsimd_bf16_t const* a, simsimd_bf
|
|
|
54
55
|
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
55
56
|
|
|
56
57
|
SIMSIMD_PUBLIC void simsimd_dot_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
58
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
57
59
|
|
|
58
60
|
/* Double-precision serial backends for all numeric types.
|
|
59
61
|
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
@@ -82,7 +84,12 @@ SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const* a, simsimd_f16_t c
|
|
|
82
84
|
SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
83
85
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
84
86
|
|
|
85
|
-
SIMSIMD_PUBLIC void
|
|
87
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
88
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
89
|
+
|
|
90
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
91
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
92
|
+
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
86
93
|
|
|
87
94
|
/* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
|
|
88
95
|
* Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
|
|
@@ -116,6 +123,7 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
|
|
|
116
123
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
117
124
|
|
|
118
125
|
SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
126
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
119
127
|
|
|
120
128
|
/* SIMD-powered backends for various generations of AVX512 CPUs.
|
|
121
129
|
* Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
|
|
@@ -131,7 +139,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const* a, simsimd_f32_
|
|
|
131
139
|
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
132
140
|
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
133
141
|
|
|
134
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t*);
|
|
142
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
143
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
135
144
|
|
|
136
145
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
137
146
|
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
@@ -140,93 +149,107 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf
|
|
|
140
149
|
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
141
150
|
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
142
151
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
152
|
+
|
|
153
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
143
154
|
// clang-format on
|
|
144
155
|
|
|
145
|
-
#define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type,
|
|
146
|
-
SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const*
|
|
147
|
-
simsimd_##input_type##_t const*
|
|
148
|
-
simsimd_distance_t*
|
|
149
|
-
simsimd_##accumulator_type##_t ab = 0;
|
|
150
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
151
|
-
simsimd_##accumulator_type##_t ai =
|
|
152
|
-
simsimd_##accumulator_type##_t bi =
|
|
153
|
-
ab += ai * bi;
|
|
154
|
-
}
|
|
155
|
-
*result = ab;
|
|
156
|
+
#define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, load_and_convert) \
|
|
157
|
+
SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
158
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
159
|
+
simsimd_distance_t *result) { \
|
|
160
|
+
simsimd_##accumulator_type##_t ab = 0; \
|
|
161
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
162
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
163
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
164
|
+
ab += ai * bi; \
|
|
165
|
+
} \
|
|
166
|
+
*result = ab; \
|
|
156
167
|
}
|
|
157
168
|
|
|
158
|
-
#define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type,
|
|
159
|
-
SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const*
|
|
160
|
-
simsimd_##input_type##_t const*
|
|
161
|
-
simsimd_distance_t*
|
|
162
|
-
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0;
|
|
163
|
-
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) {
|
|
164
|
-
simsimd_##accumulator_type##_t ar =
|
|
165
|
-
simsimd_##accumulator_type##_t br =
|
|
166
|
-
simsimd_##accumulator_type##_t ai =
|
|
167
|
-
simsimd_##accumulator_type##_t bi =
|
|
168
|
-
ab_real += ar * br - ai * bi;
|
|
169
|
-
ab_imag += ar * bi + ai * br;
|
|
170
|
-
}
|
|
171
|
-
results[0] = ab_real;
|
|
172
|
-
results[1] = ab_imag;
|
|
169
|
+
#define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, load_and_convert) \
|
|
170
|
+
SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const *a, \
|
|
171
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
172
|
+
simsimd_distance_t *results) { \
|
|
173
|
+
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
|
|
174
|
+
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
|
|
175
|
+
simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
|
|
176
|
+
simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
|
|
177
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
|
|
178
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
|
|
179
|
+
ab_real += ar * br - ai * bi; \
|
|
180
|
+
ab_imag += ar * bi + ai * br; \
|
|
181
|
+
} \
|
|
182
|
+
results[0] = ab_real; \
|
|
183
|
+
results[1] = ab_imag; \
|
|
173
184
|
}
|
|
174
185
|
|
|
175
|
-
#define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type,
|
|
176
|
-
SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const*
|
|
177
|
-
simsimd_##input_type##_t const*
|
|
178
|
-
simsimd_distance_t*
|
|
179
|
-
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0;
|
|
180
|
-
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) {
|
|
181
|
-
simsimd_##accumulator_type##_t ar =
|
|
182
|
-
simsimd_##accumulator_type##_t br =
|
|
183
|
-
simsimd_##accumulator_type##_t ai =
|
|
184
|
-
simsimd_##accumulator_type##_t bi =
|
|
185
|
-
ab_real += ar * br + ai * bi;
|
|
186
|
-
ab_imag += ar * bi - ai * br;
|
|
187
|
-
}
|
|
188
|
-
results[0] = ab_real;
|
|
189
|
-
results[1] = ab_imag;
|
|
186
|
+
#define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, load_and_convert) \
|
|
187
|
+
SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const *a, \
|
|
188
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
189
|
+
simsimd_distance_t *results) { \
|
|
190
|
+
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
|
|
191
|
+
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
|
|
192
|
+
simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
|
|
193
|
+
simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
|
|
194
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
|
|
195
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
|
|
196
|
+
ab_real += ar * br + ai * bi; \
|
|
197
|
+
ab_imag += ar * bi - ai * br; \
|
|
198
|
+
} \
|
|
199
|
+
results[0] = ab_real; \
|
|
200
|
+
results[1] = ab_imag; \
|
|
190
201
|
}
|
|
191
202
|
|
|
192
|
-
SIMSIMD_MAKE_DOT(serial, f64, f64,
|
|
193
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64,
|
|
194
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64,
|
|
203
|
+
SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64_serial
|
|
204
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64c_serial
|
|
205
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f64c_serial
|
|
195
206
|
|
|
196
|
-
SIMSIMD_MAKE_DOT(serial, f32, f32,
|
|
197
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32,
|
|
198
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32,
|
|
207
|
+
SIMSIMD_MAKE_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_serial
|
|
208
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_serial
|
|
209
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_serial
|
|
199
210
|
|
|
200
|
-
SIMSIMD_MAKE_DOT(serial, f16, f32,
|
|
201
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32,
|
|
202
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32,
|
|
211
|
+
SIMSIMD_MAKE_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_serial
|
|
212
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_serial
|
|
213
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_serial
|
|
203
214
|
|
|
204
|
-
SIMSIMD_MAKE_DOT(serial, bf16, f32,
|
|
205
|
-
SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32,
|
|
206
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32,
|
|
215
|
+
SIMSIMD_MAKE_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_serial
|
|
216
|
+
SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_serial
|
|
217
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_serial
|
|
207
218
|
|
|
208
|
-
SIMSIMD_MAKE_DOT(serial, i8, i64,
|
|
219
|
+
SIMSIMD_MAKE_DOT(serial, i8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_i8_serial
|
|
220
|
+
SIMSIMD_MAKE_DOT(serial, u8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_u8_serial
|
|
209
221
|
|
|
210
|
-
SIMSIMD_MAKE_DOT(accurate, f32, f64,
|
|
211
|
-
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64,
|
|
212
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64,
|
|
222
|
+
SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_accurate
|
|
223
|
+
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_accurate
|
|
224
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_vdot_f32c_accurate
|
|
213
225
|
|
|
214
|
-
SIMSIMD_MAKE_DOT(accurate, f16, f64,
|
|
215
|
-
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64,
|
|
216
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64,
|
|
226
|
+
SIMSIMD_MAKE_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16_accurate
|
|
227
|
+
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_dot_f16c_accurate
|
|
228
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_vdot_f16c_accurate
|
|
217
229
|
|
|
218
|
-
SIMSIMD_MAKE_DOT(accurate, bf16, f64,
|
|
219
|
-
SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64,
|
|
220
|
-
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64,
|
|
230
|
+
SIMSIMD_MAKE_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16_accurate
|
|
231
|
+
SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_accurate
|
|
232
|
+
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_accurate
|
|
221
233
|
|
|
222
|
-
#if
|
|
234
|
+
#if _SIMSIMD_TARGET_ARM
|
|
223
235
|
#if SIMSIMD_TARGET_NEON
|
|
224
236
|
#pragma GCC push_options
|
|
225
|
-
#pragma GCC target("+simd")
|
|
226
|
-
#pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
|
|
237
|
+
#pragma GCC target("arch=armv8.2-a+simd")
|
|
238
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
239
|
+
|
|
240
|
+
SIMSIMD_INTERNAL float32x4_t _simsimd_partial_load_f32x4_neon(simsimd_f32_t const *a, simsimd_size_t n) {
|
|
241
|
+
union {
|
|
242
|
+
float32x4_t vec;
|
|
243
|
+
simsimd_f32_t scalars[4];
|
|
244
|
+
} result;
|
|
245
|
+
simsimd_size_t i = 0;
|
|
246
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
247
|
+
for (; i < 4; ++i) result.scalars[i] = 0;
|
|
248
|
+
return result.vec;
|
|
249
|
+
}
|
|
227
250
|
|
|
228
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const*
|
|
229
|
-
simsimd_distance_t*
|
|
251
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
252
|
+
simsimd_distance_t *result) {
|
|
230
253
|
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
231
254
|
simsimd_size_t i = 0;
|
|
232
255
|
for (; i + 4 <= n; i += 4) {
|
|
@@ -235,13 +258,12 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
235
258
|
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
236
259
|
}
|
|
237
260
|
simsimd_f32_t ab = vaddvq_f32(ab_vec);
|
|
238
|
-
for (; i < n; ++i)
|
|
239
|
-
ab += a[i] * b[i];
|
|
261
|
+
for (; i < n; ++i) ab += a[i] * b[i];
|
|
240
262
|
*result = ab;
|
|
241
263
|
}
|
|
242
264
|
|
|
243
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const*
|
|
244
|
-
simsimd_distance_t*
|
|
265
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
266
|
+
simsimd_distance_t *results) {
|
|
245
267
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
246
268
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
247
269
|
simsimd_size_t i = 0;
|
|
@@ -275,8 +297,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
275
297
|
results[1] = ab_imag;
|
|
276
298
|
}
|
|
277
299
|
|
|
278
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const*
|
|
279
|
-
simsimd_distance_t*
|
|
300
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
301
|
+
simsimd_distance_t *results) {
|
|
280
302
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
281
303
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
282
304
|
simsimd_size_t i = 0;
|
|
@@ -312,13 +334,15 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
312
334
|
|
|
313
335
|
#pragma clang attribute pop
|
|
314
336
|
#pragma GCC pop_options
|
|
337
|
+
#endif // SIMSIMD_TARGET_NEON
|
|
315
338
|
|
|
339
|
+
#if SIMSIMD_TARGET_NEON_I8
|
|
316
340
|
#pragma GCC push_options
|
|
317
341
|
#pragma GCC target("arch=armv8.2-a+dotprod")
|
|
318
342
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod"))), apply_to = function)
|
|
319
343
|
|
|
320
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const*
|
|
321
|
-
simsimd_distance_t*
|
|
344
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
345
|
+
simsimd_distance_t *result) {
|
|
322
346
|
|
|
323
347
|
int32x4_t ab_vec = vdupq_n_s32(0);
|
|
324
348
|
simsimd_size_t i = 0;
|
|
@@ -347,56 +371,87 @@ SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
347
371
|
*result = ab;
|
|
348
372
|
}
|
|
349
373
|
|
|
374
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
375
|
+
simsimd_distance_t *result) {
|
|
376
|
+
|
|
377
|
+
uint32x4_t ab_vec = vdupq_n_u32(0);
|
|
378
|
+
simsimd_size_t i = 0;
|
|
379
|
+
for (; i + 16 <= n; i += 16) {
|
|
380
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
381
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
382
|
+
ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// Take care of the tail:
|
|
386
|
+
uint32_t ab = vaddvq_u32(ab_vec);
|
|
387
|
+
for (; i < n; ++i) {
|
|
388
|
+
uint32_t ai = a[i], bi = b[i];
|
|
389
|
+
ab += ai * bi;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
*result = ab;
|
|
393
|
+
}
|
|
394
|
+
|
|
350
395
|
#pragma clang attribute pop
|
|
351
396
|
#pragma GCC pop_options
|
|
397
|
+
#endif // SIMSIMD_TARGET_NEON_I8
|
|
352
398
|
|
|
399
|
+
#if SIMSIMD_TARGET_NEON_F16
|
|
353
400
|
#pragma GCC push_options
|
|
354
|
-
#pragma GCC target("+simd+fp16")
|
|
355
|
-
#pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
|
|
401
|
+
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
402
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
356
403
|
|
|
357
|
-
|
|
358
|
-
|
|
404
|
+
SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t const *a, simsimd_size_t n) {
|
|
405
|
+
// In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
|
|
406
|
+
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
407
|
+
// loads and stores with vectorized math.
|
|
408
|
+
union {
|
|
409
|
+
float16x4_t vec;
|
|
410
|
+
simsimd_f16_t scalars[4];
|
|
411
|
+
} result;
|
|
412
|
+
simsimd_size_t i = 0;
|
|
413
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
414
|
+
for (; i < 4; ++i) result.scalars[i] = 0;
|
|
415
|
+
return result.vec;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
419
|
+
simsimd_distance_t *result) {
|
|
359
420
|
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
421
|
+
float32x4_t a_vec, b_vec;
|
|
360
422
|
simsimd_size_t i = 0;
|
|
361
|
-
for (; i + 4 <= n; i += 4) {
|
|
362
|
-
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
|
|
363
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
|
|
364
|
-
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
365
|
-
}
|
|
366
423
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
378
|
-
for (; j < 4; ++j)
|
|
379
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
380
|
-
ab_vec = vfmaq_f32(ab_vec, vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
|
|
424
|
+
simsimd_dot_f16_neon_cycle:
|
|
425
|
+
if (n < 4) {
|
|
426
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
427
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
428
|
+
n = 0;
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
432
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
433
|
+
a += 4, b += 4, n -= 4;
|
|
381
434
|
}
|
|
435
|
+
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
436
|
+
if (n) goto simsimd_dot_f16_neon_cycle;
|
|
382
437
|
*result = vaddvq_f32(ab_vec);
|
|
383
438
|
}
|
|
384
439
|
|
|
385
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const*
|
|
386
|
-
simsimd_distance_t*
|
|
440
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
441
|
+
simsimd_distance_t *results) {
|
|
387
442
|
|
|
388
443
|
// A nicer approach is to use `f16` arithmetic for the dot product, but that requires
|
|
389
444
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
390
445
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
391
446
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
392
447
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
393
|
-
|
|
394
|
-
|
|
448
|
+
|
|
449
|
+
while (n >= 8) {
|
|
395
450
|
// Unpack the input arrays into real and imaginary parts.
|
|
396
451
|
// MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
|
|
397
452
|
// integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
|
|
398
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
399
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
453
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
454
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
400
455
|
float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
|
|
401
456
|
float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
|
|
402
457
|
float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
|
|
@@ -407,37 +462,31 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
407
462
|
ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
408
463
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
409
464
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
410
|
-
}
|
|
411
465
|
|
|
412
|
-
|
|
413
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
414
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
415
|
-
|
|
416
|
-
// Handle the tail:
|
|
417
|
-
for (; i + 2 <= n; i += 2) {
|
|
418
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
419
|
-
ab_real += ar * br - ai * bi;
|
|
420
|
-
ab_imag += ar * bi + ai * br;
|
|
466
|
+
n -= 8, a += 8, b += 8;
|
|
421
467
|
}
|
|
422
|
-
|
|
423
|
-
|
|
468
|
+
|
|
469
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
470
|
+
simsimd_dot_f16c_serial(a, b, n, results);
|
|
471
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
472
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
424
473
|
}
|
|
425
474
|
|
|
426
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const*
|
|
427
|
-
simsimd_distance_t*
|
|
475
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
476
|
+
simsimd_distance_t *results) {
|
|
428
477
|
|
|
429
478
|
// A nicer approach is to use `f16` arithmetic for the dot product, but that requires
|
|
430
479
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
431
480
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
432
481
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
433
482
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
434
|
-
|
|
435
|
-
|
|
483
|
+
|
|
484
|
+
while (n >= 8) {
|
|
436
485
|
// Unpack the input arrays into real and imaginary parts.
|
|
437
486
|
// MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
|
|
438
487
|
// integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
|
|
439
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
440
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
488
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
489
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
441
490
|
float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
|
|
442
491
|
float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
|
|
443
492
|
float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
|
|
@@ -448,76 +497,73 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
448
497
|
ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
449
498
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
450
499
|
ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
// Reduce horizontal sums:
|
|
454
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
455
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
456
500
|
|
|
457
|
-
|
|
458
|
-
for (; i + 2 <= n; i += 2) {
|
|
459
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
460
|
-
ab_real += ar * br + ai * bi;
|
|
461
|
-
ab_imag += ar * bi - ai * br;
|
|
501
|
+
n -= 8, a += 8, b += 8;
|
|
462
502
|
}
|
|
463
|
-
|
|
464
|
-
|
|
503
|
+
|
|
504
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
505
|
+
simsimd_vdot_f16c_serial(a, b, n, results);
|
|
506
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
507
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
465
508
|
}
|
|
466
509
|
|
|
467
510
|
#pragma clang attribute pop
|
|
468
511
|
#pragma GCC pop_options
|
|
469
|
-
#endif //
|
|
512
|
+
#endif // SIMSIMD_TARGET_NEON_F16
|
|
470
513
|
|
|
471
514
|
#if SIMSIMD_TARGET_NEON_BF16
|
|
472
515
|
#pragma GCC push_options
|
|
473
516
|
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
474
517
|
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
475
518
|
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
519
|
+
SIMSIMD_INTERNAL bfloat16x8_t _simsimd_partial_load_bf16x8_neon(simsimd_bf16_t const *a, simsimd_size_t n) {
|
|
520
|
+
union {
|
|
521
|
+
bfloat16x8_t vec;
|
|
522
|
+
simsimd_bf16_t scalars[8];
|
|
523
|
+
} result;
|
|
479
524
|
simsimd_size_t i = 0;
|
|
480
|
-
for (; i
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
|
|
485
|
-
}
|
|
525
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
526
|
+
for (; i < 8; ++i) result.scalars[i] = 0;
|
|
527
|
+
return result.vec;
|
|
528
|
+
}
|
|
486
529
|
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
498
|
-
for (; j < 8; ++j)
|
|
499
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
500
|
-
ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
501
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
530
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
531
|
+
simsimd_distance_t *result) {
|
|
532
|
+
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
533
|
+
bfloat16x8_t a_vec, b_vec;
|
|
534
|
+
|
|
535
|
+
simsimd_dot_bf16_neon_cycle:
|
|
536
|
+
if (n < 8) {
|
|
537
|
+
a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
538
|
+
b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
539
|
+
n = 0;
|
|
502
540
|
}
|
|
503
|
-
|
|
541
|
+
else {
|
|
542
|
+
a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
|
|
543
|
+
b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
|
|
544
|
+
a += 8, b += 8, n -= 8;
|
|
545
|
+
}
|
|
546
|
+
ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
|
|
547
|
+
if (n) goto simsimd_dot_bf16_neon_cycle;
|
|
548
|
+
|
|
549
|
+
*result = vaddvq_f32(ab_vec);
|
|
504
550
|
}
|
|
505
551
|
|
|
506
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const*
|
|
507
|
-
simsimd_distance_t*
|
|
552
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
553
|
+
simsimd_distance_t *results) {
|
|
508
554
|
|
|
509
555
|
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
510
556
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
511
557
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
512
558
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
513
559
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
514
|
-
|
|
515
|
-
|
|
560
|
+
|
|
561
|
+
while (n >= 8) {
|
|
516
562
|
// Unpack the input arrays into real and imaginary parts.
|
|
517
563
|
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
518
564
|
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
519
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
520
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
565
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
566
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
521
567
|
float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
|
|
522
568
|
float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
|
|
523
569
|
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
@@ -528,37 +574,31 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
528
574
|
ab_real_vec = vfmsq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
529
575
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
530
576
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
// Reduce horizontal sums:
|
|
534
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
535
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
536
577
|
|
|
537
|
-
|
|
538
|
-
for (; i + 2 <= n; i += 2) {
|
|
539
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
540
|
-
ab_real += ar * br - ai * bi;
|
|
541
|
-
ab_imag += ar * bi + ai * br;
|
|
578
|
+
n -= 8, a += 8, b += 8;
|
|
542
579
|
}
|
|
543
|
-
|
|
544
|
-
|
|
580
|
+
|
|
581
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
582
|
+
simsimd_dot_bf16c_serial(a, b, n, results);
|
|
583
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
584
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
545
585
|
}
|
|
546
586
|
|
|
547
|
-
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const*
|
|
548
|
-
simsimd_distance_t*
|
|
587
|
+
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
588
|
+
simsimd_distance_t *results) {
|
|
549
589
|
|
|
550
590
|
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
551
591
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
552
592
|
// at once. That's how the original implementation worked, but compiling it was a nightmare :)
|
|
553
593
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
554
594
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
555
|
-
|
|
556
|
-
|
|
595
|
+
|
|
596
|
+
while (n >= 8) {
|
|
557
597
|
// Unpack the input arrays into real and imaginary parts.
|
|
558
598
|
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
559
599
|
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
560
|
-
int16x4x2_t a_vec = vld2_s16((short*)a
|
|
561
|
-
int16x4x2_t b_vec = vld2_s16((short*)b
|
|
600
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
601
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
562
602
|
float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
|
|
563
603
|
float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
|
|
564
604
|
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
@@ -569,20 +609,14 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
569
609
|
ab_real_vec = vfmaq_f32(ab_real_vec, a_imag_vec, b_imag_vec);
|
|
570
610
|
ab_imag_vec = vfmaq_f32(ab_imag_vec, a_real_vec, b_imag_vec);
|
|
571
611
|
ab_imag_vec = vfmsq_f32(ab_imag_vec, a_imag_vec, b_real_vec);
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
// Reduce horizontal sums:
|
|
575
|
-
simsimd_f32_t ab_real = vaddvq_f32(ab_real_vec);
|
|
576
|
-
simsimd_f32_t ab_imag = vaddvq_f32(ab_imag_vec);
|
|
577
612
|
|
|
578
|
-
|
|
579
|
-
for (; i + 2 <= n; i += 2) {
|
|
580
|
-
simsimd_f32_t ar = a[i], ai = a[i + 1], br = b[i], bi = b[i + 1];
|
|
581
|
-
ab_real += ar * br + ai * bi;
|
|
582
|
-
ab_imag += ar * bi - ai * br;
|
|
613
|
+
n -= 8, a += 8, b += 8;
|
|
583
614
|
}
|
|
584
|
-
|
|
585
|
-
|
|
615
|
+
|
|
616
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
617
|
+
simsimd_vdot_bf16c_serial(a, b, n, results);
|
|
618
|
+
results[0] += vaddvq_f32(ab_real_vec);
|
|
619
|
+
results[1] += vaddvq_f32(ab_imag_vec);
|
|
586
620
|
}
|
|
587
621
|
|
|
588
622
|
#pragma clang attribute pop
|
|
@@ -592,11 +626,11 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
592
626
|
#if SIMSIMD_TARGET_SVE
|
|
593
627
|
|
|
594
628
|
#pragma GCC push_options
|
|
595
|
-
#pragma GCC target("+sve")
|
|
596
|
-
#pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
|
|
629
|
+
#pragma GCC target("arch=armv8.2-a+sve")
|
|
630
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
|
|
597
631
|
|
|
598
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const*
|
|
599
|
-
simsimd_distance_t*
|
|
632
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
633
|
+
simsimd_distance_t *result) {
|
|
600
634
|
simsimd_size_t i = 0;
|
|
601
635
|
svfloat32_t ab_vec = svdup_f32(0.f);
|
|
602
636
|
do {
|
|
@@ -609,8 +643,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
|
|
|
609
643
|
*result = svaddv_f32(svptrue_b32(), ab_vec);
|
|
610
644
|
}
|
|
611
645
|
|
|
612
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const*
|
|
613
|
-
simsimd_distance_t*
|
|
646
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
647
|
+
simsimd_distance_t *results) {
|
|
614
648
|
simsimd_size_t i = 0;
|
|
615
649
|
svfloat32_t ab_real_vec = svdup_f32(0.f);
|
|
616
650
|
svfloat32_t ab_imag_vec = svdup_f32(0.f);
|
|
@@ -632,8 +666,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
632
666
|
results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
|
|
633
667
|
}
|
|
634
668
|
|
|
635
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const*
|
|
636
|
-
simsimd_distance_t*
|
|
669
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
670
|
+
simsimd_distance_t *results) {
|
|
637
671
|
simsimd_size_t i = 0;
|
|
638
672
|
svfloat32_t ab_real_vec = svdup_f32(0.f);
|
|
639
673
|
svfloat32_t ab_imag_vec = svdup_f32(0.f);
|
|
@@ -655,8 +689,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
655
689
|
results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
|
|
656
690
|
}
|
|
657
691
|
|
|
658
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const*
|
|
659
|
-
simsimd_distance_t*
|
|
692
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
693
|
+
simsimd_distance_t *result) {
|
|
660
694
|
simsimd_size_t i = 0;
|
|
661
695
|
svfloat64_t ab_vec = svdup_f64(0.);
|
|
662
696
|
do {
|
|
@@ -669,8 +703,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
669
703
|
*result = svaddv_f64(svptrue_b32(), ab_vec);
|
|
670
704
|
}
|
|
671
705
|
|
|
672
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const*
|
|
673
|
-
simsimd_distance_t*
|
|
706
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
707
|
+
simsimd_distance_t *results) {
|
|
674
708
|
simsimd_size_t i = 0;
|
|
675
709
|
svfloat64_t ab_real_vec = svdup_f64(0.);
|
|
676
710
|
svfloat64_t ab_imag_vec = svdup_f64(0.);
|
|
@@ -692,8 +726,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t c
|
|
|
692
726
|
results[1] = svaddv_f64(svptrue_b64(), ab_imag_vec);
|
|
693
727
|
}
|
|
694
728
|
|
|
695
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const*
|
|
696
|
-
simsimd_distance_t*
|
|
729
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
730
|
+
simsimd_distance_t *results) {
|
|
697
731
|
simsimd_size_t i = 0;
|
|
698
732
|
svfloat64_t ab_real_vec = svdup_f64(0.);
|
|
699
733
|
svfloat64_t ab_imag_vec = svdup_f64(0.);
|
|
@@ -719,15 +753,15 @@ SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t
|
|
|
719
753
|
#pragma GCC pop_options
|
|
720
754
|
|
|
721
755
|
#pragma GCC push_options
|
|
722
|
-
#pragma GCC target("+sve+fp16")
|
|
723
|
-
#pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
|
|
756
|
+
#pragma GCC target("arch=armv8.2-a+sve+fp16")
|
|
757
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
|
|
724
758
|
|
|
725
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const*
|
|
726
|
-
simsimd_distance_t*
|
|
759
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
|
|
760
|
+
simsimd_distance_t *result) {
|
|
727
761
|
simsimd_size_t i = 0;
|
|
728
762
|
svfloat16_t ab_vec = svdup_f16(0);
|
|
729
|
-
simsimd_f16_for_arm_simd_t const*
|
|
730
|
-
simsimd_f16_for_arm_simd_t const*
|
|
763
|
+
simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
|
|
764
|
+
simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
|
|
731
765
|
do {
|
|
732
766
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
733
767
|
svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
|
|
@@ -739,15 +773,15 @@ SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
|
|
|
739
773
|
*result = ab;
|
|
740
774
|
}
|
|
741
775
|
|
|
742
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const*
|
|
743
|
-
simsimd_distance_t*
|
|
776
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
777
|
+
simsimd_distance_t *results) {
|
|
744
778
|
simsimd_size_t i = 0;
|
|
745
779
|
svfloat16_t ab_real_vec = svdup_f16(0);
|
|
746
780
|
svfloat16_t ab_imag_vec = svdup_f16(0);
|
|
747
781
|
do {
|
|
748
782
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i / 2, (unsigned int)n / 2);
|
|
749
|
-
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)a + i);
|
|
750
|
-
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)b + i);
|
|
783
|
+
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)a + i);
|
|
784
|
+
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)b + i);
|
|
751
785
|
svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
|
|
752
786
|
svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
|
|
753
787
|
svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
|
|
@@ -762,15 +796,15 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t c
|
|
|
762
796
|
results[1] = svaddv_f16(svptrue_b16(), ab_imag_vec);
|
|
763
797
|
}
|
|
764
798
|
|
|
765
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const*
|
|
766
|
-
simsimd_distance_t*
|
|
799
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
800
|
+
simsimd_distance_t *results) {
|
|
767
801
|
simsimd_size_t i = 0;
|
|
768
802
|
svfloat16_t ab_real_vec = svdup_f16(0);
|
|
769
803
|
svfloat16_t ab_imag_vec = svdup_f16(0);
|
|
770
804
|
do {
|
|
771
805
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i / 2, (unsigned int)n / 2);
|
|
772
|
-
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)a + i);
|
|
773
|
-
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)b + i);
|
|
806
|
+
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)a + i);
|
|
807
|
+
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)b + i);
|
|
774
808
|
svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
|
|
775
809
|
svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
|
|
776
810
|
svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
|
|
@@ -788,52 +822,29 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
788
822
|
#pragma clang attribute pop
|
|
789
823
|
#pragma GCC pop_options
|
|
790
824
|
#endif // SIMSIMD_TARGET_SVE
|
|
791
|
-
#endif //
|
|
825
|
+
#endif // _SIMSIMD_TARGET_ARM
|
|
792
826
|
|
|
793
|
-
#if
|
|
827
|
+
#if _SIMSIMD_TARGET_X86
|
|
794
828
|
#if SIMSIMD_TARGET_HASWELL
|
|
795
829
|
#pragma GCC push_options
|
|
796
830
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
797
831
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
798
832
|
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
806
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
// In case the software emulation for `f16` scalars is enabled, the `simsimd_uncompress_f16`
|
|
810
|
-
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
811
|
-
// loads and stores with vectorized math.
|
|
812
|
-
if (i < n) {
|
|
813
|
-
union {
|
|
814
|
-
__m128i f16_vec;
|
|
815
|
-
simsimd_f16_t f16[8];
|
|
816
|
-
} a_padded_tail, b_padded_tail;
|
|
817
|
-
simsimd_size_t j = 0;
|
|
818
|
-
for (; i < n; ++i, ++j)
|
|
819
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
820
|
-
for (; j < 8; ++j)
|
|
821
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
822
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
823
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
824
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
825
|
-
}
|
|
833
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f64x4_haswell(__m256d vec) {
|
|
834
|
+
// Reduce the double-precision vector to a scalar
|
|
835
|
+
// Horizontal add the first and second double-precision values, and third and fourth
|
|
836
|
+
__m128d vec_low = _mm256_castpd256_pd128(vec);
|
|
837
|
+
__m128d vec_high = _mm256_extractf128_pd(vec, 1);
|
|
838
|
+
__m128d vec128 = _mm_add_pd(vec_low, vec_high);
|
|
826
839
|
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
840
|
+
// Horizontal add again to accumulate all four values into one
|
|
841
|
+
vec128 = _mm_hadd_pd(vec128, vec128);
|
|
830
842
|
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
*result = f32_result;
|
|
843
|
+
// Convert the final sum to a scalar double-precision value and return
|
|
844
|
+
return _mm_cvtsd_f64(vec128);
|
|
834
845
|
}
|
|
835
846
|
|
|
836
|
-
|
|
847
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x8_haswell(__m256 vec) {
|
|
837
848
|
// Convert the lower and higher 128-bit lanes of the input vector to double precision
|
|
838
849
|
__m128 low_f32 = _mm256_castps256_ps128(vec);
|
|
839
850
|
__m128 high_f32 = _mm256_extractf128_ps(vec, 1);
|
|
@@ -844,22 +855,20 @@ inline simsimd_f64_t _mm256_reduce_add_ps_dbl(__m256 vec) {
|
|
|
844
855
|
|
|
845
856
|
// Perform the addition in double-precision
|
|
846
857
|
__m256d sum = _mm256_add_pd(low_f64, high_f64);
|
|
858
|
+
return _simsimd_reduce_f64x4_haswell(sum);
|
|
859
|
+
}
|
|
847
860
|
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
sum128 = _mm_hadd_pd(sum128, sum128);
|
|
856
|
-
|
|
857
|
-
// Convert the final sum to a scalar double-precision value and return
|
|
858
|
-
return _mm_cvtsd_f64(sum128);
|
|
861
|
+
SIMSIMD_INTERNAL simsimd_i32_t _simsimd_reduce_i32x8_haswell(__m256i vec) {
|
|
862
|
+
__m128i low = _mm256_castsi256_si128(vec);
|
|
863
|
+
__m128i high = _mm256_extracti128_si256(vec, 1);
|
|
864
|
+
__m128i sum = _mm_add_epi32(low, high);
|
|
865
|
+
sum = _mm_hadd_epi32(sum, sum);
|
|
866
|
+
sum = _mm_hadd_epi32(sum, sum);
|
|
867
|
+
return _mm_cvtsi128_si32(sum);
|
|
859
868
|
}
|
|
860
869
|
|
|
861
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const*
|
|
862
|
-
simsimd_distance_t*
|
|
870
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
871
|
+
simsimd_distance_t *results) {
|
|
863
872
|
|
|
864
873
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
865
874
|
simsimd_size_t i = 0;
|
|
@@ -868,14 +877,13 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
868
877
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
869
878
|
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
870
879
|
}
|
|
871
|
-
simsimd_f64_t ab =
|
|
872
|
-
for (; i < n; ++i)
|
|
873
|
-
ab += a[i] * b[i];
|
|
880
|
+
simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
881
|
+
for (; i < n; ++i) ab += a[i] * b[i];
|
|
874
882
|
*results = ab;
|
|
875
883
|
}
|
|
876
884
|
|
|
877
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const*
|
|
878
|
-
simsimd_distance_t*
|
|
885
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
886
|
+
simsimd_distance_t *results) {
|
|
879
887
|
|
|
880
888
|
// The naive approach would be to use FMA and FMS instructions on different parts of the vectors.
|
|
881
889
|
// Prior to that we would need to shuffle the input vectors to separate real and imaginary parts.
|
|
@@ -903,7 +911,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
903
911
|
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
904
912
|
// This way we can avoid the shuffling and the need for separate real and imaginary parts.
|
|
905
913
|
// For the imaginary part of the product, we would need to swap the real and imaginary parts of
|
|
906
|
-
// one of the vectors.
|
|
914
|
+
// one of the vectors. Moreover, `XOR` can be placed after the primary loop.
|
|
907
915
|
// Both operations are quite cheap, and the throughput doubles from 2.5 GB/s to 5 GB/s.
|
|
908
916
|
__m256 ab_real_vec = _mm256_setzero_ps();
|
|
909
917
|
__m256 ab_imag_vec = _mm256_setzero_ps();
|
|
@@ -922,15 +930,17 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
922
930
|
for (; i + 8 <= n; i += 8) {
|
|
923
931
|
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
924
932
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
925
|
-
__m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
|
|
926
933
|
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
927
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec,
|
|
934
|
+
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
928
935
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
929
936
|
}
|
|
930
937
|
|
|
938
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
939
|
+
ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
|
|
940
|
+
|
|
931
941
|
// Reduce horizontal sums:
|
|
932
|
-
simsimd_distance_t ab_real =
|
|
933
|
-
simsimd_distance_t ab_imag =
|
|
942
|
+
simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
943
|
+
simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
934
944
|
|
|
935
945
|
// Handle the tail:
|
|
936
946
|
for (; i + 2 <= n; i += 2) {
|
|
@@ -942,8 +952,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
942
952
|
results[1] = ab_imag;
|
|
943
953
|
}
|
|
944
954
|
|
|
945
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const*
|
|
946
|
-
simsimd_distance_t*
|
|
955
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
956
|
+
simsimd_distance_t *results) {
|
|
947
957
|
|
|
948
958
|
__m256 ab_real_vec = _mm256_setzero_ps();
|
|
949
959
|
__m256 ab_imag_vec = _mm256_setzero_ps();
|
|
@@ -963,14 +973,16 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
|
|
|
963
973
|
__m256 a_vec = _mm256_loadu_ps(a + i);
|
|
964
974
|
__m256 b_vec = _mm256_loadu_ps(b + i);
|
|
965
975
|
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
966
|
-
a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
|
|
967
976
|
b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
968
977
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
969
978
|
}
|
|
970
979
|
|
|
980
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
981
|
+
ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
|
|
982
|
+
|
|
971
983
|
// Reduce horizontal sums:
|
|
972
|
-
simsimd_distance_t ab_real =
|
|
973
|
-
simsimd_distance_t ab_imag =
|
|
984
|
+
simsimd_distance_t ab_real = _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
985
|
+
simsimd_distance_t ab_imag = _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
974
986
|
|
|
975
987
|
// Handle the tail:
|
|
976
988
|
for (; i + 2 <= n; i += 2) {
|
|
@@ -982,8 +994,50 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
|
|
|
982
994
|
results[1] = ab_imag;
|
|
983
995
|
}
|
|
984
996
|
|
|
985
|
-
|
|
986
|
-
|
|
997
|
+
SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const *a, simsimd_size_t n) {
|
|
998
|
+
// In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
|
|
999
|
+
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
1000
|
+
// loads and stores with vectorized math.
|
|
1001
|
+
union {
|
|
1002
|
+
__m128i vec;
|
|
1003
|
+
simsimd_f16_t scalars[8];
|
|
1004
|
+
} result;
|
|
1005
|
+
simsimd_size_t i = 0;
|
|
1006
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
1007
|
+
for (; i < 8; ++i) result.scalars[i] = 0;
|
|
1008
|
+
return _mm256_cvtph_ps(result.vec);
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1012
|
+
simsimd_distance_t *result) {
|
|
1013
|
+
__m256 a_vec, b_vec;
|
|
1014
|
+
__m256 ab_vec = _mm256_setzero_ps();
|
|
1015
|
+
|
|
1016
|
+
simsimd_dot_f16_haswell_cycle:
|
|
1017
|
+
if (n < 8) {
|
|
1018
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1019
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1020
|
+
n = 0;
|
|
1021
|
+
}
|
|
1022
|
+
else {
|
|
1023
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1024
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1025
|
+
n -= 8, a += 8, b += 8;
|
|
1026
|
+
}
|
|
1027
|
+
// We can silence the NaNs using blends:
|
|
1028
|
+
//
|
|
1029
|
+
// __m256 a_is_nan = _mm256_cmp_ps(a_vec, a_vec, _CMP_UNORD_Q);
|
|
1030
|
+
// __m256 b_is_nan = _mm256_cmp_ps(b_vec, b_vec, _CMP_UNORD_Q);
|
|
1031
|
+
// ab_vec = _mm256_blendv_ps(_mm256_fmadd_ps(a_vec, b_vec, ab_vec), ab_vec, _mm256_or_ps(a_is_nan, b_is_nan));
|
|
1032
|
+
//
|
|
1033
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1034
|
+
if (n) goto simsimd_dot_f16_haswell_cycle;
|
|
1035
|
+
|
|
1036
|
+
*result = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1040
|
+
simsimd_distance_t *results) {
|
|
987
1041
|
// Ideally the implementation would load 256 bits worth of vector data at a time,
|
|
988
1042
|
// shuffle those within a register, split in halfs, and only then upcast.
|
|
989
1043
|
// That way, we are stepping through 32x 16-bit vector components at a time, or 16 dimensions.
|
|
@@ -1004,44 +1058,28 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16
|
|
|
1004
1058
|
3, 2, 1, 0, // Points to the first f32 in 128-bit lane
|
|
1005
1059
|
7, 6, 5, 4 // Points to the second f32 in 128-bit lane
|
|
1006
1060
|
);
|
|
1007
|
-
simsimd_size_t i = 0;
|
|
1008
|
-
for (; i + 8 <= n; i += 8) {
|
|
1009
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
1010
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
1011
|
-
__m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
|
|
1012
|
-
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1013
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec, b_flipped_vec, ab_real_vec);
|
|
1014
|
-
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
1015
|
-
}
|
|
1016
1061
|
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
if (i < n) {
|
|
1021
|
-
union {
|
|
1022
|
-
__m128i f16_vec;
|
|
1023
|
-
simsimd_f16_t f16[8];
|
|
1024
|
-
} a_padded_tail, b_padded_tail;
|
|
1025
|
-
simsimd_size_t j = 0;
|
|
1026
|
-
for (; i < n; ++i, ++j)
|
|
1027
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
1028
|
-
for (; j < 8; ++j)
|
|
1029
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
1030
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
1031
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
1032
|
-
__m256 b_flipped_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(b_vec), sign_flip_vec));
|
|
1062
|
+
while (n >= 8) {
|
|
1063
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1064
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1033
1065
|
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1034
|
-
ab_real_vec = _mm256_fmadd_ps(a_vec,
|
|
1066
|
+
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1035
1067
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
1068
|
+
|
|
1069
|
+
n -= 8, a += 8, b += 8;
|
|
1036
1070
|
}
|
|
1037
1071
|
|
|
1038
|
-
//
|
|
1039
|
-
|
|
1040
|
-
|
|
1072
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1073
|
+
ab_real_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_real_vec), sign_flip_vec));
|
|
1074
|
+
|
|
1075
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
1076
|
+
simsimd_dot_f16c_serial(a, b, n, results);
|
|
1077
|
+
results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
1078
|
+
results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
1041
1079
|
}
|
|
1042
1080
|
|
|
1043
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const*
|
|
1044
|
-
simsimd_distance_t*
|
|
1081
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1082
|
+
simsimd_distance_t *results) {
|
|
1045
1083
|
|
|
1046
1084
|
__m256 ab_real_vec = _mm256_setzero_ps();
|
|
1047
1085
|
__m256 ab_imag_vec = _mm256_setzero_ps();
|
|
@@ -1056,116 +1094,157 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
|
|
|
1056
1094
|
3, 2, 1, 0, // Points to the first f32 in 128-bit lane
|
|
1057
1095
|
7, 6, 5, 4 // Points to the second f32 in 128-bit lane
|
|
1058
1096
|
);
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
__m256 a_vec = _mm256_cvtph_ps(
|
|
1062
|
-
__m256 b_vec = _mm256_cvtph_ps(
|
|
1097
|
+
|
|
1098
|
+
while (n >= 8) {
|
|
1099
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1100
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1063
1101
|
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1064
|
-
a_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(a_vec), sign_flip_vec));
|
|
1065
1102
|
b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1066
1103
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
1104
|
+
|
|
1105
|
+
n -= 8, a += 8, b += 8;
|
|
1067
1106
|
}
|
|
1068
1107
|
|
|
1069
|
-
//
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1108
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1109
|
+
ab_imag_vec = _mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(ab_imag_vec), sign_flip_vec));
|
|
1110
|
+
|
|
1111
|
+
// Reduce horizontal sums and aggregate with the tail:
|
|
1112
|
+
simsimd_dot_f16c_serial(a, b, n, results);
|
|
1113
|
+
results[0] += _simsimd_reduce_f32x8_haswell(ab_real_vec);
|
|
1114
|
+
results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1118
|
+
simsimd_distance_t *result) {
|
|
1119
|
+
|
|
1120
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1121
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1122
|
+
|
|
1123
|
+
// AVX2 has no instructions for 8-bit signed integer dot-products,
|
|
1124
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1125
|
+
// So we need to normalize the first vector to its absolute value,
|
|
1126
|
+
// and shift the product sign into the second vector.
|
|
1127
|
+
//
|
|
1128
|
+
// __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
|
|
1129
|
+
// __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
|
|
1130
|
+
// __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
|
|
1131
|
+
//
|
|
1132
|
+
// The problem with this approach, however, is the `-128` value in the second vector.
|
|
1133
|
+
// Flipping it's sign will do nothing, and the result will be incorrect.
|
|
1134
|
+
// This can easily lead to noticeable numerical errors in the final result.
|
|
1135
|
+
simsimd_size_t i = 0;
|
|
1136
|
+
for (; i + 32 <= n; i += 32) {
|
|
1137
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1138
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1139
|
+
|
|
1140
|
+
// Upcast `int8` to `int16`
|
|
1141
|
+
__m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
|
|
1142
|
+
__m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
|
|
1143
|
+
__m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
|
|
1144
|
+
__m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
|
|
1145
|
+
|
|
1146
|
+
// Multiply and accumulate at int16 level, accumulate at `int32` level
|
|
1147
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1148
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1088
1149
|
}
|
|
1089
1150
|
|
|
1090
|
-
//
|
|
1091
|
-
|
|
1092
|
-
|
|
1151
|
+
// Horizontal sum across the 256-bit register
|
|
1152
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1153
|
+
|
|
1154
|
+
// Take care of the tail:
|
|
1155
|
+
for (; i < n; ++i) ab += (int)(a[i]) * b[i];
|
|
1156
|
+
*result = ab;
|
|
1093
1157
|
}
|
|
1094
1158
|
|
|
1095
|
-
SIMSIMD_PUBLIC void
|
|
1096
|
-
simsimd_distance_t*
|
|
1159
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1160
|
+
simsimd_distance_t *result) {
|
|
1097
1161
|
|
|
1098
|
-
__m256i
|
|
1099
|
-
__m256i
|
|
1162
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1163
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1164
|
+
__m256i const zeros_vec = _mm256_setzero_si256();
|
|
1100
1165
|
|
|
1166
|
+
// AVX2 has no instructions for unsigned 8-bit integer dot-products,
|
|
1167
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1101
1168
|
simsimd_size_t i = 0;
|
|
1102
1169
|
for (; i + 32 <= n; i += 32) {
|
|
1103
|
-
__m256i
|
|
1104
|
-
__m256i
|
|
1170
|
+
__m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1171
|
+
__m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1105
1172
|
|
|
1106
|
-
//
|
|
1107
|
-
|
|
1108
|
-
__m256i
|
|
1109
|
-
__m256i
|
|
1110
|
-
__m256i
|
|
1173
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1174
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1175
|
+
__m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
1176
|
+
__m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
1177
|
+
__m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
1178
|
+
__m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
1111
1179
|
|
|
1112
1180
|
// Multiply and accumulate at int16 level, accumulate at int32 level
|
|
1113
|
-
|
|
1114
|
-
|
|
1181
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1182
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1115
1183
|
}
|
|
1116
1184
|
|
|
1117
1185
|
// Horizontal sum across the 256-bit register
|
|
1118
|
-
|
|
1119
|
-
__m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
|
|
1120
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1121
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1186
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1122
1187
|
|
|
1123
1188
|
// Take care of the tail:
|
|
1124
|
-
|
|
1125
|
-
for (; i < n; ++i)
|
|
1126
|
-
ab += a[i] * b[i];
|
|
1189
|
+
for (; i < n; ++i) ab += (int)(a[i]) * b[i];
|
|
1127
1190
|
*result = ab;
|
|
1128
1191
|
}
|
|
1129
1192
|
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
for (; i + 8 <= n; i += 8) {
|
|
1135
|
-
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
1136
|
-
// x = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(x), 16))
|
|
1137
|
-
__m256 a_vec =
|
|
1138
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
|
|
1139
|
-
__m256 b_vec =
|
|
1140
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
|
|
1141
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1142
|
-
}
|
|
1193
|
+
SIMSIMD_INTERNAL __m256 _simsimd_bf16x8_to_f32x8_haswell(__m128i a) {
|
|
1194
|
+
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
1195
|
+
return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a), 16));
|
|
1196
|
+
}
|
|
1143
1197
|
|
|
1144
|
-
|
|
1198
|
+
SIMSIMD_INTERNAL __m128i _simsimd_f32x8_to_bf16x8_haswell(__m256 a) {
|
|
1199
|
+
// Pack the 32-bit integers into 16-bit integers.
|
|
1200
|
+
// This is less trivial than unpacking: https://stackoverflow.com/a/77781241/2766161
|
|
1201
|
+
// The best approach is to shuffle within lanes first: https://stackoverflow.com/a/49723746/2766161
|
|
1202
|
+
// Our shuffling mask will drop the low 2-bytes from every 4-byte word.
|
|
1203
|
+
__m256i trunc_elements = _mm256_shuffle_epi8( //
|
|
1204
|
+
_mm256_castps_si256(a), //
|
|
1205
|
+
_mm256_set_epi8( //
|
|
1206
|
+
-1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 11, 10, 7, 6, 3, 2, //
|
|
1207
|
+
-1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 11, 10, 7, 6, 3, 2 //
|
|
1208
|
+
));
|
|
1209
|
+
__m256i ordered = _mm256_permute4x64_epi64(trunc_elements, 0x58);
|
|
1210
|
+
__m128i result = _mm256_castsi256_si128(ordered);
|
|
1211
|
+
return result;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
SIMSIMD_INTERNAL __m128i _simsimd_partial_load_bf16x8_haswell(simsimd_bf16_t const *a, simsimd_size_t n) {
|
|
1215
|
+
// In case the software emulation for `bf16` scalars is enabled, the `simsimd_bf16_to_f32`
|
|
1145
1216
|
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
1146
1217
|
// loads and stores with vectorized math.
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
1157
|
-
__m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
|
|
1158
|
-
__m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
|
|
1159
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1160
|
-
}
|
|
1218
|
+
union {
|
|
1219
|
+
__m128i vec;
|
|
1220
|
+
simsimd_bf16_t scalars[8];
|
|
1221
|
+
} result;
|
|
1222
|
+
simsimd_size_t i = 0;
|
|
1223
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
1224
|
+
for (; i < 8; ++i) result.scalars[i] = 0;
|
|
1225
|
+
return result.vec;
|
|
1226
|
+
}
|
|
1161
1227
|
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1228
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1229
|
+
simsimd_distance_t *result) {
|
|
1230
|
+
__m128i a_vec, b_vec;
|
|
1231
|
+
__m256 ab_vec = _mm256_setzero_ps();
|
|
1165
1232
|
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1233
|
+
simsimd_dot_bf16_haswell_cycle:
|
|
1234
|
+
if (n < 8) {
|
|
1235
|
+
a_vec = _simsimd_partial_load_bf16x8_haswell(a, n);
|
|
1236
|
+
b_vec = _simsimd_partial_load_bf16x8_haswell(b, n);
|
|
1237
|
+
n = 0;
|
|
1238
|
+
}
|
|
1239
|
+
else {
|
|
1240
|
+
a_vec = _mm_lddqu_si128((__m128i const *)a);
|
|
1241
|
+
b_vec = _mm_lddqu_si128((__m128i const *)b);
|
|
1242
|
+
a += 8, b += 8, n -= 8;
|
|
1243
|
+
}
|
|
1244
|
+
ab_vec = _mm256_fmadd_ps(_simsimd_bf16x8_to_f32x8_haswell(a_vec), _simsimd_bf16x8_to_f32x8_haswell(b_vec), ab_vec);
|
|
1245
|
+
if (n) goto simsimd_dot_bf16_haswell_cycle;
|
|
1246
|
+
|
|
1247
|
+
*result = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1169
1248
|
}
|
|
1170
1249
|
|
|
1171
1250
|
#pragma clang attribute pop
|
|
@@ -1174,11 +1253,29 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf
|
|
|
1174
1253
|
|
|
1175
1254
|
#if SIMSIMD_TARGET_SKYLAKE
|
|
1176
1255
|
#pragma GCC push_options
|
|
1177
|
-
#pragma GCC target("avx512f", "avx512vl", "avx512bw", "bmi2")
|
|
1178
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
|
|
1256
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
|
|
1257
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
|
|
1258
|
+
|
|
1259
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x16_skylake(__m512 a) {
|
|
1260
|
+
__m512 x = _mm512_add_ps(a, _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 0, 3, 2)));
|
|
1261
|
+
__m128 r = _mm512_castps512_ps128(_mm512_add_ps(x, _mm512_shuffle_f32x4(x, x, _MM_SHUFFLE(0, 0, 0, 1))));
|
|
1262
|
+
r = _mm_hadd_ps(r, r);
|
|
1263
|
+
return _mm_cvtss_f32(_mm_hadd_ps(r, r));
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
SIMSIMD_INTERNAL __m512 _simsimd_bf16x16_to_f32x16_skylake(__m256i a) {
|
|
1267
|
+
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
1268
|
+
return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
SIMSIMD_INTERNAL __m256i _simsimd_f32x16_to_bf16x16_skylake(__m512 a) {
|
|
1272
|
+
// Add 2^15 and right shift 16 to do round-nearest
|
|
1273
|
+
__m512i x = _mm512_srli_epi32(_mm512_add_epi32(_mm512_castps_si512(a), _mm512_set1_epi32(1 << 15)), 16);
|
|
1274
|
+
return _mm512_cvtepi32_epi16(x);
|
|
1275
|
+
}
|
|
1179
1276
|
|
|
1180
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const*
|
|
1181
|
-
simsimd_distance_t*
|
|
1277
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1278
|
+
simsimd_distance_t *result) {
|
|
1182
1279
|
__m512 ab_vec = _mm512_setzero();
|
|
1183
1280
|
__m512 a_vec, b_vec;
|
|
1184
1281
|
|
|
@@ -1188,20 +1285,20 @@ simsimd_dot_f32_skylake_cycle:
|
|
|
1188
1285
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1189
1286
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1190
1287
|
n = 0;
|
|
1191
|
-
}
|
|
1288
|
+
}
|
|
1289
|
+
else {
|
|
1192
1290
|
a_vec = _mm512_loadu_ps(a);
|
|
1193
1291
|
b_vec = _mm512_loadu_ps(b);
|
|
1194
1292
|
a += 16, b += 16, n -= 16;
|
|
1195
1293
|
}
|
|
1196
1294
|
ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1197
|
-
if (n)
|
|
1198
|
-
goto simsimd_dot_f32_skylake_cycle;
|
|
1295
|
+
if (n) goto simsimd_dot_f32_skylake_cycle;
|
|
1199
1296
|
|
|
1200
|
-
*result =
|
|
1297
|
+
*result = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1201
1298
|
}
|
|
1202
1299
|
|
|
1203
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const*
|
|
1204
|
-
simsimd_distance_t*
|
|
1300
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1301
|
+
simsimd_distance_t *result) {
|
|
1205
1302
|
__m512d ab_vec = _mm512_setzero_pd();
|
|
1206
1303
|
__m512d a_vec, b_vec;
|
|
1207
1304
|
|
|
@@ -1211,25 +1308,24 @@ simsimd_dot_f64_skylake_cycle:
|
|
|
1211
1308
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1212
1309
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1213
1310
|
n = 0;
|
|
1214
|
-
}
|
|
1311
|
+
}
|
|
1312
|
+
else {
|
|
1215
1313
|
a_vec = _mm512_loadu_pd(a);
|
|
1216
1314
|
b_vec = _mm512_loadu_pd(b);
|
|
1217
1315
|
a += 8, b += 8, n -= 8;
|
|
1218
1316
|
}
|
|
1219
1317
|
ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
|
|
1220
|
-
if (n)
|
|
1221
|
-
goto simsimd_dot_f64_skylake_cycle;
|
|
1318
|
+
if (n) goto simsimd_dot_f64_skylake_cycle;
|
|
1222
1319
|
|
|
1223
1320
|
*result = _mm512_reduce_add_pd(ab_vec);
|
|
1224
1321
|
}
|
|
1225
1322
|
|
|
1226
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const*
|
|
1227
|
-
simsimd_distance_t*
|
|
1228
|
-
|
|
1323
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1324
|
+
simsimd_distance_t *results) {
|
|
1229
1325
|
__m512 ab_real_vec = _mm512_setzero();
|
|
1230
1326
|
__m512 ab_imag_vec = _mm512_setzero();
|
|
1231
1327
|
__m512 a_vec;
|
|
1232
|
-
|
|
1328
|
+
__m512 b_vec;
|
|
1233
1329
|
|
|
1234
1330
|
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1235
1331
|
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
@@ -1247,27 +1343,29 @@ simsimd_dot_f32c_skylake_cycle:
|
|
|
1247
1343
|
if (n < 16) {
|
|
1248
1344
|
__mmask16 mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
|
|
1249
1345
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1250
|
-
b_vec =
|
|
1346
|
+
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1251
1347
|
n = 0;
|
|
1252
|
-
}
|
|
1348
|
+
}
|
|
1349
|
+
else {
|
|
1253
1350
|
a_vec = _mm512_loadu_ps(a);
|
|
1254
|
-
b_vec =
|
|
1351
|
+
b_vec = _mm512_loadu_ps(b);
|
|
1255
1352
|
a += 16, b += 16, n -= 16;
|
|
1256
1353
|
}
|
|
1257
|
-
ab_real_vec = _mm512_fmadd_ps(
|
|
1258
|
-
ab_imag_vec =
|
|
1259
|
-
|
|
1260
|
-
if (n)
|
|
1261
|
-
|
|
1354
|
+
ab_real_vec = _mm512_fmadd_ps(b_vec, a_vec, ab_real_vec);
|
|
1355
|
+
ab_imag_vec = _mm512_fmadd_ps(
|
|
1356
|
+
_mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
|
|
1357
|
+
if (n) goto simsimd_dot_f32c_skylake_cycle;
|
|
1358
|
+
|
|
1359
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1360
|
+
ab_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_real_vec), sign_flip_vec));
|
|
1262
1361
|
|
|
1263
1362
|
// Reduce horizontal sums:
|
|
1264
|
-
results[0] =
|
|
1265
|
-
results[1] =
|
|
1363
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1364
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1266
1365
|
}
|
|
1267
1366
|
|
|
1268
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const*
|
|
1269
|
-
simsimd_distance_t*
|
|
1270
|
-
|
|
1367
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1368
|
+
simsimd_distance_t *results) {
|
|
1271
1369
|
__m512 ab_real_vec = _mm512_setzero();
|
|
1272
1370
|
__m512 ab_imag_vec = _mm512_setzero();
|
|
1273
1371
|
__m512 a_vec;
|
|
@@ -1291,30 +1389,31 @@ simsimd_vdot_f32c_skylake_cycle:
|
|
|
1291
1389
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1292
1390
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1293
1391
|
n = 0;
|
|
1294
|
-
}
|
|
1392
|
+
}
|
|
1393
|
+
else {
|
|
1295
1394
|
a_vec = _mm512_loadu_ps(a);
|
|
1296
1395
|
b_vec = _mm512_loadu_ps(b);
|
|
1297
1396
|
a += 16, b += 16, n -= 16;
|
|
1298
1397
|
}
|
|
1299
1398
|
ab_real_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1300
|
-
a_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(a_vec), sign_flip_vec));
|
|
1301
1399
|
b_vec = _mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec));
|
|
1302
1400
|
ab_imag_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
1303
|
-
if (n)
|
|
1304
|
-
|
|
1401
|
+
if (n) goto simsimd_vdot_f32c_skylake_cycle;
|
|
1402
|
+
|
|
1403
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1404
|
+
ab_imag_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_imag_vec), sign_flip_vec));
|
|
1305
1405
|
|
|
1306
1406
|
// Reduce horizontal sums:
|
|
1307
|
-
results[0] =
|
|
1308
|
-
results[1] =
|
|
1407
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1408
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1309
1409
|
}
|
|
1310
1410
|
|
|
1311
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const*
|
|
1312
|
-
simsimd_distance_t*
|
|
1313
|
-
|
|
1411
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1412
|
+
simsimd_distance_t *results) {
|
|
1314
1413
|
__m512d ab_real_vec = _mm512_setzero_pd();
|
|
1315
1414
|
__m512d ab_imag_vec = _mm512_setzero_pd();
|
|
1316
1415
|
__m512d a_vec;
|
|
1317
|
-
|
|
1416
|
+
__m512d b_vec;
|
|
1318
1417
|
|
|
1319
1418
|
// We take into account, that FMS is the same as FMA with a negative multiplier.
|
|
1320
1419
|
// To multiply a floating-point value by -1, we can use the `XOR` instruction to flip the sign bit.
|
|
@@ -1335,27 +1434,29 @@ simsimd_dot_f64c_skylake_cycle:
|
|
|
1335
1434
|
if (n < 8) {
|
|
1336
1435
|
__mmask8 mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
|
|
1337
1436
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1338
|
-
b_vec =
|
|
1437
|
+
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1339
1438
|
n = 0;
|
|
1340
|
-
}
|
|
1439
|
+
}
|
|
1440
|
+
else {
|
|
1341
1441
|
a_vec = _mm512_loadu_pd(a);
|
|
1342
|
-
b_vec =
|
|
1442
|
+
b_vec = _mm512_loadu_pd(b);
|
|
1343
1443
|
a += 8, b += 8, n -= 8;
|
|
1344
1444
|
}
|
|
1345
|
-
ab_real_vec = _mm512_fmadd_pd(
|
|
1346
|
-
ab_imag_vec =
|
|
1347
|
-
|
|
1348
|
-
if (n)
|
|
1349
|
-
|
|
1445
|
+
ab_real_vec = _mm512_fmadd_pd(b_vec, a_vec, ab_real_vec);
|
|
1446
|
+
ab_imag_vec = _mm512_fmadd_pd(
|
|
1447
|
+
_mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
|
|
1448
|
+
if (n) goto simsimd_dot_f64c_skylake_cycle;
|
|
1449
|
+
|
|
1450
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1451
|
+
ab_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_real_vec), sign_flip_vec));
|
|
1350
1452
|
|
|
1351
1453
|
// Reduce horizontal sums:
|
|
1352
1454
|
results[0] = _mm512_reduce_add_pd(ab_real_vec);
|
|
1353
1455
|
results[1] = _mm512_reduce_add_pd(ab_imag_vec);
|
|
1354
1456
|
}
|
|
1355
1457
|
|
|
1356
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const*
|
|
1357
|
-
simsimd_distance_t*
|
|
1358
|
-
|
|
1458
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1459
|
+
simsimd_distance_t *results) {
|
|
1359
1460
|
__m512d ab_real_vec = _mm512_setzero_pd();
|
|
1360
1461
|
__m512d ab_imag_vec = _mm512_setzero_pd();
|
|
1361
1462
|
__m512d a_vec;
|
|
@@ -1382,17 +1483,19 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1382
1483
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1383
1484
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1384
1485
|
n = 0;
|
|
1385
|
-
}
|
|
1486
|
+
}
|
|
1487
|
+
else {
|
|
1386
1488
|
a_vec = _mm512_loadu_pd(a);
|
|
1387
1489
|
b_vec = _mm512_loadu_pd(b);
|
|
1388
1490
|
a += 8, b += 8, n -= 8;
|
|
1389
1491
|
}
|
|
1390
1492
|
ab_real_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_real_vec);
|
|
1391
|
-
a_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(a_vec), sign_flip_vec));
|
|
1392
1493
|
b_vec = _mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec));
|
|
1393
1494
|
ab_imag_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_imag_vec);
|
|
1394
|
-
if (n)
|
|
1395
|
-
|
|
1495
|
+
if (n) goto simsimd_vdot_f64c_skylake_cycle;
|
|
1496
|
+
|
|
1497
|
+
// Flip the sign bit in every second scalar before accumulation:
|
|
1498
|
+
ab_imag_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_imag_vec), sign_flip_vec));
|
|
1396
1499
|
|
|
1397
1500
|
// Reduce horizontal sums:
|
|
1398
1501
|
results[0] = _mm512_reduce_add_pd(ab_real_vec);
|
|
@@ -1405,11 +1508,12 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1405
1508
|
|
|
1406
1509
|
#if SIMSIMD_TARGET_GENOA
|
|
1407
1510
|
#pragma GCC push_options
|
|
1408
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1409
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))),
|
|
1511
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1512
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1513
|
+
apply_to = function)
|
|
1410
1514
|
|
|
1411
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const*
|
|
1412
|
-
simsimd_distance_t*
|
|
1515
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1516
|
+
simsimd_distance_t *result) {
|
|
1413
1517
|
__m512 ab_vec = _mm512_setzero_ps();
|
|
1414
1518
|
__m512i a_i16_vec, b_i16_vec;
|
|
1415
1519
|
|
|
@@ -1419,21 +1523,20 @@ simsimd_dot_bf16_genoa_cycle:
|
|
|
1419
1523
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1420
1524
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1421
1525
|
n = 0;
|
|
1422
|
-
}
|
|
1526
|
+
}
|
|
1527
|
+
else {
|
|
1423
1528
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1424
1529
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1425
1530
|
a += 32, b += 32, n -= 32;
|
|
1426
1531
|
}
|
|
1427
1532
|
ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
|
|
1428
|
-
if (n)
|
|
1429
|
-
goto simsimd_dot_bf16_genoa_cycle;
|
|
1533
|
+
if (n) goto simsimd_dot_bf16_genoa_cycle;
|
|
1430
1534
|
|
|
1431
|
-
*result =
|
|
1535
|
+
*result = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1432
1536
|
}
|
|
1433
1537
|
|
|
1434
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const*
|
|
1435
|
-
simsimd_distance_t*
|
|
1436
|
-
|
|
1538
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1539
|
+
simsimd_distance_t *results) {
|
|
1437
1540
|
__m512 ab_real_vec = _mm512_setzero_ps();
|
|
1438
1541
|
__m512 ab_imag_vec = _mm512_setzero_ps();
|
|
1439
1542
|
__m512i a_vec;
|
|
@@ -1458,7 +1561,8 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1458
1561
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1459
1562
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1460
1563
|
n = 0;
|
|
1461
|
-
}
|
|
1564
|
+
}
|
|
1565
|
+
else {
|
|
1462
1566
|
a_vec = _mm512_loadu_epi16(a);
|
|
1463
1567
|
b_vec = _mm512_loadu_epi16(b);
|
|
1464
1568
|
a += 32, b += 32, n -= 32;
|
|
@@ -1466,17 +1570,15 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1466
1570
|
ab_real_vec = _mm512_dpbf16_ps(ab_real_vec, (__m512bh)(_mm512_xor_si512(b_vec, sign_flip_vec)), (__m512bh)(a_vec));
|
|
1467
1571
|
ab_imag_vec =
|
|
1468
1572
|
_mm512_dpbf16_ps(ab_imag_vec, (__m512bh)(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), (__m512bh)(a_vec));
|
|
1469
|
-
if (n)
|
|
1470
|
-
goto simsimd_dot_bf16c_genoa_cycle;
|
|
1573
|
+
if (n) goto simsimd_dot_bf16c_genoa_cycle;
|
|
1471
1574
|
|
|
1472
1575
|
// Reduce horizontal sums:
|
|
1473
|
-
results[0] =
|
|
1474
|
-
results[1] =
|
|
1576
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1577
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1475
1578
|
}
|
|
1476
1579
|
|
|
1477
|
-
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const*
|
|
1478
|
-
simsimd_distance_t*
|
|
1479
|
-
|
|
1580
|
+
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1581
|
+
simsimd_distance_t *results) {
|
|
1480
1582
|
__m512 ab_real_vec = _mm512_setzero_ps();
|
|
1481
1583
|
__m512 ab_imag_vec = _mm512_setzero_ps();
|
|
1482
1584
|
__m512i a_vec;
|
|
@@ -1501,7 +1603,8 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1501
1603
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1502
1604
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1503
1605
|
n = 0;
|
|
1504
|
-
}
|
|
1606
|
+
}
|
|
1607
|
+
else {
|
|
1505
1608
|
a_vec = _mm512_loadu_epi16(a);
|
|
1506
1609
|
b_vec = _mm512_loadu_epi16(b);
|
|
1507
1610
|
a += 32, b += 32, n -= 32;
|
|
@@ -1510,12 +1613,11 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1510
1613
|
a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
|
|
1511
1614
|
b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
|
|
1512
1615
|
ab_imag_vec = _mm512_dpbf16_ps(ab_imag_vec, (__m512bh)(a_vec), (__m512bh)(b_vec));
|
|
1513
|
-
if (n)
|
|
1514
|
-
goto simsimd_dot_bf16c_genoa_cycle;
|
|
1616
|
+
if (n) goto simsimd_dot_bf16c_genoa_cycle;
|
|
1515
1617
|
|
|
1516
1618
|
// Reduce horizontal sums:
|
|
1517
|
-
results[0] =
|
|
1518
|
-
results[1] =
|
|
1619
|
+
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1620
|
+
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1519
1621
|
}
|
|
1520
1622
|
|
|
1521
1623
|
#pragma clang attribute pop
|
|
@@ -1524,11 +1626,12 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1524
1626
|
|
|
1525
1627
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1526
1628
|
#pragma GCC push_options
|
|
1527
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1528
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))),
|
|
1629
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1630
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
|
|
1631
|
+
apply_to = function)
|
|
1529
1632
|
|
|
1530
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const*
|
|
1531
|
-
simsimd_distance_t*
|
|
1633
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1634
|
+
simsimd_distance_t *result) {
|
|
1532
1635
|
__m512h ab_vec = _mm512_setzero_ph();
|
|
1533
1636
|
__m512i a_i16_vec, b_i16_vec;
|
|
1534
1637
|
|
|
@@ -1538,21 +1641,20 @@ simsimd_dot_f16_sapphire_cycle:
|
|
|
1538
1641
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1539
1642
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1540
1643
|
n = 0;
|
|
1541
|
-
}
|
|
1644
|
+
}
|
|
1645
|
+
else {
|
|
1542
1646
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1543
1647
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1544
1648
|
a += 32, b += 32, n -= 32;
|
|
1545
1649
|
}
|
|
1546
1650
|
ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
|
|
1547
|
-
if (n)
|
|
1548
|
-
goto simsimd_dot_f16_sapphire_cycle;
|
|
1651
|
+
if (n) goto simsimd_dot_f16_sapphire_cycle;
|
|
1549
1652
|
|
|
1550
1653
|
*result = _mm512_reduce_add_ph(ab_vec);
|
|
1551
1654
|
}
|
|
1552
1655
|
|
|
1553
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const*
|
|
1554
|
-
simsimd_distance_t*
|
|
1555
|
-
|
|
1656
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1657
|
+
simsimd_distance_t *results) {
|
|
1556
1658
|
__m512h ab_real_vec = _mm512_setzero_ph();
|
|
1557
1659
|
__m512h ab_imag_vec = _mm512_setzero_ph();
|
|
1558
1660
|
__m512i a_vec;
|
|
@@ -1577,26 +1679,27 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1577
1679
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1578
1680
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1579
1681
|
n = 0;
|
|
1580
|
-
}
|
|
1682
|
+
}
|
|
1683
|
+
else {
|
|
1581
1684
|
a_vec = _mm512_loadu_epi16(a);
|
|
1582
1685
|
b_vec = _mm512_loadu_epi16(b);
|
|
1583
1686
|
a += 32, b += 32, n -= 32;
|
|
1584
1687
|
}
|
|
1688
|
+
// TODO: Consider using `_mm512_fmaddsub`
|
|
1585
1689
|
ab_real_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_xor_si512(b_vec, sign_flip_vec)),
|
|
1586
1690
|
_mm512_castsi512_ph(a_vec), ab_real_vec);
|
|
1587
1691
|
ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)),
|
|
1588
1692
|
_mm512_castsi512_ph(a_vec), ab_imag_vec);
|
|
1589
|
-
if (n)
|
|
1590
|
-
goto simsimd_dot_f16c_sapphire_cycle;
|
|
1693
|
+
if (n) goto simsimd_dot_f16c_sapphire_cycle;
|
|
1591
1694
|
|
|
1592
1695
|
// Reduce horizontal sums:
|
|
1696
|
+
// TODO: Optimize this with tree-like reductions
|
|
1593
1697
|
results[0] = _mm512_reduce_add_ph(ab_real_vec);
|
|
1594
1698
|
results[1] = _mm512_reduce_add_ph(ab_imag_vec);
|
|
1595
1699
|
}
|
|
1596
1700
|
|
|
1597
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const*
|
|
1598
|
-
simsimd_distance_t*
|
|
1599
|
-
|
|
1701
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1702
|
+
simsimd_distance_t *results) {
|
|
1600
1703
|
__m512h ab_real_vec = _mm512_setzero_ph();
|
|
1601
1704
|
__m512h ab_imag_vec = _mm512_setzero_ph();
|
|
1602
1705
|
__m512i a_vec;
|
|
@@ -1621,7 +1724,8 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1621
1724
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1622
1725
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1623
1726
|
n = 0;
|
|
1624
|
-
}
|
|
1727
|
+
}
|
|
1728
|
+
else {
|
|
1625
1729
|
a_vec = _mm512_loadu_epi16(a);
|
|
1626
1730
|
b_vec = _mm512_loadu_epi16(b);
|
|
1627
1731
|
a += 32, b += 32, n -= 32;
|
|
@@ -1630,8 +1734,7 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1630
1734
|
a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
|
|
1631
1735
|
b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
|
|
1632
1736
|
ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_imag_vec);
|
|
1633
|
-
if (n)
|
|
1634
|
-
goto simsimd_dot_f16c_sapphire_cycle;
|
|
1737
|
+
if (n) goto simsimd_dot_f16c_sapphire_cycle;
|
|
1635
1738
|
|
|
1636
1739
|
// Reduce horizontal sums:
|
|
1637
1740
|
results[0] = _mm512_reduce_add_ph(ab_real_vec);
|
|
@@ -1644,23 +1747,25 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1644
1747
|
|
|
1645
1748
|
#if SIMSIMD_TARGET_ICE
|
|
1646
1749
|
#pragma GCC push_options
|
|
1647
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1648
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))),
|
|
1750
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1751
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
|
|
1752
|
+
apply_to = function)
|
|
1649
1753
|
|
|
1650
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const*
|
|
1651
|
-
simsimd_distance_t*
|
|
1652
|
-
__m512i
|
|
1653
|
-
__m512i
|
|
1754
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1755
|
+
simsimd_distance_t *result) {
|
|
1756
|
+
__m512i ab_i32_vec = _mm512_setzero_si512();
|
|
1757
|
+
__m512i a_i16_vec, b_i16_vec;
|
|
1654
1758
|
|
|
1655
1759
|
simsimd_dot_i8_ice_cycle:
|
|
1656
1760
|
if (n < 32) {
|
|
1657
1761
|
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1658
|
-
|
|
1659
|
-
|
|
1762
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1763
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
1660
1764
|
n = 0;
|
|
1661
|
-
}
|
|
1662
|
-
|
|
1663
|
-
|
|
1765
|
+
}
|
|
1766
|
+
else {
|
|
1767
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
|
|
1768
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
|
|
1664
1769
|
a += 32, b += 32, n -= 32;
|
|
1665
1770
|
}
|
|
1666
1771
|
// Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
|
|
@@ -1668,17 +1773,85 @@ simsimd_dot_i8_ice_cycle:
|
|
|
1668
1773
|
// Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
|
|
1669
1774
|
// So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
|
|
1670
1775
|
// to 16-bit beforehand.
|
|
1671
|
-
|
|
1672
|
-
if (n)
|
|
1673
|
-
|
|
1776
|
+
ab_i32_vec = _mm512_dpwssd_epi32(ab_i32_vec, a_i16_vec, b_i16_vec);
|
|
1777
|
+
if (n) goto simsimd_dot_i8_ice_cycle;
|
|
1778
|
+
|
|
1779
|
+
*result = _mm512_reduce_add_epi32(ab_i32_vec);
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1783
|
+
simsimd_distance_t *result) {
|
|
1784
|
+
__m512i ab_i32_low_vec = _mm512_setzero_si512();
|
|
1785
|
+
__m512i ab_i32_high_vec = _mm512_setzero_si512();
|
|
1786
|
+
__m512i const zeros_vec = _mm512_setzero_si512();
|
|
1787
|
+
__m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
|
|
1788
|
+
__m512i a_u8_vec, b_u8_vec;
|
|
1789
|
+
|
|
1790
|
+
simsimd_dot_u8_ice_cycle:
|
|
1791
|
+
if (n < 64) {
|
|
1792
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
1793
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1794
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1795
|
+
n = 0;
|
|
1796
|
+
}
|
|
1797
|
+
else {
|
|
1798
|
+
a_u8_vec = _mm512_loadu_si512(a);
|
|
1799
|
+
b_u8_vec = _mm512_loadu_si512(b);
|
|
1800
|
+
a += 64, b += 64, n -= 64;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1804
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1805
|
+
a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
1806
|
+
a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
1807
|
+
b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
1808
|
+
b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
1809
|
+
// Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
|
|
1810
|
+
// as it's asymmetric with respect to the sign of the input arguments:
|
|
1811
|
+
// Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
|
|
1812
|
+
// So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
|
|
1813
|
+
// to 16-bit beforehand.
|
|
1814
|
+
ab_i32_low_vec = _mm512_dpwssd_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
|
|
1815
|
+
ab_i32_high_vec = _mm512_dpwssd_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
|
|
1816
|
+
if (n) goto simsimd_dot_u8_ice_cycle;
|
|
1674
1817
|
|
|
1675
|
-
*result = _mm512_reduce_add_epi32(
|
|
1818
|
+
*result = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1676
1819
|
}
|
|
1677
1820
|
|
|
1678
1821
|
#pragma clang attribute pop
|
|
1679
1822
|
#pragma GCC pop_options
|
|
1680
1823
|
#endif // SIMSIMD_TARGET_ICE
|
|
1681
|
-
|
|
1824
|
+
|
|
1825
|
+
#if SIMSIMD_TARGET_SIERRA
|
|
1826
|
+
#pragma GCC push_options
|
|
1827
|
+
#pragma GCC target("avx2", "bmi2", "avx2vnni")
|
|
1828
|
+
#pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
|
|
1829
|
+
|
|
1830
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1831
|
+
simsimd_distance_t *result) {
|
|
1832
|
+
|
|
1833
|
+
__m256i ab_i32_vec = _mm256_setzero_si256();
|
|
1834
|
+
|
|
1835
|
+
simsimd_size_t i = 0;
|
|
1836
|
+
for (; i + 32 <= n; i += 32) {
|
|
1837
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1838
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1839
|
+
ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// Further reduce to a single sum for each vector
|
|
1843
|
+
int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
|
|
1844
|
+
|
|
1845
|
+
// Take care of the tail:
|
|
1846
|
+
for (; i < n; ++i) ab += (int)(a[i]) * b[i];
|
|
1847
|
+
|
|
1848
|
+
*result = ab;
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
#pragma clang attribute pop
|
|
1852
|
+
#pragma GCC pop_options
|
|
1853
|
+
#endif // SIMSIMD_TARGET_SIERRA
|
|
1854
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
1682
1855
|
|
|
1683
1856
|
#ifdef __cplusplus
|
|
1684
1857
|
}
|