usearch 2.15.3 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/cpp/test.cpp +16 -0
- package/include/usearch/index.hpp +21 -5
- package/include/usearch/index_dense.hpp +6 -1
- package/include/usearch/index_plugins.hpp +345 -8
- package/javascript/lib.cpp +17 -5
- package/javascript/usearch.test.js +190 -21
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.git-blame-ignore-revs +2 -0
- package/simsimd/CONTRIBUTING.md +71 -23
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +4 -4
- package/simsimd/MANIFEST.in +2 -1
- package/simsimd/Package.swift +15 -3
- package/simsimd/README.md +104 -35
- package/simsimd/binding.gyp +6 -2
- package/simsimd/build.rs +2 -0
- package/simsimd/c/lib.c +180 -99
- package/simsimd/include/simsimd/binary.h +311 -102
- package/simsimd/include/simsimd/curved.h +134 -135
- package/simsimd/include/simsimd/dot.h +448 -274
- package/simsimd/include/simsimd/elementwise.h +1437 -0
- package/simsimd/include/simsimd/geospatial.h +2 -2
- package/simsimd/include/simsimd/mesh.h +2 -2
- package/simsimd/include/simsimd/probability.h +88 -88
- package/simsimd/include/simsimd/simsimd.h +1157 -772
- package/simsimd/include/simsimd/sparse.h +768 -147
- package/simsimd/include/simsimd/spatial.h +955 -376
- package/simsimd/include/simsimd/types.h +145 -71
- package/simsimd/javascript/lib.c +1 -2
- package/simsimd/package.json +4 -4
- package/simsimd/rust/lib.rs +14 -1
- package/simsimd/scripts/bench.cxx +1070 -0
- package/simsimd/scripts/bench.ipynb +187 -0
- package/simsimd/{javascript → scripts}/bench.js +1 -1
- package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
- package/simsimd/scripts/bench_vectors.py +700 -0
- package/simsimd/scripts/bench_vectors_live.py +213 -0
- package/simsimd/scripts/test.c +155 -0
- package/simsimd/{javascript → scripts}/test.mjs +2 -2
- package/simsimd/scripts/test.py +1410 -0
- package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
- package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
- /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
- /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
* - 32-bit IEEE floating point numbers
|
|
14
14
|
* - 16-bit IEEE floating point numbers
|
|
15
15
|
* - 16-bit brain floating point numbers
|
|
16
|
+
* - 8-bit unsigned integers
|
|
16
17
|
* - 8-bit signed integers
|
|
17
18
|
*
|
|
18
19
|
* For hardware architectures:
|
|
19
|
-
* - Arm
|
|
20
|
-
* - x86
|
|
20
|
+
* - Arm: NEON, SVE
|
|
21
|
+
* - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
|
|
21
22
|
*
|
|
22
23
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
23
24
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -54,6 +55,7 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_serial(simsimd_bf16_t const* a, simsimd_bf
|
|
|
54
55
|
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_serial(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
55
56
|
|
|
56
57
|
SIMSIMD_PUBLIC void simsimd_dot_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
58
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
57
59
|
|
|
58
60
|
/* Double-precision serial backends for all numeric types.
|
|
59
61
|
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
@@ -83,6 +85,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
83
85
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
84
86
|
|
|
85
87
|
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
88
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
86
89
|
|
|
87
90
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
88
91
|
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* results);
|
|
@@ -120,6 +123,7 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
|
|
|
120
123
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
121
124
|
|
|
122
125
|
SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
126
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
123
127
|
|
|
124
128
|
/* SIMD-powered backends for various generations of AVX512 CPUs.
|
|
125
129
|
* Skylake is handy, as it supports masked loads and other operations, avoiding the need for the tail loop.
|
|
@@ -136,6 +140,7 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32
|
|
|
136
140
|
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
137
141
|
|
|
138
142
|
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
143
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
139
144
|
|
|
140
145
|
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
141
146
|
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
@@ -144,53 +149,55 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const* a, simsimd_bf
|
|
|
144
149
|
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
145
150
|
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
146
151
|
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
152
|
+
|
|
153
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* result);
|
|
147
154
|
// clang-format on
|
|
148
155
|
|
|
149
|
-
#define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, load_and_convert)
|
|
150
|
-
SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const*
|
|
151
|
-
simsimd_##input_type##_t const*
|
|
152
|
-
simsimd_distance_t*
|
|
153
|
-
simsimd_##accumulator_type##_t ab = 0;
|
|
154
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
155
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i);
|
|
156
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i);
|
|
157
|
-
ab += ai * bi;
|
|
158
|
-
}
|
|
159
|
-
*result = ab;
|
|
156
|
+
#define SIMSIMD_MAKE_DOT(name, input_type, accumulator_type, load_and_convert) \
|
|
157
|
+
SIMSIMD_PUBLIC void simsimd_dot_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
158
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
159
|
+
simsimd_distance_t *result) { \
|
|
160
|
+
simsimd_##accumulator_type##_t ab = 0; \
|
|
161
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
162
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
163
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
164
|
+
ab += ai * bi; \
|
|
165
|
+
} \
|
|
166
|
+
*result = ab; \
|
|
160
167
|
}
|
|
161
168
|
|
|
162
|
-
#define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, load_and_convert)
|
|
163
|
-
SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const*
|
|
164
|
-
simsimd_##input_type##_t const*
|
|
165
|
-
simsimd_distance_t*
|
|
166
|
-
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0;
|
|
167
|
-
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) {
|
|
168
|
-
simsimd_##accumulator_type##_t ar = load_and_convert(a + i);
|
|
169
|
-
simsimd_##accumulator_type##_t br = load_and_convert(b + i);
|
|
170
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1);
|
|
171
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1);
|
|
172
|
-
ab_real += ar * br - ai * bi;
|
|
173
|
-
ab_imag += ar * bi + ai * br;
|
|
174
|
-
}
|
|
175
|
-
results[0] = ab_real;
|
|
176
|
-
results[1] = ab_imag;
|
|
169
|
+
#define SIMSIMD_MAKE_COMPLEX_DOT(name, input_type, accumulator_type, load_and_convert) \
|
|
170
|
+
SIMSIMD_PUBLIC void simsimd_dot_##input_type##c_##name(simsimd_##input_type##_t const *a, \
|
|
171
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
172
|
+
simsimd_distance_t *results) { \
|
|
173
|
+
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
|
|
174
|
+
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
|
|
175
|
+
simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
|
|
176
|
+
simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
|
|
177
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
|
|
178
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
|
|
179
|
+
ab_real += ar * br - ai * bi; \
|
|
180
|
+
ab_imag += ar * bi + ai * br; \
|
|
181
|
+
} \
|
|
182
|
+
results[0] = ab_real; \
|
|
183
|
+
results[1] = ab_imag; \
|
|
177
184
|
}
|
|
178
185
|
|
|
179
|
-
#define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, load_and_convert)
|
|
180
|
-
SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const*
|
|
181
|
-
simsimd_##input_type##_t const*
|
|
182
|
-
simsimd_distance_t*
|
|
183
|
-
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0;
|
|
184
|
-
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) {
|
|
185
|
-
simsimd_##accumulator_type##_t ar = load_and_convert(a + i);
|
|
186
|
-
simsimd_##accumulator_type##_t br = load_and_convert(b + i);
|
|
187
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1);
|
|
188
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1);
|
|
189
|
-
ab_real += ar * br + ai * bi;
|
|
190
|
-
ab_imag += ar * bi - ai * br;
|
|
191
|
-
}
|
|
192
|
-
results[0] = ab_real;
|
|
193
|
-
results[1] = ab_imag;
|
|
186
|
+
#define SIMSIMD_MAKE_COMPLEX_VDOT(name, input_type, accumulator_type, load_and_convert) \
|
|
187
|
+
SIMSIMD_PUBLIC void simsimd_vdot_##input_type##c_##name(simsimd_##input_type##_t const *a, \
|
|
188
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
189
|
+
simsimd_distance_t *results) { \
|
|
190
|
+
simsimd_##accumulator_type##_t ab_real = 0, ab_imag = 0; \
|
|
191
|
+
for (simsimd_size_t i = 0; i + 2 <= n; i += 2) { \
|
|
192
|
+
simsimd_##accumulator_type##_t ar = load_and_convert(a + i); \
|
|
193
|
+
simsimd_##accumulator_type##_t br = load_and_convert(b + i); \
|
|
194
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i + 1); \
|
|
195
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i + 1); \
|
|
196
|
+
ab_real += ar * br + ai * bi; \
|
|
197
|
+
ab_imag += ar * bi - ai * br; \
|
|
198
|
+
} \
|
|
199
|
+
results[0] = ab_real; \
|
|
200
|
+
results[1] = ab_imag; \
|
|
194
201
|
}
|
|
195
202
|
|
|
196
203
|
SIMSIMD_MAKE_DOT(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f64_serial
|
|
@@ -210,6 +217,7 @@ SIMSIMD_MAKE_COMPLEX_DOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_dot
|
|
|
210
217
|
SIMSIMD_MAKE_COMPLEX_VDOT(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_serial
|
|
211
218
|
|
|
212
219
|
SIMSIMD_MAKE_DOT(serial, i8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_i8_serial
|
|
220
|
+
SIMSIMD_MAKE_DOT(serial, u8, i64, SIMSIMD_DEREFERENCE) // simsimd_dot_u8_serial
|
|
213
221
|
|
|
214
222
|
SIMSIMD_MAKE_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32_accurate
|
|
215
223
|
SIMSIMD_MAKE_COMPLEX_DOT(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_dot_f32c_accurate
|
|
@@ -223,27 +231,25 @@ SIMSIMD_MAKE_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_d
|
|
|
223
231
|
SIMSIMD_MAKE_COMPLEX_DOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_dot_bf16c_accurate
|
|
224
232
|
SIMSIMD_MAKE_COMPLEX_VDOT(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_vdot_bf16c_accurate
|
|
225
233
|
|
|
226
|
-
#if
|
|
234
|
+
#if _SIMSIMD_TARGET_ARM
|
|
227
235
|
#if SIMSIMD_TARGET_NEON
|
|
228
236
|
#pragma GCC push_options
|
|
229
237
|
#pragma GCC target("arch=armv8.2-a+simd")
|
|
230
238
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
231
239
|
|
|
232
|
-
SIMSIMD_INTERNAL float32x4_t _simsimd_partial_load_f32x4_neon(simsimd_f32_t const*
|
|
240
|
+
SIMSIMD_INTERNAL float32x4_t _simsimd_partial_load_f32x4_neon(simsimd_f32_t const *a, simsimd_size_t n) {
|
|
233
241
|
union {
|
|
234
242
|
float32x4_t vec;
|
|
235
243
|
simsimd_f32_t scalars[4];
|
|
236
244
|
} result;
|
|
237
245
|
simsimd_size_t i = 0;
|
|
238
|
-
for (; i < n; ++i)
|
|
239
|
-
|
|
240
|
-
for (; i < 4; ++i)
|
|
241
|
-
result.scalars[i] = 0;
|
|
246
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
247
|
+
for (; i < 4; ++i) result.scalars[i] = 0;
|
|
242
248
|
return result.vec;
|
|
243
249
|
}
|
|
244
250
|
|
|
245
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const*
|
|
246
|
-
simsimd_distance_t*
|
|
251
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
252
|
+
simsimd_distance_t *result) {
|
|
247
253
|
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
248
254
|
simsimd_size_t i = 0;
|
|
249
255
|
for (; i + 4 <= n; i += 4) {
|
|
@@ -252,13 +258,12 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
252
258
|
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
253
259
|
}
|
|
254
260
|
simsimd_f32_t ab = vaddvq_f32(ab_vec);
|
|
255
|
-
for (; i < n; ++i)
|
|
256
|
-
ab += a[i] * b[i];
|
|
261
|
+
for (; i < n; ++i) ab += a[i] * b[i];
|
|
257
262
|
*result = ab;
|
|
258
263
|
}
|
|
259
264
|
|
|
260
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const*
|
|
261
|
-
simsimd_distance_t*
|
|
265
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
266
|
+
simsimd_distance_t *results) {
|
|
262
267
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
263
268
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
264
269
|
simsimd_size_t i = 0;
|
|
@@ -292,8 +297,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
292
297
|
results[1] = ab_imag;
|
|
293
298
|
}
|
|
294
299
|
|
|
295
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const*
|
|
296
|
-
simsimd_distance_t*
|
|
300
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
|
|
301
|
+
simsimd_distance_t *results) {
|
|
297
302
|
float32x4_t ab_real_vec = vdupq_n_f32(0);
|
|
298
303
|
float32x4_t ab_imag_vec = vdupq_n_f32(0);
|
|
299
304
|
simsimd_size_t i = 0;
|
|
@@ -329,13 +334,15 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_neon(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
329
334
|
|
|
330
335
|
#pragma clang attribute pop
|
|
331
336
|
#pragma GCC pop_options
|
|
337
|
+
#endif // SIMSIMD_TARGET_NEON
|
|
332
338
|
|
|
339
|
+
#if SIMSIMD_TARGET_NEON_I8
|
|
333
340
|
#pragma GCC push_options
|
|
334
341
|
#pragma GCC target("arch=armv8.2-a+dotprod")
|
|
335
342
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod"))), apply_to = function)
|
|
336
343
|
|
|
337
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const*
|
|
338
|
-
simsimd_distance_t*
|
|
344
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
345
|
+
simsimd_distance_t *result) {
|
|
339
346
|
|
|
340
347
|
int32x4_t ab_vec = vdupq_n_s32(0);
|
|
341
348
|
simsimd_size_t i = 0;
|
|
@@ -364,16 +371,37 @@ SIMSIMD_PUBLIC void simsimd_dot_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
364
371
|
*result = ab;
|
|
365
372
|
}
|
|
366
373
|
|
|
374
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
375
|
+
simsimd_distance_t *result) {
|
|
376
|
+
|
|
377
|
+
uint32x4_t ab_vec = vdupq_n_u32(0);
|
|
378
|
+
simsimd_size_t i = 0;
|
|
379
|
+
for (; i + 16 <= n; i += 16) {
|
|
380
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
381
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
382
|
+
ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// Take care of the tail:
|
|
386
|
+
uint32_t ab = vaddvq_u32(ab_vec);
|
|
387
|
+
for (; i < n; ++i) {
|
|
388
|
+
uint32_t ai = a[i], bi = b[i];
|
|
389
|
+
ab += ai * bi;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
*result = ab;
|
|
393
|
+
}
|
|
394
|
+
|
|
367
395
|
#pragma clang attribute pop
|
|
368
396
|
#pragma GCC pop_options
|
|
369
|
-
#endif
|
|
397
|
+
#endif // SIMSIMD_TARGET_NEON_I8
|
|
370
398
|
|
|
371
399
|
#if SIMSIMD_TARGET_NEON_F16
|
|
372
400
|
#pragma GCC push_options
|
|
373
401
|
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
374
402
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
375
403
|
|
|
376
|
-
SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t const*
|
|
404
|
+
SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t const *a, simsimd_size_t n) {
|
|
377
405
|
// In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
|
|
378
406
|
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
379
407
|
// loads and stores with vectorized math.
|
|
@@ -382,15 +410,13 @@ SIMSIMD_INTERNAL float16x4_t _simsimd_partial_load_f16x4_neon(simsimd_f16_t cons
|
|
|
382
410
|
simsimd_f16_t scalars[4];
|
|
383
411
|
} result;
|
|
384
412
|
simsimd_size_t i = 0;
|
|
385
|
-
for (; i < n; ++i)
|
|
386
|
-
|
|
387
|
-
for (; i < 4; ++i)
|
|
388
|
-
result.scalars[i] = 0;
|
|
413
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
414
|
+
for (; i < 4; ++i) result.scalars[i] = 0;
|
|
389
415
|
return result.vec;
|
|
390
416
|
}
|
|
391
417
|
|
|
392
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const*
|
|
393
|
-
simsimd_distance_t*
|
|
418
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
419
|
+
simsimd_distance_t *result) {
|
|
394
420
|
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
395
421
|
float32x4_t a_vec, b_vec;
|
|
396
422
|
simsimd_size_t i = 0;
|
|
@@ -400,19 +426,19 @@ simsimd_dot_f16_neon_cycle:
|
|
|
400
426
|
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
401
427
|
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
402
428
|
n = 0;
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
432
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
406
433
|
a += 4, b += 4, n -= 4;
|
|
407
434
|
}
|
|
408
435
|
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
409
|
-
if (n)
|
|
410
|
-
goto simsimd_dot_f16_neon_cycle;
|
|
436
|
+
if (n) goto simsimd_dot_f16_neon_cycle;
|
|
411
437
|
*result = vaddvq_f32(ab_vec);
|
|
412
438
|
}
|
|
413
439
|
|
|
414
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const*
|
|
415
|
-
simsimd_distance_t*
|
|
440
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
441
|
+
simsimd_distance_t *results) {
|
|
416
442
|
|
|
417
443
|
// A nicer approach is to use `f16` arithmetic for the dot product, but that requires
|
|
418
444
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
@@ -424,8 +450,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
424
450
|
// Unpack the input arrays into real and imaginary parts.
|
|
425
451
|
// MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
|
|
426
452
|
// integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
|
|
427
|
-
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
428
|
-
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
453
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
454
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
429
455
|
float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
|
|
430
456
|
float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
|
|
431
457
|
float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
|
|
@@ -446,8 +472,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
446
472
|
results[1] += vaddvq_f32(ab_imag_vec);
|
|
447
473
|
}
|
|
448
474
|
|
|
449
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const*
|
|
450
|
-
simsimd_distance_t*
|
|
475
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
|
|
476
|
+
simsimd_distance_t *results) {
|
|
451
477
|
|
|
452
478
|
// A nicer approach is to use `f16` arithmetic for the dot product, but that requires
|
|
453
479
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
@@ -459,8 +485,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
459
485
|
// Unpack the input arrays into real and imaginary parts.
|
|
460
486
|
// MSVC sadly doesn't recognize the `vld2_f16`, so we load the data as signed
|
|
461
487
|
// integers of the same size and reinterpret with `vreinterpret_f16_s16` afterwards.
|
|
462
|
-
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
463
|
-
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
488
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
489
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
464
490
|
float32x4_t a_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[0]));
|
|
465
491
|
float32x4_t a_imag_vec = vcvt_f32_f16(vreinterpret_f16_s16(a_vec.val[1]));
|
|
466
492
|
float32x4_t b_real_vec = vcvt_f32_f16(vreinterpret_f16_s16(b_vec.val[0]));
|
|
@@ -490,21 +516,19 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_neon(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
490
516
|
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
491
517
|
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
492
518
|
|
|
493
|
-
SIMSIMD_INTERNAL bfloat16x8_t _simsimd_partial_load_bf16x8_neon(simsimd_bf16_t const*
|
|
519
|
+
SIMSIMD_INTERNAL bfloat16x8_t _simsimd_partial_load_bf16x8_neon(simsimd_bf16_t const *a, simsimd_size_t n) {
|
|
494
520
|
union {
|
|
495
521
|
bfloat16x8_t vec;
|
|
496
522
|
simsimd_bf16_t scalars[8];
|
|
497
523
|
} result;
|
|
498
524
|
simsimd_size_t i = 0;
|
|
499
|
-
for (; i < n; ++i)
|
|
500
|
-
|
|
501
|
-
for (; i < 8; ++i)
|
|
502
|
-
result.scalars[i] = 0;
|
|
525
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
526
|
+
for (; i < 8; ++i) result.scalars[i] = 0;
|
|
503
527
|
return result.vec;
|
|
504
528
|
}
|
|
505
529
|
|
|
506
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const*
|
|
507
|
-
simsimd_distance_t*
|
|
530
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
531
|
+
simsimd_distance_t *result) {
|
|
508
532
|
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
509
533
|
bfloat16x8_t a_vec, b_vec;
|
|
510
534
|
|
|
@@ -513,20 +537,20 @@ simsimd_dot_bf16_neon_cycle:
|
|
|
513
537
|
a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
514
538
|
b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
515
539
|
n = 0;
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
|
|
540
|
+
}
|
|
541
|
+
else {
|
|
542
|
+
a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
|
|
543
|
+
b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
|
|
519
544
|
a += 8, b += 8, n -= 8;
|
|
520
545
|
}
|
|
521
546
|
ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
|
|
522
|
-
if (n)
|
|
523
|
-
goto simsimd_dot_bf16_neon_cycle;
|
|
547
|
+
if (n) goto simsimd_dot_bf16_neon_cycle;
|
|
524
548
|
|
|
525
549
|
*result = vaddvq_f32(ab_vec);
|
|
526
550
|
}
|
|
527
551
|
|
|
528
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const*
|
|
529
|
-
simsimd_distance_t*
|
|
552
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
553
|
+
simsimd_distance_t *results) {
|
|
530
554
|
|
|
531
555
|
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
532
556
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
@@ -538,8 +562,8 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
538
562
|
// Unpack the input arrays into real and imaginary parts.
|
|
539
563
|
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
540
564
|
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
541
|
-
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
542
|
-
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
565
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
566
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
543
567
|
float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
|
|
544
568
|
float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
|
|
545
569
|
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
@@ -560,8 +584,8 @@ SIMSIMD_PUBLIC void simsimd_dot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
560
584
|
results[1] += vaddvq_f32(ab_imag_vec);
|
|
561
585
|
}
|
|
562
586
|
|
|
563
|
-
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const*
|
|
564
|
-
simsimd_distance_t*
|
|
587
|
+
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
|
|
588
|
+
simsimd_distance_t *results) {
|
|
565
589
|
|
|
566
590
|
// A nicer approach is to use `bf16` arithmetic for the dot product, but that requires
|
|
567
591
|
// FMLA extensions available on Arm v8.3 and later. That we can also process 16 entries
|
|
@@ -573,8 +597,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
573
597
|
// Unpack the input arrays into real and imaginary parts.
|
|
574
598
|
// MSVC sadly doesn't recognize the `vld2_bf16`, so we load the data as signed
|
|
575
599
|
// integers of the same size and reinterpret with `vreinterpret_bf16_s16` afterwards.
|
|
576
|
-
int16x4x2_t a_vec = vld2_s16((short*)a);
|
|
577
|
-
int16x4x2_t b_vec = vld2_s16((short*)b);
|
|
600
|
+
int16x4x2_t a_vec = vld2_s16((short *)a);
|
|
601
|
+
int16x4x2_t b_vec = vld2_s16((short *)b);
|
|
578
602
|
float32x4_t a_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[0]));
|
|
579
603
|
float32x4_t a_imag_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(a_vec.val[1]));
|
|
580
604
|
float32x4_t b_real_vec = vcvt_f32_bf16(vreinterpret_bf16_s16(b_vec.val[0]));
|
|
@@ -605,8 +629,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_bf16c_neon(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
605
629
|
#pragma GCC target("arch=armv8.2-a+sve")
|
|
606
630
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
|
|
607
631
|
|
|
608
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const*
|
|
609
|
-
simsimd_distance_t*
|
|
632
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
633
|
+
simsimd_distance_t *result) {
|
|
610
634
|
simsimd_size_t i = 0;
|
|
611
635
|
svfloat32_t ab_vec = svdup_f32(0.f);
|
|
612
636
|
do {
|
|
@@ -619,8 +643,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
|
|
|
619
643
|
*result = svaddv_f32(svptrue_b32(), ab_vec);
|
|
620
644
|
}
|
|
621
645
|
|
|
622
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const*
|
|
623
|
-
simsimd_distance_t*
|
|
646
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
647
|
+
simsimd_distance_t *results) {
|
|
624
648
|
simsimd_size_t i = 0;
|
|
625
649
|
svfloat32_t ab_real_vec = svdup_f32(0.f);
|
|
626
650
|
svfloat32_t ab_imag_vec = svdup_f32(0.f);
|
|
@@ -642,8 +666,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
642
666
|
results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
|
|
643
667
|
}
|
|
644
668
|
|
|
645
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const*
|
|
646
|
-
simsimd_distance_t*
|
|
669
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
670
|
+
simsimd_distance_t *results) {
|
|
647
671
|
simsimd_size_t i = 0;
|
|
648
672
|
svfloat32_t ab_real_vec = svdup_f32(0.f);
|
|
649
673
|
svfloat32_t ab_imag_vec = svdup_f32(0.f);
|
|
@@ -665,8 +689,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_sve(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
665
689
|
results[1] = svaddv_f32(svptrue_b32(), ab_imag_vec);
|
|
666
690
|
}
|
|
667
691
|
|
|
668
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const*
|
|
669
|
-
simsimd_distance_t*
|
|
692
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
693
|
+
simsimd_distance_t *result) {
|
|
670
694
|
simsimd_size_t i = 0;
|
|
671
695
|
svfloat64_t ab_vec = svdup_f64(0.);
|
|
672
696
|
do {
|
|
@@ -679,8 +703,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
679
703
|
*result = svaddv_f64(svptrue_b32(), ab_vec);
|
|
680
704
|
}
|
|
681
705
|
|
|
682
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const*
|
|
683
|
-
simsimd_distance_t*
|
|
706
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
707
|
+
simsimd_distance_t *results) {
|
|
684
708
|
simsimd_size_t i = 0;
|
|
685
709
|
svfloat64_t ab_real_vec = svdup_f64(0.);
|
|
686
710
|
svfloat64_t ab_imag_vec = svdup_f64(0.);
|
|
@@ -702,8 +726,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t c
|
|
|
702
726
|
results[1] = svaddv_f64(svptrue_b64(), ab_imag_vec);
|
|
703
727
|
}
|
|
704
728
|
|
|
705
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const*
|
|
706
|
-
simsimd_distance_t*
|
|
729
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
730
|
+
simsimd_distance_t *results) {
|
|
707
731
|
simsimd_size_t i = 0;
|
|
708
732
|
svfloat64_t ab_real_vec = svdup_f64(0.);
|
|
709
733
|
svfloat64_t ab_imag_vec = svdup_f64(0.);
|
|
@@ -732,12 +756,12 @@ SIMSIMD_PUBLIC void simsimd_vdot_f64c_sve(simsimd_f64_t const* a, simsimd_f64_t
|
|
|
732
756
|
#pragma GCC target("arch=armv8.2-a+sve+fp16")
|
|
733
757
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
|
|
734
758
|
|
|
735
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const*
|
|
736
|
-
simsimd_distance_t*
|
|
759
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
|
|
760
|
+
simsimd_distance_t *result) {
|
|
737
761
|
simsimd_size_t i = 0;
|
|
738
762
|
svfloat16_t ab_vec = svdup_f16(0);
|
|
739
|
-
simsimd_f16_for_arm_simd_t const*
|
|
740
|
-
simsimd_f16_for_arm_simd_t const*
|
|
763
|
+
simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
|
|
764
|
+
simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
|
|
741
765
|
do {
|
|
742
766
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
743
767
|
svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
|
|
@@ -749,15 +773,15 @@ SIMSIMD_PUBLIC void simsimd_dot_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
|
|
|
749
773
|
*result = ab;
|
|
750
774
|
}
|
|
751
775
|
|
|
752
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const*
|
|
753
|
-
simsimd_distance_t*
|
|
776
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
777
|
+
simsimd_distance_t *results) {
|
|
754
778
|
simsimd_size_t i = 0;
|
|
755
779
|
svfloat16_t ab_real_vec = svdup_f16(0);
|
|
756
780
|
svfloat16_t ab_imag_vec = svdup_f16(0);
|
|
757
781
|
do {
|
|
758
782
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i / 2, (unsigned int)n / 2);
|
|
759
|
-
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)a + i);
|
|
760
|
-
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)b + i);
|
|
783
|
+
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)a + i);
|
|
784
|
+
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)b + i);
|
|
761
785
|
svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
|
|
762
786
|
svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
|
|
763
787
|
svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
|
|
@@ -772,15 +796,15 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t c
|
|
|
772
796
|
results[1] = svaddv_f16(svptrue_b16(), ab_imag_vec);
|
|
773
797
|
}
|
|
774
798
|
|
|
775
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const*
|
|
776
|
-
simsimd_distance_t*
|
|
799
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
800
|
+
simsimd_distance_t *results) {
|
|
777
801
|
simsimd_size_t i = 0;
|
|
778
802
|
svfloat16_t ab_real_vec = svdup_f16(0);
|
|
779
803
|
svfloat16_t ab_imag_vec = svdup_f16(0);
|
|
780
804
|
do {
|
|
781
805
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i / 2, (unsigned int)n / 2);
|
|
782
|
-
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)a + i);
|
|
783
|
-
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const*)b + i);
|
|
806
|
+
svfloat16x2_t a_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)a + i);
|
|
807
|
+
svfloat16x2_t b_vec = svld2_f16(pg_vec, (simsimd_f16_for_arm_simd_t const *)b + i);
|
|
784
808
|
svfloat16_t a_real_vec = svget2_f16(a_vec, 0);
|
|
785
809
|
svfloat16_t a_imag_vec = svget2_f16(a_vec, 1);
|
|
786
810
|
svfloat16_t b_real_vec = svget2_f16(b_vec, 0);
|
|
@@ -798,14 +822,28 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_sve(simsimd_f16_t const* a, simsimd_f16_t
|
|
|
798
822
|
#pragma clang attribute pop
|
|
799
823
|
#pragma GCC pop_options
|
|
800
824
|
#endif // SIMSIMD_TARGET_SVE
|
|
801
|
-
#endif //
|
|
825
|
+
#endif // _SIMSIMD_TARGET_ARM
|
|
802
826
|
|
|
803
|
-
#if
|
|
827
|
+
#if _SIMSIMD_TARGET_X86
|
|
804
828
|
#if SIMSIMD_TARGET_HASWELL
|
|
805
829
|
#pragma GCC push_options
|
|
806
830
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
807
831
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
808
832
|
|
|
833
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f64x4_haswell(__m256d vec) {
|
|
834
|
+
// Reduce the double-precision vector to a scalar
|
|
835
|
+
// Horizontal add the first and second double-precision values, and third and fourth
|
|
836
|
+
__m128d vec_low = _mm256_castpd256_pd128(vec);
|
|
837
|
+
__m128d vec_high = _mm256_extractf128_pd(vec, 1);
|
|
838
|
+
__m128d vec128 = _mm_add_pd(vec_low, vec_high);
|
|
839
|
+
|
|
840
|
+
// Horizontal add again to accumulate all four values into one
|
|
841
|
+
vec128 = _mm_hadd_pd(vec128, vec128);
|
|
842
|
+
|
|
843
|
+
// Convert the final sum to a scalar double-precision value and return
|
|
844
|
+
return _mm_cvtsd_f64(vec128);
|
|
845
|
+
}
|
|
846
|
+
|
|
809
847
|
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x8_haswell(__m256 vec) {
|
|
810
848
|
// Convert the lower and higher 128-bit lanes of the input vector to double precision
|
|
811
849
|
__m128 low_f32 = _mm256_castps256_ps128(vec);
|
|
@@ -817,22 +855,20 @@ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x8_haswell(__m256 vec) {
|
|
|
817
855
|
|
|
818
856
|
// Perform the addition in double-precision
|
|
819
857
|
__m256d sum = _mm256_add_pd(low_f64, high_f64);
|
|
858
|
+
return _simsimd_reduce_f64x4_haswell(sum);
|
|
859
|
+
}
|
|
820
860
|
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
sum128 = _mm_hadd_pd(sum128, sum128);
|
|
829
|
-
|
|
830
|
-
// Convert the final sum to a scalar double-precision value and return
|
|
831
|
-
return _mm_cvtsd_f64(sum128);
|
|
861
|
+
SIMSIMD_INTERNAL simsimd_i32_t _simsimd_reduce_i32x8_haswell(__m256i vec) {
|
|
862
|
+
__m128i low = _mm256_castsi256_si128(vec);
|
|
863
|
+
__m128i high = _mm256_extracti128_si256(vec, 1);
|
|
864
|
+
__m128i sum = _mm_add_epi32(low, high);
|
|
865
|
+
sum = _mm_hadd_epi32(sum, sum);
|
|
866
|
+
sum = _mm_hadd_epi32(sum, sum);
|
|
867
|
+
return _mm_cvtsi128_si32(sum);
|
|
832
868
|
}
|
|
833
869
|
|
|
834
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const*
|
|
835
|
-
simsimd_distance_t*
|
|
870
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
871
|
+
simsimd_distance_t *results) {
|
|
836
872
|
|
|
837
873
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
838
874
|
simsimd_size_t i = 0;
|
|
@@ -842,13 +878,12 @@ SIMSIMD_PUBLIC void simsimd_dot_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
842
878
|
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
843
879
|
}
|
|
844
880
|
simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
845
|
-
for (; i < n; ++i)
|
|
846
|
-
ab += a[i] * b[i];
|
|
881
|
+
for (; i < n; ++i) ab += a[i] * b[i];
|
|
847
882
|
*results = ab;
|
|
848
883
|
}
|
|
849
884
|
|
|
850
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const*
|
|
851
|
-
simsimd_distance_t*
|
|
885
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
886
|
+
simsimd_distance_t *results) {
|
|
852
887
|
|
|
853
888
|
// The naive approach would be to use FMA and FMS instructions on different parts of the vectors.
|
|
854
889
|
// Prior to that we would need to shuffle the input vectors to separate real and imaginary parts.
|
|
@@ -917,8 +952,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f32c_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
917
952
|
results[1] = ab_imag;
|
|
918
953
|
}
|
|
919
954
|
|
|
920
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const*
|
|
921
|
-
simsimd_distance_t*
|
|
955
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
956
|
+
simsimd_distance_t *results) {
|
|
922
957
|
|
|
923
958
|
__m256 ab_real_vec = _mm256_setzero_ps();
|
|
924
959
|
__m256 ab_imag_vec = _mm256_setzero_ps();
|
|
@@ -959,7 +994,7 @@ SIMSIMD_PUBLIC void simsimd_vdot_f32c_haswell(simsimd_f32_t const* a, simsimd_f3
|
|
|
959
994
|
results[1] = ab_imag;
|
|
960
995
|
}
|
|
961
996
|
|
|
962
|
-
SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const*
|
|
997
|
+
SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const *a, simsimd_size_t n) {
|
|
963
998
|
// In case the software emulation for `f16` scalars is enabled, the `simsimd_f16_to_f32`
|
|
964
999
|
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
965
1000
|
// loads and stores with vectorized math.
|
|
@@ -968,15 +1003,13 @@ SIMSIMD_INTERNAL __m256 _simsimd_partial_load_f16x8_haswell(simsimd_f16_t const*
|
|
|
968
1003
|
simsimd_f16_t scalars[8];
|
|
969
1004
|
} result;
|
|
970
1005
|
simsimd_size_t i = 0;
|
|
971
|
-
for (; i < n; ++i)
|
|
972
|
-
|
|
973
|
-
for (; i < 8; ++i)
|
|
974
|
-
result.scalars[i] = 0;
|
|
1006
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
1007
|
+
for (; i < 8; ++i) result.scalars[i] = 0;
|
|
975
1008
|
return _mm256_cvtph_ps(result.vec);
|
|
976
1009
|
}
|
|
977
1010
|
|
|
978
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const*
|
|
979
|
-
simsimd_distance_t*
|
|
1011
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1012
|
+
simsimd_distance_t *result) {
|
|
980
1013
|
__m256 a_vec, b_vec;
|
|
981
1014
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
982
1015
|
|
|
@@ -985,20 +1018,26 @@ simsimd_dot_f16_haswell_cycle:
|
|
|
985
1018
|
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
986
1019
|
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
987
1020
|
n = 0;
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
|
|
1021
|
+
}
|
|
1022
|
+
else {
|
|
1023
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1024
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
991
1025
|
n -= 8, a += 8, b += 8;
|
|
992
1026
|
}
|
|
1027
|
+
// We can silence the NaNs using blends:
|
|
1028
|
+
//
|
|
1029
|
+
// __m256 a_is_nan = _mm256_cmp_ps(a_vec, a_vec, _CMP_UNORD_Q);
|
|
1030
|
+
// __m256 b_is_nan = _mm256_cmp_ps(b_vec, b_vec, _CMP_UNORD_Q);
|
|
1031
|
+
// ab_vec = _mm256_blendv_ps(_mm256_fmadd_ps(a_vec, b_vec, ab_vec), ab_vec, _mm256_or_ps(a_is_nan, b_is_nan));
|
|
1032
|
+
//
|
|
993
1033
|
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
994
|
-
if (n)
|
|
995
|
-
goto simsimd_dot_f16_haswell_cycle;
|
|
1034
|
+
if (n) goto simsimd_dot_f16_haswell_cycle;
|
|
996
1035
|
|
|
997
1036
|
*result = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
998
1037
|
}
|
|
999
1038
|
|
|
1000
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const*
|
|
1001
|
-
simsimd_distance_t*
|
|
1039
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1040
|
+
simsimd_distance_t *results) {
|
|
1002
1041
|
// Ideally the implementation would load 256 bits worth of vector data at a time,
|
|
1003
1042
|
// shuffle those within a register, split in halfs, and only then upcast.
|
|
1004
1043
|
// That way, we are stepping through 32x 16-bit vector components at a time, or 16 dimensions.
|
|
@@ -1021,8 +1060,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16
|
|
|
1021
1060
|
);
|
|
1022
1061
|
|
|
1023
1062
|
while (n >= 8) {
|
|
1024
|
-
__m256 a_vec = _mm256_cvtph_ps(
|
|
1025
|
-
__m256 b_vec = _mm256_cvtph_ps(
|
|
1063
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1064
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1026
1065
|
__m256 b_swapped_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1027
1066
|
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1028
1067
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_swapped_vec, ab_imag_vec);
|
|
@@ -1039,8 +1078,8 @@ SIMSIMD_PUBLIC void simsimd_dot_f16c_haswell(simsimd_f16_t const* a, simsimd_f16
|
|
|
1039
1078
|
results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
1040
1079
|
}
|
|
1041
1080
|
|
|
1042
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const*
|
|
1043
|
-
simsimd_distance_t*
|
|
1081
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1082
|
+
simsimd_distance_t *results) {
|
|
1044
1083
|
|
|
1045
1084
|
__m256 ab_real_vec = _mm256_setzero_ps();
|
|
1046
1085
|
__m256 ab_imag_vec = _mm256_setzero_ps();
|
|
@@ -1057,8 +1096,8 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
|
|
|
1057
1096
|
);
|
|
1058
1097
|
|
|
1059
1098
|
while (n >= 8) {
|
|
1060
|
-
__m256 a_vec = _mm256_cvtph_ps(
|
|
1061
|
-
__m256 b_vec = _mm256_cvtph_ps(
|
|
1099
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1100
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1062
1101
|
ab_real_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1063
1102
|
b_vec = _mm256_castsi256_ps(_mm256_shuffle_epi8(_mm256_castps_si256(b_vec), swap_adjacent_vec));
|
|
1064
1103
|
ab_imag_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
@@ -1075,38 +1114,79 @@ SIMSIMD_PUBLIC void simsimd_vdot_f16c_haswell(simsimd_f16_t const* a, simsimd_f1
|
|
|
1075
1114
|
results[1] += _simsimd_reduce_f32x8_haswell(ab_imag_vec);
|
|
1076
1115
|
}
|
|
1077
1116
|
|
|
1078
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const*
|
|
1079
|
-
simsimd_distance_t*
|
|
1117
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1118
|
+
simsimd_distance_t *result) {
|
|
1080
1119
|
|
|
1081
|
-
__m256i
|
|
1082
|
-
__m256i
|
|
1120
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1121
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1083
1122
|
|
|
1123
|
+
// AVX2 has no instructions for 8-bit signed integer dot-products,
|
|
1124
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1125
|
+
// So we need to normalize the first vector to its absolute value,
|
|
1126
|
+
// and shift the product sign into the second vector.
|
|
1127
|
+
//
|
|
1128
|
+
// __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
|
|
1129
|
+
// __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
|
|
1130
|
+
// __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
|
|
1131
|
+
//
|
|
1132
|
+
// The problem with this approach, however, is the `-128` value in the second vector.
|
|
1133
|
+
// Flipping it's sign will do nothing, and the result will be incorrect.
|
|
1134
|
+
// This can easily lead to noticeable numerical errors in the final result.
|
|
1084
1135
|
simsimd_size_t i = 0;
|
|
1085
1136
|
for (; i + 32 <= n; i += 32) {
|
|
1086
|
-
__m256i
|
|
1087
|
-
__m256i
|
|
1137
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1138
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1139
|
+
|
|
1140
|
+
// Upcast `int8` to `int16`
|
|
1141
|
+
__m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
|
|
1142
|
+
__m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
|
|
1143
|
+
__m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
|
|
1144
|
+
__m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
|
|
1145
|
+
|
|
1146
|
+
// Multiply and accumulate at int16 level, accumulate at `int32` level
|
|
1147
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1148
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1149
|
+
}
|
|
1088
1150
|
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1151
|
+
// Horizontal sum across the 256-bit register
|
|
1152
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1153
|
+
|
|
1154
|
+
// Take care of the tail:
|
|
1155
|
+
for (; i < n; ++i) ab += (int)(a[i]) * b[i];
|
|
1156
|
+
*result = ab;
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1160
|
+
simsimd_distance_t *result) {
|
|
1161
|
+
|
|
1162
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1163
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1164
|
+
__m256i const zeros_vec = _mm256_setzero_si256();
|
|
1165
|
+
|
|
1166
|
+
// AVX2 has no instructions for unsigned 8-bit integer dot-products,
|
|
1167
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1168
|
+
simsimd_size_t i = 0;
|
|
1169
|
+
for (; i + 32 <= n; i += 32) {
|
|
1170
|
+
__m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1171
|
+
__m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1172
|
+
|
|
1173
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1174
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1175
|
+
__m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
1176
|
+
__m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
1177
|
+
__m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
1178
|
+
__m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
1094
1179
|
|
|
1095
1180
|
// Multiply and accumulate at int16 level, accumulate at int32 level
|
|
1096
|
-
|
|
1097
|
-
|
|
1181
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1182
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1098
1183
|
}
|
|
1099
1184
|
|
|
1100
1185
|
// Horizontal sum across the 256-bit register
|
|
1101
|
-
|
|
1102
|
-
__m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
|
|
1103
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1104
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1186
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1105
1187
|
|
|
1106
1188
|
// Take care of the tail:
|
|
1107
|
-
|
|
1108
|
-
for (; i < n; ++i)
|
|
1109
|
-
ab += (int)(a[i]) * b[i];
|
|
1189
|
+
for (; i < n; ++i) ab += (int)(a[i]) * b[i];
|
|
1110
1190
|
*result = ab;
|
|
1111
1191
|
}
|
|
1112
1192
|
|
|
@@ -1115,7 +1195,23 @@ SIMSIMD_INTERNAL __m256 _simsimd_bf16x8_to_f32x8_haswell(__m128i a) {
|
|
|
1115
1195
|
return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a), 16));
|
|
1116
1196
|
}
|
|
1117
1197
|
|
|
1118
|
-
SIMSIMD_INTERNAL __m128i
|
|
1198
|
+
SIMSIMD_INTERNAL __m128i _simsimd_f32x8_to_bf16x8_haswell(__m256 a) {
|
|
1199
|
+
// Pack the 32-bit integers into 16-bit integers.
|
|
1200
|
+
// This is less trivial than unpacking: https://stackoverflow.com/a/77781241/2766161
|
|
1201
|
+
// The best approach is to shuffle within lanes first: https://stackoverflow.com/a/49723746/2766161
|
|
1202
|
+
// Our shuffling mask will drop the low 2-bytes from every 4-byte word.
|
|
1203
|
+
__m256i trunc_elements = _mm256_shuffle_epi8( //
|
|
1204
|
+
_mm256_castps_si256(a), //
|
|
1205
|
+
_mm256_set_epi8( //
|
|
1206
|
+
-1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 11, 10, 7, 6, 3, 2, //
|
|
1207
|
+
-1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 11, 10, 7, 6, 3, 2 //
|
|
1208
|
+
));
|
|
1209
|
+
__m256i ordered = _mm256_permute4x64_epi64(trunc_elements, 0x58);
|
|
1210
|
+
__m128i result = _mm256_castsi256_si128(ordered);
|
|
1211
|
+
return result;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
SIMSIMD_INTERNAL __m128i _simsimd_partial_load_bf16x8_haswell(simsimd_bf16_t const *a, simsimd_size_t n) {
|
|
1119
1215
|
// In case the software emulation for `bf16` scalars is enabled, the `simsimd_bf16_to_f32`
|
|
1120
1216
|
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
1121
1217
|
// loads and stores with vectorized math.
|
|
@@ -1124,15 +1220,13 @@ SIMSIMD_INTERNAL __m128i _simsimd_partial_load_bf16x8_haswell(simsimd_bf16_t con
|
|
|
1124
1220
|
simsimd_bf16_t scalars[8];
|
|
1125
1221
|
} result;
|
|
1126
1222
|
simsimd_size_t i = 0;
|
|
1127
|
-
for (; i < n; ++i)
|
|
1128
|
-
|
|
1129
|
-
for (; i < 8; ++i)
|
|
1130
|
-
result.scalars[i] = 0;
|
|
1223
|
+
for (; i < n; ++i) result.scalars[i] = a[i];
|
|
1224
|
+
for (; i < 8; ++i) result.scalars[i] = 0;
|
|
1131
1225
|
return result.vec;
|
|
1132
1226
|
}
|
|
1133
1227
|
|
|
1134
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const*
|
|
1135
|
-
simsimd_distance_t*
|
|
1228
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1229
|
+
simsimd_distance_t *result) {
|
|
1136
1230
|
__m128i a_vec, b_vec;
|
|
1137
1231
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
1138
1232
|
|
|
@@ -1141,14 +1235,14 @@ simsimd_dot_bf16_haswell_cycle:
|
|
|
1141
1235
|
a_vec = _simsimd_partial_load_bf16x8_haswell(a, n);
|
|
1142
1236
|
b_vec = _simsimd_partial_load_bf16x8_haswell(b, n);
|
|
1143
1237
|
n = 0;
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
|
|
1238
|
+
}
|
|
1239
|
+
else {
|
|
1240
|
+
a_vec = _mm_lddqu_si128((__m128i const *)a);
|
|
1241
|
+
b_vec = _mm_lddqu_si128((__m128i const *)b);
|
|
1147
1242
|
a += 8, b += 8, n -= 8;
|
|
1148
1243
|
}
|
|
1149
1244
|
ab_vec = _mm256_fmadd_ps(_simsimd_bf16x8_to_f32x8_haswell(a_vec), _simsimd_bf16x8_to_f32x8_haswell(b_vec), ab_vec);
|
|
1150
|
-
if (n)
|
|
1151
|
-
goto simsimd_dot_bf16_haswell_cycle;
|
|
1245
|
+
if (n) goto simsimd_dot_bf16_haswell_cycle;
|
|
1152
1246
|
|
|
1153
1247
|
*result = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1154
1248
|
}
|
|
@@ -1169,8 +1263,19 @@ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_reduce_f32x16_skylake(__m512 a) {
|
|
|
1169
1263
|
return _mm_cvtss_f32(_mm_hadd_ps(r, r));
|
|
1170
1264
|
}
|
|
1171
1265
|
|
|
1172
|
-
|
|
1173
|
-
|
|
1266
|
+
SIMSIMD_INTERNAL __m512 _simsimd_bf16x16_to_f32x16_skylake(__m256i a) {
|
|
1267
|
+
// Upcasting from `bf16` to `f32` is done by shifting the `bf16` values by 16 bits to the left, like:
|
|
1268
|
+
return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
SIMSIMD_INTERNAL __m256i _simsimd_f32x16_to_bf16x16_skylake(__m512 a) {
|
|
1272
|
+
// Add 2^15 and right shift 16 to do round-nearest
|
|
1273
|
+
__m512i x = _mm512_srli_epi32(_mm512_add_epi32(_mm512_castps_si512(a), _mm512_set1_epi32(1 << 15)), 16);
|
|
1274
|
+
return _mm512_cvtepi32_epi16(x);
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1278
|
+
simsimd_distance_t *result) {
|
|
1174
1279
|
__m512 ab_vec = _mm512_setzero();
|
|
1175
1280
|
__m512 a_vec, b_vec;
|
|
1176
1281
|
|
|
@@ -1180,20 +1285,20 @@ simsimd_dot_f32_skylake_cycle:
|
|
|
1180
1285
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1181
1286
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1182
1287
|
n = 0;
|
|
1183
|
-
}
|
|
1288
|
+
}
|
|
1289
|
+
else {
|
|
1184
1290
|
a_vec = _mm512_loadu_ps(a);
|
|
1185
1291
|
b_vec = _mm512_loadu_ps(b);
|
|
1186
1292
|
a += 16, b += 16, n -= 16;
|
|
1187
1293
|
}
|
|
1188
1294
|
ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1189
|
-
if (n)
|
|
1190
|
-
goto simsimd_dot_f32_skylake_cycle;
|
|
1295
|
+
if (n) goto simsimd_dot_f32_skylake_cycle;
|
|
1191
1296
|
|
|
1192
1297
|
*result = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1193
1298
|
}
|
|
1194
1299
|
|
|
1195
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const*
|
|
1196
|
-
simsimd_distance_t*
|
|
1300
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1301
|
+
simsimd_distance_t *result) {
|
|
1197
1302
|
__m512d ab_vec = _mm512_setzero_pd();
|
|
1198
1303
|
__m512d a_vec, b_vec;
|
|
1199
1304
|
|
|
@@ -1203,20 +1308,20 @@ simsimd_dot_f64_skylake_cycle:
|
|
|
1203
1308
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1204
1309
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1205
1310
|
n = 0;
|
|
1206
|
-
}
|
|
1311
|
+
}
|
|
1312
|
+
else {
|
|
1207
1313
|
a_vec = _mm512_loadu_pd(a);
|
|
1208
1314
|
b_vec = _mm512_loadu_pd(b);
|
|
1209
1315
|
a += 8, b += 8, n -= 8;
|
|
1210
1316
|
}
|
|
1211
1317
|
ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
|
|
1212
|
-
if (n)
|
|
1213
|
-
goto simsimd_dot_f64_skylake_cycle;
|
|
1318
|
+
if (n) goto simsimd_dot_f64_skylake_cycle;
|
|
1214
1319
|
|
|
1215
1320
|
*result = _mm512_reduce_add_pd(ab_vec);
|
|
1216
1321
|
}
|
|
1217
1322
|
|
|
1218
|
-
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const*
|
|
1219
|
-
simsimd_distance_t*
|
|
1323
|
+
SIMSIMD_PUBLIC void simsimd_dot_f32c_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1324
|
+
simsimd_distance_t *results) {
|
|
1220
1325
|
__m512 ab_real_vec = _mm512_setzero();
|
|
1221
1326
|
__m512 ab_imag_vec = _mm512_setzero();
|
|
1222
1327
|
__m512 a_vec;
|
|
@@ -1240,7 +1345,8 @@ simsimd_dot_f32c_skylake_cycle:
|
|
|
1240
1345
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1241
1346
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1242
1347
|
n = 0;
|
|
1243
|
-
}
|
|
1348
|
+
}
|
|
1349
|
+
else {
|
|
1244
1350
|
a_vec = _mm512_loadu_ps(a);
|
|
1245
1351
|
b_vec = _mm512_loadu_ps(b);
|
|
1246
1352
|
a += 16, b += 16, n -= 16;
|
|
@@ -1248,8 +1354,7 @@ simsimd_dot_f32c_skylake_cycle:
|
|
|
1248
1354
|
ab_real_vec = _mm512_fmadd_ps(b_vec, a_vec, ab_real_vec);
|
|
1249
1355
|
ab_imag_vec = _mm512_fmadd_ps(
|
|
1250
1356
|
_mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
|
|
1251
|
-
if (n)
|
|
1252
|
-
goto simsimd_dot_f32c_skylake_cycle;
|
|
1357
|
+
if (n) goto simsimd_dot_f32c_skylake_cycle;
|
|
1253
1358
|
|
|
1254
1359
|
// Flip the sign bit in every second scalar before accumulation:
|
|
1255
1360
|
ab_real_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_real_vec), sign_flip_vec));
|
|
@@ -1259,8 +1364,8 @@ simsimd_dot_f32c_skylake_cycle:
|
|
|
1259
1364
|
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1260
1365
|
}
|
|
1261
1366
|
|
|
1262
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const*
|
|
1263
|
-
simsimd_distance_t*
|
|
1367
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f32c_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1368
|
+
simsimd_distance_t *results) {
|
|
1264
1369
|
__m512 ab_real_vec = _mm512_setzero();
|
|
1265
1370
|
__m512 ab_imag_vec = _mm512_setzero();
|
|
1266
1371
|
__m512 a_vec;
|
|
@@ -1284,7 +1389,8 @@ simsimd_vdot_f32c_skylake_cycle:
|
|
|
1284
1389
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1285
1390
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1286
1391
|
n = 0;
|
|
1287
|
-
}
|
|
1392
|
+
}
|
|
1393
|
+
else {
|
|
1288
1394
|
a_vec = _mm512_loadu_ps(a);
|
|
1289
1395
|
b_vec = _mm512_loadu_ps(b);
|
|
1290
1396
|
a += 16, b += 16, n -= 16;
|
|
@@ -1292,8 +1398,7 @@ simsimd_vdot_f32c_skylake_cycle:
|
|
|
1292
1398
|
ab_real_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_real_vec);
|
|
1293
1399
|
b_vec = _mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(b_vec), swap_adjacent_vec));
|
|
1294
1400
|
ab_imag_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_imag_vec);
|
|
1295
|
-
if (n)
|
|
1296
|
-
goto simsimd_vdot_f32c_skylake_cycle;
|
|
1401
|
+
if (n) goto simsimd_vdot_f32c_skylake_cycle;
|
|
1297
1402
|
|
|
1298
1403
|
// Flip the sign bit in every second scalar before accumulation:
|
|
1299
1404
|
ab_imag_vec = _mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(ab_imag_vec), sign_flip_vec));
|
|
@@ -1303,8 +1408,8 @@ simsimd_vdot_f32c_skylake_cycle:
|
|
|
1303
1408
|
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1304
1409
|
}
|
|
1305
1410
|
|
|
1306
|
-
SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const*
|
|
1307
|
-
simsimd_distance_t*
|
|
1411
|
+
SIMSIMD_PUBLIC void simsimd_dot_f64c_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1412
|
+
simsimd_distance_t *results) {
|
|
1308
1413
|
__m512d ab_real_vec = _mm512_setzero_pd();
|
|
1309
1414
|
__m512d ab_imag_vec = _mm512_setzero_pd();
|
|
1310
1415
|
__m512d a_vec;
|
|
@@ -1331,7 +1436,8 @@ simsimd_dot_f64c_skylake_cycle:
|
|
|
1331
1436
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1332
1437
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1333
1438
|
n = 0;
|
|
1334
|
-
}
|
|
1439
|
+
}
|
|
1440
|
+
else {
|
|
1335
1441
|
a_vec = _mm512_loadu_pd(a);
|
|
1336
1442
|
b_vec = _mm512_loadu_pd(b);
|
|
1337
1443
|
a += 8, b += 8, n -= 8;
|
|
@@ -1339,8 +1445,7 @@ simsimd_dot_f64c_skylake_cycle:
|
|
|
1339
1445
|
ab_real_vec = _mm512_fmadd_pd(b_vec, a_vec, ab_real_vec);
|
|
1340
1446
|
ab_imag_vec = _mm512_fmadd_pd(
|
|
1341
1447
|
_mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec)), a_vec, ab_imag_vec);
|
|
1342
|
-
if (n)
|
|
1343
|
-
goto simsimd_dot_f64c_skylake_cycle;
|
|
1448
|
+
if (n) goto simsimd_dot_f64c_skylake_cycle;
|
|
1344
1449
|
|
|
1345
1450
|
// Flip the sign bit in every second scalar before accumulation:
|
|
1346
1451
|
ab_real_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_real_vec), sign_flip_vec));
|
|
@@ -1350,8 +1455,8 @@ simsimd_dot_f64c_skylake_cycle:
|
|
|
1350
1455
|
results[1] = _mm512_reduce_add_pd(ab_imag_vec);
|
|
1351
1456
|
}
|
|
1352
1457
|
|
|
1353
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const*
|
|
1354
|
-
simsimd_distance_t*
|
|
1458
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f64c_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1459
|
+
simsimd_distance_t *results) {
|
|
1355
1460
|
__m512d ab_real_vec = _mm512_setzero_pd();
|
|
1356
1461
|
__m512d ab_imag_vec = _mm512_setzero_pd();
|
|
1357
1462
|
__m512d a_vec;
|
|
@@ -1378,7 +1483,8 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1378
1483
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1379
1484
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1380
1485
|
n = 0;
|
|
1381
|
-
}
|
|
1486
|
+
}
|
|
1487
|
+
else {
|
|
1382
1488
|
a_vec = _mm512_loadu_pd(a);
|
|
1383
1489
|
b_vec = _mm512_loadu_pd(b);
|
|
1384
1490
|
a += 8, b += 8, n -= 8;
|
|
@@ -1386,8 +1492,7 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1386
1492
|
ab_real_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_real_vec);
|
|
1387
1493
|
b_vec = _mm512_castsi512_pd(_mm512_shuffle_epi8(_mm512_castpd_si512(b_vec), swap_adjacent_vec));
|
|
1388
1494
|
ab_imag_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_imag_vec);
|
|
1389
|
-
if (n)
|
|
1390
|
-
goto simsimd_vdot_f64c_skylake_cycle;
|
|
1495
|
+
if (n) goto simsimd_vdot_f64c_skylake_cycle;
|
|
1391
1496
|
|
|
1392
1497
|
// Flip the sign bit in every second scalar before accumulation:
|
|
1393
1498
|
ab_imag_vec = _mm512_castsi512_pd(_mm512_xor_si512(_mm512_castpd_si512(ab_imag_vec), sign_flip_vec));
|
|
@@ -1404,11 +1509,11 @@ simsimd_vdot_f64c_skylake_cycle:
|
|
|
1404
1509
|
#if SIMSIMD_TARGET_GENOA
|
|
1405
1510
|
#pragma GCC push_options
|
|
1406
1511
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1407
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))),
|
|
1512
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1408
1513
|
apply_to = function)
|
|
1409
1514
|
|
|
1410
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const*
|
|
1411
|
-
simsimd_distance_t*
|
|
1515
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1516
|
+
simsimd_distance_t *result) {
|
|
1412
1517
|
__m512 ab_vec = _mm512_setzero_ps();
|
|
1413
1518
|
__m512i a_i16_vec, b_i16_vec;
|
|
1414
1519
|
|
|
@@ -1418,20 +1523,20 @@ simsimd_dot_bf16_genoa_cycle:
|
|
|
1418
1523
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1419
1524
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1420
1525
|
n = 0;
|
|
1421
|
-
}
|
|
1526
|
+
}
|
|
1527
|
+
else {
|
|
1422
1528
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1423
1529
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1424
1530
|
a += 32, b += 32, n -= 32;
|
|
1425
1531
|
}
|
|
1426
1532
|
ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
|
|
1427
|
-
if (n)
|
|
1428
|
-
goto simsimd_dot_bf16_genoa_cycle;
|
|
1533
|
+
if (n) goto simsimd_dot_bf16_genoa_cycle;
|
|
1429
1534
|
|
|
1430
1535
|
*result = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1431
1536
|
}
|
|
1432
1537
|
|
|
1433
|
-
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const*
|
|
1434
|
-
simsimd_distance_t*
|
|
1538
|
+
SIMSIMD_PUBLIC void simsimd_dot_bf16c_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1539
|
+
simsimd_distance_t *results) {
|
|
1435
1540
|
__m512 ab_real_vec = _mm512_setzero_ps();
|
|
1436
1541
|
__m512 ab_imag_vec = _mm512_setzero_ps();
|
|
1437
1542
|
__m512i a_vec;
|
|
@@ -1456,7 +1561,8 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1456
1561
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1457
1562
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1458
1563
|
n = 0;
|
|
1459
|
-
}
|
|
1564
|
+
}
|
|
1565
|
+
else {
|
|
1460
1566
|
a_vec = _mm512_loadu_epi16(a);
|
|
1461
1567
|
b_vec = _mm512_loadu_epi16(b);
|
|
1462
1568
|
a += 32, b += 32, n -= 32;
|
|
@@ -1464,16 +1570,15 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1464
1570
|
ab_real_vec = _mm512_dpbf16_ps(ab_real_vec, (__m512bh)(_mm512_xor_si512(b_vec, sign_flip_vec)), (__m512bh)(a_vec));
|
|
1465
1571
|
ab_imag_vec =
|
|
1466
1572
|
_mm512_dpbf16_ps(ab_imag_vec, (__m512bh)(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)), (__m512bh)(a_vec));
|
|
1467
|
-
if (n)
|
|
1468
|
-
goto simsimd_dot_bf16c_genoa_cycle;
|
|
1573
|
+
if (n) goto simsimd_dot_bf16c_genoa_cycle;
|
|
1469
1574
|
|
|
1470
1575
|
// Reduce horizontal sums:
|
|
1471
1576
|
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
1472
1577
|
results[1] = _simsimd_reduce_f32x16_skylake(ab_imag_vec);
|
|
1473
1578
|
}
|
|
1474
1579
|
|
|
1475
|
-
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const*
|
|
1476
|
-
simsimd_distance_t*
|
|
1580
|
+
SIMSIMD_PUBLIC void simsimd_vdot_bf16c_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1581
|
+
simsimd_distance_t *results) {
|
|
1477
1582
|
__m512 ab_real_vec = _mm512_setzero_ps();
|
|
1478
1583
|
__m512 ab_imag_vec = _mm512_setzero_ps();
|
|
1479
1584
|
__m512i a_vec;
|
|
@@ -1498,7 +1603,8 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1498
1603
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1499
1604
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1500
1605
|
n = 0;
|
|
1501
|
-
}
|
|
1606
|
+
}
|
|
1607
|
+
else {
|
|
1502
1608
|
a_vec = _mm512_loadu_epi16(a);
|
|
1503
1609
|
b_vec = _mm512_loadu_epi16(b);
|
|
1504
1610
|
a += 32, b += 32, n -= 32;
|
|
@@ -1507,8 +1613,7 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1507
1613
|
a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
|
|
1508
1614
|
b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
|
|
1509
1615
|
ab_imag_vec = _mm512_dpbf16_ps(ab_imag_vec, (__m512bh)(a_vec), (__m512bh)(b_vec));
|
|
1510
|
-
if (n)
|
|
1511
|
-
goto simsimd_dot_bf16c_genoa_cycle;
|
|
1616
|
+
if (n) goto simsimd_dot_bf16c_genoa_cycle;
|
|
1512
1617
|
|
|
1513
1618
|
// Reduce horizontal sums:
|
|
1514
1619
|
results[0] = _simsimd_reduce_f32x16_skylake(ab_real_vec);
|
|
@@ -1522,11 +1627,11 @@ simsimd_dot_bf16c_genoa_cycle:
|
|
|
1522
1627
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1523
1628
|
#pragma GCC push_options
|
|
1524
1629
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
|
|
1525
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))),
|
|
1630
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
|
|
1526
1631
|
apply_to = function)
|
|
1527
1632
|
|
|
1528
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const*
|
|
1529
|
-
simsimd_distance_t*
|
|
1633
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1634
|
+
simsimd_distance_t *result) {
|
|
1530
1635
|
__m512h ab_vec = _mm512_setzero_ph();
|
|
1531
1636
|
__m512i a_i16_vec, b_i16_vec;
|
|
1532
1637
|
|
|
@@ -1536,20 +1641,20 @@ simsimd_dot_f16_sapphire_cycle:
|
|
|
1536
1641
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1537
1642
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1538
1643
|
n = 0;
|
|
1539
|
-
}
|
|
1644
|
+
}
|
|
1645
|
+
else {
|
|
1540
1646
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1541
1647
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1542
1648
|
a += 32, b += 32, n -= 32;
|
|
1543
1649
|
}
|
|
1544
1650
|
ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
|
|
1545
|
-
if (n)
|
|
1546
|
-
goto simsimd_dot_f16_sapphire_cycle;
|
|
1651
|
+
if (n) goto simsimd_dot_f16_sapphire_cycle;
|
|
1547
1652
|
|
|
1548
1653
|
*result = _mm512_reduce_add_ph(ab_vec);
|
|
1549
1654
|
}
|
|
1550
1655
|
|
|
1551
|
-
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const*
|
|
1552
|
-
simsimd_distance_t*
|
|
1656
|
+
SIMSIMD_PUBLIC void simsimd_dot_f16c_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1657
|
+
simsimd_distance_t *results) {
|
|
1553
1658
|
__m512h ab_real_vec = _mm512_setzero_ph();
|
|
1554
1659
|
__m512h ab_imag_vec = _mm512_setzero_ph();
|
|
1555
1660
|
__m512i a_vec;
|
|
@@ -1574,7 +1679,8 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1574
1679
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1575
1680
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1576
1681
|
n = 0;
|
|
1577
|
-
}
|
|
1682
|
+
}
|
|
1683
|
+
else {
|
|
1578
1684
|
a_vec = _mm512_loadu_epi16(a);
|
|
1579
1685
|
b_vec = _mm512_loadu_epi16(b);
|
|
1580
1686
|
a += 32, b += 32, n -= 32;
|
|
@@ -1584,8 +1690,7 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1584
1690
|
_mm512_castsi512_ph(a_vec), ab_real_vec);
|
|
1585
1691
|
ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(_mm512_shuffle_epi8(b_vec, swap_adjacent_vec)),
|
|
1586
1692
|
_mm512_castsi512_ph(a_vec), ab_imag_vec);
|
|
1587
|
-
if (n)
|
|
1588
|
-
goto simsimd_dot_f16c_sapphire_cycle;
|
|
1693
|
+
if (n) goto simsimd_dot_f16c_sapphire_cycle;
|
|
1589
1694
|
|
|
1590
1695
|
// Reduce horizontal sums:
|
|
1591
1696
|
// TODO: Optimize this with tree-like reductions
|
|
@@ -1593,8 +1698,8 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1593
1698
|
results[1] = _mm512_reduce_add_ph(ab_imag_vec);
|
|
1594
1699
|
}
|
|
1595
1700
|
|
|
1596
|
-
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const*
|
|
1597
|
-
simsimd_distance_t*
|
|
1701
|
+
SIMSIMD_PUBLIC void simsimd_vdot_f16c_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1702
|
+
simsimd_distance_t *results) {
|
|
1598
1703
|
__m512h ab_real_vec = _mm512_setzero_ph();
|
|
1599
1704
|
__m512h ab_imag_vec = _mm512_setzero_ph();
|
|
1600
1705
|
__m512i a_vec;
|
|
@@ -1619,7 +1724,8 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1619
1724
|
a_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1620
1725
|
b_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1621
1726
|
n = 0;
|
|
1622
|
-
}
|
|
1727
|
+
}
|
|
1728
|
+
else {
|
|
1623
1729
|
a_vec = _mm512_loadu_epi16(a);
|
|
1624
1730
|
b_vec = _mm512_loadu_epi16(b);
|
|
1625
1731
|
a += 32, b += 32, n -= 32;
|
|
@@ -1628,8 +1734,7 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1628
1734
|
a_vec = _mm512_xor_si512(a_vec, sign_flip_vec);
|
|
1629
1735
|
b_vec = _mm512_shuffle_epi8(b_vec, swap_adjacent_vec);
|
|
1630
1736
|
ab_imag_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_imag_vec);
|
|
1631
|
-
if (n)
|
|
1632
|
-
goto simsimd_dot_f16c_sapphire_cycle;
|
|
1737
|
+
if (n) goto simsimd_dot_f16c_sapphire_cycle;
|
|
1633
1738
|
|
|
1634
1739
|
// Reduce horizontal sums:
|
|
1635
1740
|
results[0] = _mm512_reduce_add_ph(ab_real_vec);
|
|
@@ -1643,23 +1748,24 @@ simsimd_dot_f16c_sapphire_cycle:
|
|
|
1643
1748
|
#if SIMSIMD_TARGET_ICE
|
|
1644
1749
|
#pragma GCC push_options
|
|
1645
1750
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1646
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))),
|
|
1751
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
|
|
1647
1752
|
apply_to = function)
|
|
1648
1753
|
|
|
1649
|
-
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const*
|
|
1650
|
-
simsimd_distance_t*
|
|
1651
|
-
__m512i
|
|
1652
|
-
__m512i
|
|
1754
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1755
|
+
simsimd_distance_t *result) {
|
|
1756
|
+
__m512i ab_i32_vec = _mm512_setzero_si512();
|
|
1757
|
+
__m512i a_i16_vec, b_i16_vec;
|
|
1653
1758
|
|
|
1654
1759
|
simsimd_dot_i8_ice_cycle:
|
|
1655
1760
|
if (n < 32) {
|
|
1656
1761
|
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1657
|
-
|
|
1658
|
-
|
|
1762
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1763
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
1659
1764
|
n = 0;
|
|
1660
|
-
}
|
|
1661
|
-
|
|
1662
|
-
|
|
1765
|
+
}
|
|
1766
|
+
else {
|
|
1767
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
|
|
1768
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
|
|
1663
1769
|
a += 32, b += 32, n -= 32;
|
|
1664
1770
|
}
|
|
1665
1771
|
// Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
|
|
@@ -1667,17 +1773,85 @@ simsimd_dot_i8_ice_cycle:
|
|
|
1667
1773
|
// Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
|
|
1668
1774
|
// So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
|
|
1669
1775
|
// to 16-bit beforehand.
|
|
1670
|
-
|
|
1671
|
-
if (n)
|
|
1672
|
-
|
|
1776
|
+
ab_i32_vec = _mm512_dpwssd_epi32(ab_i32_vec, a_i16_vec, b_i16_vec);
|
|
1777
|
+
if (n) goto simsimd_dot_i8_ice_cycle;
|
|
1778
|
+
|
|
1779
|
+
*result = _mm512_reduce_add_epi32(ab_i32_vec);
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
SIMSIMD_PUBLIC void simsimd_dot_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1783
|
+
simsimd_distance_t *result) {
|
|
1784
|
+
__m512i ab_i32_low_vec = _mm512_setzero_si512();
|
|
1785
|
+
__m512i ab_i32_high_vec = _mm512_setzero_si512();
|
|
1786
|
+
__m512i const zeros_vec = _mm512_setzero_si512();
|
|
1787
|
+
__m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
|
|
1788
|
+
__m512i a_u8_vec, b_u8_vec;
|
|
1789
|
+
|
|
1790
|
+
simsimd_dot_u8_ice_cycle:
|
|
1791
|
+
if (n < 64) {
|
|
1792
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
1793
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1794
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1795
|
+
n = 0;
|
|
1796
|
+
}
|
|
1797
|
+
else {
|
|
1798
|
+
a_u8_vec = _mm512_loadu_si512(a);
|
|
1799
|
+
b_u8_vec = _mm512_loadu_si512(b);
|
|
1800
|
+
a += 64, b += 64, n -= 64;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1804
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1805
|
+
a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
1806
|
+
a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
1807
|
+
b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
1808
|
+
b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
1809
|
+
// Unfortunately we can't use the `_mm512_dpbusd_epi32` intrinsics here either,
|
|
1810
|
+
// as it's asymmetric with respect to the sign of the input arguments:
|
|
1811
|
+
// Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
|
|
1812
|
+
// So we have to use the `_mm512_dpwssd_epi32` intrinsics instead, upcasting
|
|
1813
|
+
// to 16-bit beforehand.
|
|
1814
|
+
ab_i32_low_vec = _mm512_dpwssd_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
|
|
1815
|
+
ab_i32_high_vec = _mm512_dpwssd_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
|
|
1816
|
+
if (n) goto simsimd_dot_u8_ice_cycle;
|
|
1673
1817
|
|
|
1674
|
-
*result = _mm512_reduce_add_epi32(
|
|
1818
|
+
*result = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1675
1819
|
}
|
|
1676
1820
|
|
|
1677
1821
|
#pragma clang attribute pop
|
|
1678
1822
|
#pragma GCC pop_options
|
|
1679
1823
|
#endif // SIMSIMD_TARGET_ICE
|
|
1680
|
-
|
|
1824
|
+
|
|
1825
|
+
#if SIMSIMD_TARGET_SIERRA
|
|
1826
|
+
#pragma GCC push_options
|
|
1827
|
+
#pragma GCC target("avx2", "bmi2", "avx2vnni")
|
|
1828
|
+
#pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
|
|
1829
|
+
|
|
1830
|
+
SIMSIMD_PUBLIC void simsimd_dot_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1831
|
+
simsimd_distance_t *result) {
|
|
1832
|
+
|
|
1833
|
+
__m256i ab_i32_vec = _mm256_setzero_si256();
|
|
1834
|
+
|
|
1835
|
+
simsimd_size_t i = 0;
|
|
1836
|
+
for (; i + 32 <= n; i += 32) {
|
|
1837
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1838
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1839
|
+
ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// Further reduce to a single sum for each vector
|
|
1843
|
+
int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
|
|
1844
|
+
|
|
1845
|
+
// Take care of the tail:
|
|
1846
|
+
for (; i < n; ++i) ab += (int)(a[i]) * b[i];
|
|
1847
|
+
|
|
1848
|
+
*result = ab;
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
#pragma clang attribute pop
|
|
1852
|
+
#pragma GCC pop_options
|
|
1853
|
+
#endif // SIMSIMD_TARGET_SIERRA
|
|
1854
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
1681
1855
|
|
|
1682
1856
|
#ifdef __cplusplus
|
|
1683
1857
|
}
|