usearch 2.15.3 → 2.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +1 -1
  2. package/cpp/test.cpp +16 -0
  3. package/include/usearch/index.hpp +21 -5
  4. package/include/usearch/index_dense.hpp +6 -1
  5. package/include/usearch/index_plugins.hpp +345 -8
  6. package/javascript/lib.cpp +17 -5
  7. package/javascript/usearch.test.js +190 -21
  8. package/package.json +1 -1
  9. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  10. package/prebuilds/linux-arm64/usearch.node +0 -0
  11. package/prebuilds/linux-x64/usearch.node +0 -0
  12. package/simsimd/.git-blame-ignore-revs +2 -0
  13. package/simsimd/CONTRIBUTING.md +71 -23
  14. package/simsimd/Cargo.lock +1 -1
  15. package/simsimd/Cargo.toml +4 -4
  16. package/simsimd/MANIFEST.in +2 -1
  17. package/simsimd/Package.swift +15 -3
  18. package/simsimd/README.md +104 -35
  19. package/simsimd/binding.gyp +6 -2
  20. package/simsimd/build.rs +2 -0
  21. package/simsimd/c/lib.c +180 -99
  22. package/simsimd/include/simsimd/binary.h +311 -102
  23. package/simsimd/include/simsimd/curved.h +134 -135
  24. package/simsimd/include/simsimd/dot.h +448 -274
  25. package/simsimd/include/simsimd/elementwise.h +1437 -0
  26. package/simsimd/include/simsimd/geospatial.h +2 -2
  27. package/simsimd/include/simsimd/mesh.h +2 -2
  28. package/simsimd/include/simsimd/probability.h +88 -88
  29. package/simsimd/include/simsimd/simsimd.h +1157 -772
  30. package/simsimd/include/simsimd/sparse.h +768 -147
  31. package/simsimd/include/simsimd/spatial.h +955 -376
  32. package/simsimd/include/simsimd/types.h +145 -71
  33. package/simsimd/javascript/lib.c +1 -2
  34. package/simsimd/package.json +4 -4
  35. package/simsimd/rust/lib.rs +14 -1
  36. package/simsimd/scripts/bench.cxx +1070 -0
  37. package/simsimd/scripts/bench.ipynb +187 -0
  38. package/simsimd/{javascript → scripts}/bench.js +1 -1
  39. package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
  40. package/simsimd/scripts/bench_vectors.py +700 -0
  41. package/simsimd/scripts/bench_vectors_live.py +213 -0
  42. package/simsimd/scripts/test.c +155 -0
  43. package/simsimd/{javascript → scripts}/test.mjs +2 -2
  44. package/simsimd/scripts/test.py +1410 -0
  45. package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
  46. package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
  47. /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
  48. /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
@@ -0,0 +1,1437 @@
1
+ /**
2
+ * @file elementwise.h
3
+ * @brief SIMD-accelerated mixed-precision element-wise operations.
4
+ * @author Ash Vardanian
5
+ * @date October 16, 2024
6
+ *
7
+ * Contains following element-wise operations:
8
+ * - WSum or Weighted-Sum: R[i] = Alpha * A[i] + Beta * B[i]
9
+ * - FMA or Fused-Multiply-Add: R[i] = Alpha * A[i] * B[i] + Beta * C[i]
10
+ *
11
+ * This tiny set of operations if enough to implement a wide range of algorithms.
12
+ * To scale a vector by a scalar, just call WSum with $Beta$ = 0.
13
+ * To sum two vectors, just call WSum with $Alpha$ = $Beta$ = 1.
14
+ * To average two vectors, just call WSum with $Alpha$ = $Beta$ = 0.5.
15
+ * To multiply vectors element-wise, just call FMA with $Beta$ = 0.
16
+ *
17
+ * For datatypes:
18
+ * - 64-bit IEEE floating point numbers
19
+ * - 32-bit IEEE floating point numbers
20
+ * - 16-bit IEEE floating point numbers
21
+ * - 16-bit brain floating point numbers
22
+ * - 8-bit unsigned integers
23
+ * - 8-bit signed integers
24
+ *
25
+ * For hardware architectures:
26
+ * - Arm: NEON, SVE
27
+ * - x86: Haswell, Ice Lake, Skylake, Genoa, Sapphire
28
+ *
29
+ * We use `f16` for `i8` and `u8` arithmetic. This is because Arm received `f16` support earlier than `bf16`.
30
+ * For example, Apple M1 has `f16` support and `bf16` was only added in M2. On the other hand, on paper,
31
+ * AMD Genoa has `bf16` support, and `f16` is only available on Intel Sapphire Rapids and newer.
32
+ * Sadly, the SIMD support for `bf16` is limited to mixed-precision dot-products, which makes it useless here.
33
+ *
34
+ * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
35
+ * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
36
+ */
37
+ #ifndef SIMSIMD_FMA_H
38
+ #define SIMSIMD_FMA_H
39
+
40
+ #include "types.h"
41
+
42
+ #ifdef __cplusplus
43
+ extern "C" {
44
+ #endif
45
+
46
+ /* Serial backends for all numeric types.
47
+ * By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
48
+ * For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
49
+ */
50
+ SIMSIMD_PUBLIC void simsimd_wsum_f64_serial( //
51
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
52
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
53
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_serial( //
54
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
55
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
56
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_serial( //
57
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
58
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
59
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_serial( //
60
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
61
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
62
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_serial( //
63
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
64
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
65
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_serial( //
66
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
67
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
68
+
69
+ SIMSIMD_PUBLIC void simsimd_fma_f64_serial( //
70
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
71
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
72
+ SIMSIMD_PUBLIC void simsimd_fma_f32_serial( //
73
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
74
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
75
+ SIMSIMD_PUBLIC void simsimd_fma_f16_serial( //
76
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
77
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
78
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_serial( //
79
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
80
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
81
+ SIMSIMD_PUBLIC void simsimd_fma_i8_serial( //
82
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
83
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
84
+ SIMSIMD_PUBLIC void simsimd_fma_u8_serial( //
85
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
86
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
87
+
88
+ #define SIMSIMD_MAKE_WSUM(name, input_type, accumulator_type, load_and_convert, convert_and_store) \
89
+ SIMSIMD_PUBLIC void simsimd_wsum_##input_type##_##name( \
90
+ simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_size_t n, \
91
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_##input_type##_t *result) { \
92
+ for (simsimd_size_t i = 0; i != n; ++i) { \
93
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
94
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
95
+ simsimd_##accumulator_type##_t ai_scaled = (simsimd_##accumulator_type##_t)(ai * alpha); \
96
+ simsimd_##accumulator_type##_t bi_scaled = (simsimd_##accumulator_type##_t)(bi * beta); \
97
+ simsimd_##accumulator_type##_t sum = ai_scaled + bi_scaled; \
98
+ convert_and_store(sum, result + i); \
99
+ } \
100
+ }
101
+
102
+ #define SIMSIMD_MAKE_FMA(name, input_type, accumulator_type, load_and_convert, convert_and_store) \
103
+ SIMSIMD_PUBLIC void simsimd_fma_##input_type##_##name( \
104
+ simsimd_##input_type##_t const *a, simsimd_##input_type##_t const *b, simsimd_##input_type##_t const *c, \
105
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_##input_type##_t *result) { \
106
+ for (simsimd_size_t i = 0; i != n; ++i) { \
107
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
108
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
109
+ simsimd_##accumulator_type##_t ci = load_and_convert(c + i); \
110
+ simsimd_##accumulator_type##_t abi_scaled = (simsimd_##accumulator_type##_t)(ai * bi * alpha); \
111
+ simsimd_##accumulator_type##_t ci_scaled = (simsimd_##accumulator_type##_t)(ci * beta); \
112
+ simsimd_##accumulator_type##_t sum = abi_scaled + ci_scaled; \
113
+ convert_and_store(sum, result + i); \
114
+ } \
115
+ }
116
+
117
+ SIMSIMD_MAKE_WSUM(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f64_serial
118
+ SIMSIMD_MAKE_WSUM(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f32_serial
119
+ SIMSIMD_MAKE_WSUM(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_wsum_f16_serial
120
+ SIMSIMD_MAKE_WSUM(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_wsum_bf16_serial
121
+ SIMSIMD_MAKE_WSUM(serial, i8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_I8) // simsimd_wsum_i8_serial
122
+ SIMSIMD_MAKE_WSUM(serial, u8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_U8) // simsimd_wsum_u8_serial
123
+
124
+ SIMSIMD_MAKE_WSUM(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_wsum_f32_accurate
125
+ SIMSIMD_MAKE_WSUM(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_wsum_f16_accurate
126
+ SIMSIMD_MAKE_WSUM(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_wsum_bf16_accurate
127
+ SIMSIMD_MAKE_WSUM(accurate, i8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_I8) // simsimd_wsum_i8_accurate
128
+ SIMSIMD_MAKE_WSUM(accurate, u8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_U8) // simsimd_wsum_u8_accurate
129
+
130
+ SIMSIMD_MAKE_FMA(serial, f64, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f64_serial
131
+ SIMSIMD_MAKE_FMA(serial, f32, f32, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f32_serial
132
+ SIMSIMD_MAKE_FMA(serial, f16, f32, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_fma_f16_serial
133
+ SIMSIMD_MAKE_FMA(serial, bf16, f32, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_fma_bf16_serial
134
+ SIMSIMD_MAKE_FMA(serial, i8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_I8) // simsimd_fma_i8_serial
135
+ SIMSIMD_MAKE_FMA(serial, u8, f32, SIMSIMD_DEREFERENCE, SIMSIMD_F32_TO_U8) // simsimd_fma_u8_serial
136
+
137
+ SIMSIMD_MAKE_FMA(accurate, f32, f64, SIMSIMD_DEREFERENCE, SIMSIMD_EXPORT) // simsimd_fma_f32_accurate
138
+ SIMSIMD_MAKE_FMA(accurate, f16, f64, SIMSIMD_F16_TO_F32, SIMSIMD_F32_TO_F16) // simsimd_fma_f16_accurate
139
+ SIMSIMD_MAKE_FMA(accurate, bf16, f64, SIMSIMD_BF16_TO_F32, SIMSIMD_F32_TO_BF16) // simsimd_fma_bf16_accurate
140
+ SIMSIMD_MAKE_FMA(accurate, i8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_I8) // simsimd_fma_i8_accurate
141
+ SIMSIMD_MAKE_FMA(accurate, u8, f64, SIMSIMD_DEREFERENCE, SIMSIMD_F64_TO_U8) // simsimd_fma_u8_accurate
142
+
143
+ /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
144
+ * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
145
+ * server CPUs produced before 2023.
146
+ */
147
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
148
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
149
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
150
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
151
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
152
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
153
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
154
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
155
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
156
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
157
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
158
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
159
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
160
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
161
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
162
+
163
+ SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
164
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
165
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
166
+ SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
167
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
168
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
169
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( //
170
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
171
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
172
+ SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
173
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, //
174
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
175
+ SIMSIMD_PUBLIC void simsimd_fma_i8_neon( //
176
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, //
177
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
178
+
179
+ /* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
180
+ * First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
181
+ * Practically all modern x86 CPUs support AVX2, FMA, and F16C, making it a perfect baseline for SIMD algorithms.
182
+ * On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
183
+ * properly vectorized by recent compilers.
184
+ */
185
+ SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
186
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
187
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
188
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
189
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
190
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
191
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
192
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
193
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
194
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
195
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
196
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
197
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
198
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
199
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
200
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
201
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
202
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
203
+ SIMSIMD_PUBLIC void simsimd_fma_f64_haswell( //
204
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
205
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
206
+ SIMSIMD_PUBLIC void simsimd_fma_f32_haswell( //
207
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
208
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
209
+ SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
210
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
211
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
212
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( //
213
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
214
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
215
+ SIMSIMD_PUBLIC void simsimd_fma_i8_haswell( //
216
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
217
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
218
+ SIMSIMD_PUBLIC void simsimd_fma_u8_haswell( //
219
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
220
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
221
+
222
+ /* SIMD-powered backends for various generations of AVX512 CPUs.
223
+ * Unlike the distance metrics, the SIMD implementation of FMA and WSum benefits from aligned stores.
224
+ * Assuming the size of ZMM register matches the width of the cache line, we skip the unaligned head
225
+ * and tail of the output buffer, and only use aligned stores in the main loop.
226
+ */
227
+ SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
228
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
229
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
230
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
231
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
232
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
233
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
234
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
235
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
236
+
237
+ SIMSIMD_PUBLIC void simsimd_fma_f64_skylake( //
238
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
239
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result);
240
+ SIMSIMD_PUBLIC void simsimd_fma_f32_skylake( //
241
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
242
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result);
243
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_skylake( //
244
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
245
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result);
246
+
247
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
248
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
249
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
250
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
251
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
252
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
253
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
254
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
255
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
256
+
257
+ SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire( //
258
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
259
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result);
260
+ SIMSIMD_PUBLIC void simsimd_fma_i8_sapphire( //
261
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
262
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result);
263
+ SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire( //
264
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
265
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result);
266
+
267
+ #if _SIMSIMD_TARGET_X86
268
+ #if SIMSIMD_TARGET_HASWELL
269
+ #pragma GCC push_options
270
+ #pragma GCC target("avx2", "f16c", "fma")
271
+ #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
272
+
273
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_haswell( //
274
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
275
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
276
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
277
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
278
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
279
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
280
+
281
+ // The main loop:
282
+ simsimd_size_t i = 0;
283
+ for (; i + 8 <= n; i += 8) {
284
+ __m256 a_vec = _mm256_loadu_ps(a + i);
285
+ __m256 b_vec = _mm256_loadu_ps(b + i);
286
+ __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
287
+ __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
288
+ __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
289
+ _mm256_storeu_ps(result + i, sum_vec);
290
+ }
291
+
292
+ // The tail:
293
+ for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
294
+ }
295
+
296
+ SIMSIMD_PUBLIC void simsimd_wsum_f64_haswell( //
297
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
298
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
299
+ __m256d alpha_vec = _mm256_set1_pd(alpha);
300
+ __m256d beta_vec = _mm256_set1_pd(beta);
301
+
302
+ // The main loop:
303
+ simsimd_size_t i = 0;
304
+ for (; i + 4 <= n; i += 4) {
305
+ __m256d a_vec = _mm256_loadu_pd(a + i);
306
+ __m256d b_vec = _mm256_loadu_pd(b + i);
307
+ __m256d a_scaled = _mm256_mul_pd(a_vec, alpha_vec);
308
+ __m256d b_scaled = _mm256_mul_pd(b_vec, beta_vec);
309
+ __m256d sum_vec = _mm256_add_pd(a_scaled, b_scaled);
310
+ _mm256_storeu_pd(result + i, sum_vec);
311
+ }
312
+
313
+ // The tail:
314
+ for (; i < n; ++i) result[i] = alpha * a[i] + beta * b[i];
315
+ }
316
+
317
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_haswell( //
318
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
319
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
320
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
321
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
322
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
323
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
324
+
325
+ // The main loop:
326
+ simsimd_size_t i = 0;
327
+ for (; i + 8 <= n; i += 8) {
328
+ __m128i a_f16 = _mm_loadu_si128((__m128i const *)(a + i));
329
+ __m128i b_f16 = _mm_loadu_si128((__m128i const *)(b + i));
330
+ __m256 a_vec = _mm256_cvtph_ps(a_f16);
331
+ __m256 b_vec = _mm256_cvtph_ps(b_f16);
332
+ __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
333
+ __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
334
+ __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
335
+ __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
336
+ _mm_storeu_si128((__m128i *)(result + i), sum_f16);
337
+ }
338
+
339
+ // The tail:
340
+ for (; i < n; ++i) {
341
+ simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
342
+ simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
343
+ simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
344
+ SIMSIMD_F32_TO_F16(sum, result + i);
345
+ }
346
+ }
347
+
348
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_haswell( //
349
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
350
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
351
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
352
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
353
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
354
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
355
+
356
+ // The main loop:
357
+ simsimd_size_t i = 0;
358
+ for (; i + 8 <= n; i += 8) {
359
+ __m128i a_bf16 = _mm_loadu_si128((__m128i const *)(a + i));
360
+ __m128i b_bf16 = _mm_loadu_si128((__m128i const *)(b + i));
361
+ __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
362
+ __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
363
+ __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
364
+ __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
365
+ __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
366
+ __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
367
+ _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
368
+ }
369
+
370
+ // The tail:
371
+ for (; i < n; ++i) {
372
+ simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
373
+ simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
374
+ simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
375
+ SIMSIMD_F32_TO_BF16(sum, result + i);
376
+ }
377
+ }
378
+
379
+ SIMSIMD_PUBLIC void simsimd_fma_f32_haswell( //
380
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
381
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
382
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
383
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
384
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
385
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
386
+
387
+ // The main loop:
388
+ simsimd_size_t i = 0;
389
+ for (; i + 8 <= n; i += 8) {
390
+ __m256 a_vec = _mm256_loadu_ps(a + i);
391
+ __m256 b_vec = _mm256_loadu_ps(b + i);
392
+ __m256 c_vec = _mm256_loadu_ps(c + i);
393
+ __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
394
+ __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
395
+ __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
396
+ __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
397
+ _mm256_storeu_ps(result + i, sum_vec);
398
+ }
399
+
400
+ // The tail:
401
+ for (; i < n; ++i) result[i] = alpha_f32 * a[i] * b[i] + beta_f32 * c[i];
402
+ }
403
+
404
+ SIMSIMD_PUBLIC void simsimd_fma_f64_haswell( //
405
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, //
406
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
407
+ __m256d alpha_vec = _mm256_set1_pd(alpha);
408
+ __m256d beta_vec = _mm256_set1_pd(beta);
409
+
410
+ // The main loop:
411
+ simsimd_size_t i = 0;
412
+ for (; i + 4 <= n; i += 4) {
413
+ __m256d a_vec = _mm256_loadu_pd(a + i);
414
+ __m256d b_vec = _mm256_loadu_pd(b + i);
415
+ __m256d c_vec = _mm256_loadu_pd(c + i);
416
+ __m256d ab_vec = _mm256_mul_pd(a_vec, b_vec);
417
+ __m256d ab_scaled_vec = _mm256_mul_pd(ab_vec, alpha_vec);
418
+ __m256d c_scaled_vec = _mm256_mul_pd(c_vec, beta_vec);
419
+ __m256d sum_vec = _mm256_add_pd(ab_scaled_vec, c_scaled_vec);
420
+ _mm256_storeu_pd(result + i, sum_vec);
421
+ }
422
+
423
+ // The tail:
424
+ for (; i < n; ++i) result[i] = alpha * a[i] * b[i] + beta * c[i];
425
+ }
426
+
427
+ SIMSIMD_PUBLIC void simsimd_fma_f16_haswell( //
428
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
429
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
430
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
431
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
432
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
433
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
434
+
435
+ // The main loop:
436
+ simsimd_size_t i = 0;
437
+ for (; i + 8 <= n; i += 8) {
438
+ __m128i a_f16 = _mm_loadu_si128((__m128i const *)(a + i));
439
+ __m128i b_f16 = _mm_loadu_si128((__m128i const *)(b + i));
440
+ __m128i c_f16 = _mm_loadu_si128((__m128i const *)(c + i));
441
+ __m256 a_vec = _mm256_cvtph_ps(a_f16);
442
+ __m256 b_vec = _mm256_cvtph_ps(b_f16);
443
+ __m256 c_vec = _mm256_cvtph_ps(c_f16);
444
+ __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
445
+ __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
446
+ __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
447
+ __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
448
+ __m128i sum_f16 = _mm256_cvtps_ph(sum_vec, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
449
+ _mm_storeu_si128((__m128i *)(result + i), sum_f16);
450
+ }
451
+
452
+ // The tail:
453
+ for (; i < n; ++i) {
454
+ simsimd_f32_t ai = SIMSIMD_F16_TO_F32(a + i);
455
+ simsimd_f32_t bi = SIMSIMD_F16_TO_F32(b + i);
456
+ simsimd_f32_t ci = SIMSIMD_F16_TO_F32(c + i);
457
+ simsimd_f32_t sum = alpha * ai * bi + beta * ci;
458
+ SIMSIMD_F32_TO_F16(sum, result + i);
459
+ }
460
+ }
461
+
462
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_haswell( //
463
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
464
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
465
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
466
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
467
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
468
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
469
+
470
+ // The main loop:
471
+ simsimd_size_t i = 0;
472
+ for (; i + 8 <= n; i += 8) {
473
+ __m128i a_bf16 = _mm_loadu_si128((__m128i const *)(a + i));
474
+ __m128i b_bf16 = _mm_loadu_si128((__m128i const *)(b + i));
475
+ __m128i c_bf16 = _mm_loadu_si128((__m128i const *)(c + i));
476
+ __m256 a_vec = _simsimd_bf16x8_to_f32x8_haswell(a_bf16);
477
+ __m256 b_vec = _simsimd_bf16x8_to_f32x8_haswell(b_bf16);
478
+ __m256 c_vec = _simsimd_bf16x8_to_f32x8_haswell(c_bf16);
479
+ __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
480
+ __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
481
+ __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
482
+ __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
483
+ __m128i sum_bf16 = _simsimd_f32x8_to_bf16x8_haswell(sum_vec);
484
+ _mm_storeu_si128((__m128i *)(result + i), sum_bf16);
485
+ }
486
+
487
+ // The tail:
488
+ for (; i < n; ++i) {
489
+ simsimd_f32_t ai = SIMSIMD_BF16_TO_F32(a + i);
490
+ simsimd_f32_t bi = SIMSIMD_BF16_TO_F32(b + i);
491
+ simsimd_f32_t ci = SIMSIMD_BF16_TO_F32(c + i);
492
+ simsimd_f32_t sum = alpha * ai * bi + beta * ci;
493
+ SIMSIMD_F32_TO_BF16(sum, result + i);
494
+ }
495
+ }
496
+
497
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_haswell( //
498
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
499
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
500
+
501
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
502
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
503
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
504
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
505
+ int sum_i32s[8], a_i32s[8], b_i32s[8];
506
+
507
+ // The main loop:
508
+ simsimd_size_t i = 0;
509
+ for (; i + 8 <= n; i += 8) {
510
+ //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
511
+ //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
512
+ a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
513
+ a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
514
+ b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
515
+ b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
516
+ //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
517
+ //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
518
+ __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
519
+ __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
520
+ // The normal part.
521
+ __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
522
+ __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
523
+ __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
524
+ // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
525
+ __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
526
+ sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
527
+ sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
528
+ // Export into a serial buffer.
529
+ _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
530
+ result[i + 0] = (simsimd_i8_t)sum_i32s[0];
531
+ result[i + 1] = (simsimd_i8_t)sum_i32s[1];
532
+ result[i + 2] = (simsimd_i8_t)sum_i32s[2];
533
+ result[i + 3] = (simsimd_i8_t)sum_i32s[3];
534
+ result[i + 4] = (simsimd_i8_t)sum_i32s[4];
535
+ result[i + 5] = (simsimd_i8_t)sum_i32s[5];
536
+ result[i + 6] = (simsimd_i8_t)sum_i32s[6];
537
+ result[i + 7] = (simsimd_i8_t)sum_i32s[7];
538
+ }
539
+
540
+ // The tail:
541
+ for (; i < n; ++i) {
542
+ simsimd_f32_t ai = a[i], bi = b[i];
543
+ simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
544
+ SIMSIMD_F32_TO_I8(sum, result + i);
545
+ }
546
+ }
547
+
548
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_haswell( //
549
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
550
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
551
+
552
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
553
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
554
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
555
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
556
+ int sum_i32s[8], a_i32s[8], b_i32s[8];
557
+
558
+ // The main loop:
559
+ simsimd_size_t i = 0;
560
+ for (; i + 8 <= n; i += 8) {
561
+ //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
562
+ //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
563
+ a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
564
+ a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
565
+ b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
566
+ b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
567
+ //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
568
+ //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
569
+ __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
570
+ __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
571
+ // The normal part.
572
+ __m256 a_scaled = _mm256_mul_ps(a_vec, alpha_vec);
573
+ __m256 b_scaled = _mm256_mul_ps(b_vec, beta_vec);
574
+ __m256 sum_vec = _mm256_add_ps(a_scaled, b_scaled);
575
+ // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
576
+ __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
577
+ sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
578
+ sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
579
+ // Export into a serial buffer.
580
+ _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
581
+ result[i + 0] = (simsimd_u8_t)sum_i32s[0];
582
+ result[i + 1] = (simsimd_u8_t)sum_i32s[1];
583
+ result[i + 2] = (simsimd_u8_t)sum_i32s[2];
584
+ result[i + 3] = (simsimd_u8_t)sum_i32s[3];
585
+ result[i + 4] = (simsimd_u8_t)sum_i32s[4];
586
+ result[i + 5] = (simsimd_u8_t)sum_i32s[5];
587
+ result[i + 6] = (simsimd_u8_t)sum_i32s[6];
588
+ result[i + 7] = (simsimd_u8_t)sum_i32s[7];
589
+ }
590
+
591
+ // The tail:
592
+ for (; i < n; ++i) {
593
+ simsimd_f32_t ai = a[i], bi = b[i];
594
+ simsimd_f32_t sum = alpha_f32 * ai + beta_f32 * bi;
595
+ SIMSIMD_F32_TO_U8(sum, result + i);
596
+ }
597
+ }
598
+
599
+ SIMSIMD_PUBLIC void simsimd_fma_i8_haswell( //
600
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
601
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
602
+
603
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
604
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
605
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
606
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
607
+ int sum_i32s[8], a_i32s[8], b_i32s[8], c_i32s[8];
608
+
609
+ // The main loop:
610
+ simsimd_size_t i = 0;
611
+ for (; i + 8 <= n; i += 8) {
612
+ //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
613
+ //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
614
+ a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
615
+ a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
616
+ b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
617
+ b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
618
+ c_i32s[0] = c[i + 0], c_i32s[1] = c[i + 1], c_i32s[2] = c[i + 2], c_i32s[3] = c[i + 3], //
619
+ c_i32s[4] = c[i + 4], c_i32s[5] = c[i + 5], c_i32s[6] = c[i + 6], c_i32s[7] = c[i + 7];
620
+ //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
621
+ //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
622
+ __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
623
+ __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
624
+ __m256 c_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)c_i32s));
625
+ // The normal part.
626
+ __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
627
+ __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
628
+ __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
629
+ __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
630
+ // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
631
+ __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
632
+ sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(-128));
633
+ sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(127));
634
+ // Export into a serial buffer.
635
+ _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
636
+ result[i + 0] = (simsimd_i8_t)sum_i32s[0];
637
+ result[i + 1] = (simsimd_i8_t)sum_i32s[1];
638
+ result[i + 2] = (simsimd_i8_t)sum_i32s[2];
639
+ result[i + 3] = (simsimd_i8_t)sum_i32s[3];
640
+ result[i + 4] = (simsimd_i8_t)sum_i32s[4];
641
+ result[i + 5] = (simsimd_i8_t)sum_i32s[5];
642
+ result[i + 6] = (simsimd_i8_t)sum_i32s[6];
643
+ result[i + 7] = (simsimd_i8_t)sum_i32s[7];
644
+ }
645
+
646
+ // The tail:
647
+ for (; i < n; ++i) {
648
+ simsimd_f32_t ai = a[i], bi = b[i], ci = c[i];
649
+ simsimd_f32_t sum = alpha_f32 * ai * bi + beta_f32 * ci;
650
+ SIMSIMD_F32_TO_I8(sum, result + i);
651
+ }
652
+ }
653
+
654
+ SIMSIMD_PUBLIC void simsimd_fma_u8_haswell( //
655
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
656
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
657
+
658
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
659
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
660
+ __m256 alpha_vec = _mm256_set1_ps(alpha_f32);
661
+ __m256 beta_vec = _mm256_set1_ps(beta_f32);
662
+ int sum_i32s[8], a_i32s[8], b_i32s[8], c_i32s[8];
663
+
664
+ // The main loop:
665
+ simsimd_size_t i = 0;
666
+ for (; i + 8 <= n; i += 8) {
667
+ //? Handling loads and stores with SIMD is tricky. Not because of upcasting, but the
668
+ //? downcasting at the end of the loop. In AVX2 it's a drag! Keep it for another day.
669
+ a_i32s[0] = a[i + 0], a_i32s[1] = a[i + 1], a_i32s[2] = a[i + 2], a_i32s[3] = a[i + 3], //
670
+ a_i32s[4] = a[i + 4], a_i32s[5] = a[i + 5], a_i32s[6] = a[i + 6], a_i32s[7] = a[i + 7];
671
+ b_i32s[0] = b[i + 0], b_i32s[1] = b[i + 1], b_i32s[2] = b[i + 2], b_i32s[3] = b[i + 3], //
672
+ b_i32s[4] = b[i + 4], b_i32s[5] = b[i + 5], b_i32s[6] = b[i + 6], b_i32s[7] = b[i + 7];
673
+ c_i32s[0] = c[i + 0], c_i32s[1] = c[i + 1], c_i32s[2] = c[i + 2], c_i32s[3] = c[i + 3], //
674
+ c_i32s[4] = c[i + 4], c_i32s[5] = c[i + 5], c_i32s[6] = c[i + 6], c_i32s[7] = c[i + 7];
675
+ //! This can be done at least 50% faster if we convert 8-bit integers to floats instead
676
+ //! of relying on the slow `_mm256_cvtepi32_ps` instruction.
677
+ __m256 a_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)a_i32s));
678
+ __m256 b_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)b_i32s));
679
+ __m256 c_vec = _mm256_cvtepi32_ps(_mm256_lddqu_si256((__m256i *)c_i32s));
680
+ // The normal part.
681
+ __m256 ab_vec = _mm256_mul_ps(a_vec, b_vec);
682
+ __m256 ab_scaled_vec = _mm256_mul_ps(ab_vec, alpha_vec);
683
+ __m256 c_scaled_vec = _mm256_mul_ps(c_vec, beta_vec);
684
+ __m256 sum_vec = _mm256_add_ps(ab_scaled_vec, c_scaled_vec);
685
+ // Instead of serial calls to expensive `SIMSIMD_F32_TO_U8`, convert and clip with SIMD.
686
+ __m256i sum_i32_vec = _mm256_cvtps_epi32(sum_vec);
687
+ sum_i32_vec = _mm256_max_epi32(sum_i32_vec, _mm256_set1_epi32(0));
688
+ sum_i32_vec = _mm256_min_epi32(sum_i32_vec, _mm256_set1_epi32(255));
689
+ // Export into a serial buffer.
690
+ _mm256_storeu_si256((__m256i *)sum_i32s, sum_i32_vec);
691
+ result[i + 0] = (simsimd_u8_t)sum_i32s[0];
692
+ result[i + 1] = (simsimd_u8_t)sum_i32s[1];
693
+ result[i + 2] = (simsimd_u8_t)sum_i32s[2];
694
+ result[i + 3] = (simsimd_u8_t)sum_i32s[3];
695
+ result[i + 4] = (simsimd_u8_t)sum_i32s[4];
696
+ result[i + 5] = (simsimd_u8_t)sum_i32s[5];
697
+ result[i + 6] = (simsimd_u8_t)sum_i32s[6];
698
+ result[i + 7] = (simsimd_u8_t)sum_i32s[7];
699
+ }
700
+
701
+ // The tail:
702
+ for (; i < n; ++i) {
703
+ simsimd_f32_t ai = a[i], bi = b[i], ci = c[i];
704
+ simsimd_f32_t sum = alpha_f32 * ai * bi + beta_f32 * ci;
705
+ SIMSIMD_F32_TO_U8(sum, result + i);
706
+ }
707
+ }
708
+
709
+ #pragma clang attribute pop
710
+ #pragma GCC pop_options
711
+ #endif // SIMSIMD_TARGET_HASWELL
712
+
713
+ #if SIMSIMD_TARGET_SKYLAKE
714
+ #pragma GCC push_options
715
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "bmi2")
716
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,avx512bw,bmi2"))), apply_to = function)
717
+
718
+ SIMSIMD_PUBLIC void simsimd_wsum_f64_skylake( //
719
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n, //
720
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
721
+ __m512d alpha_vec = _mm512_set1_pd(alpha);
722
+ __m512d beta_vec = _mm512_set1_pd(beta);
723
+ __m512d a_vec, b_vec, a_scaled_vec, sum_vec;
724
+ __mmask8 mask = 0xFF;
725
+
726
+ simsimd_wsum_f64_skylake_cycle:
727
+ if (n < 8) {
728
+ mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
729
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
730
+ b_vec = _mm512_maskz_loadu_pd(mask, b);
731
+ n = 0;
732
+ }
733
+ else {
734
+ a_vec = _mm512_loadu_pd(a);
735
+ b_vec = _mm512_loadu_pd(b);
736
+ a += 8, b += 8, n -= 8;
737
+ }
738
+ a_scaled_vec = _mm512_mul_pd(a_vec, alpha_vec);
739
+ sum_vec = _mm512_fmadd_pd(b_vec, beta_vec, a_scaled_vec);
740
+ _mm512_mask_storeu_pd(result, mask, sum_vec);
741
+ result += 8;
742
+ if (n) goto simsimd_wsum_f64_skylake_cycle;
743
+ }
744
+
745
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_skylake( //
746
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
747
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
748
+ __m512 alpha_vec = _mm512_set1_ps(alpha);
749
+ __m512 beta_vec = _mm512_set1_ps(beta);
750
+ __m512 a_vec, b_vec, a_scaled_vec, sum_vec;
751
+ __mmask16 mask = 0xFFFF;
752
+
753
+ simsimd_wsum_f32_skylake_cycle:
754
+ if (n < 16) {
755
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
756
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
757
+ b_vec = _mm512_maskz_loadu_ps(mask, b);
758
+ n = 0;
759
+ }
760
+ else {
761
+ a_vec = _mm512_loadu_ps(a);
762
+ b_vec = _mm512_loadu_ps(b);
763
+ a += 16, b += 16, n -= 16;
764
+ }
765
+ a_scaled_vec = _mm512_mul_ps(a_vec, alpha_vec);
766
+ sum_vec = _mm512_fmadd_ps(b_vec, beta_vec, a_scaled_vec);
767
+ _mm512_mask_storeu_ps(result, mask, sum_vec);
768
+ result += 16;
769
+ if (n) goto simsimd_wsum_f32_skylake_cycle;
770
+ }
771
+
772
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_skylake( //
773
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
774
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
775
+ __m512 alpha_vec = _mm512_set1_ps(alpha);
776
+ __m512 beta_vec = _mm512_set1_ps(beta);
777
+ __m256i a_bf16_vec, b_bf16_vec, sum_bf16_vec;
778
+ __m512 a_vec, b_vec, a_scaled_vec, sum_vec;
779
+ __mmask16 mask = 0xFFFF;
780
+
781
+ simsimd_wsum_bf16_skylake_cycle:
782
+ if (n < 16) {
783
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
784
+ a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
785
+ b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
786
+ n = 0;
787
+ }
788
+ else {
789
+ a_bf16_vec = _mm256_loadu_epi16(a);
790
+ b_bf16_vec = _mm256_loadu_epi16(b);
791
+ a += 16, b += 16, n -= 16;
792
+ }
793
+ a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
794
+ b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
795
+ a_scaled_vec = _mm512_mul_ps(a_vec, alpha_vec);
796
+ sum_vec = _mm512_fmadd_ps(b_vec, beta_vec, a_scaled_vec);
797
+ sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
798
+ _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
799
+ result += 16;
800
+ if (n) goto simsimd_wsum_bf16_skylake_cycle;
801
+ }
802
+
803
+ SIMSIMD_PUBLIC void simsimd_fma_f64_skylake( //
804
+ simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_f64_t const *c, simsimd_size_t n, //
805
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f64_t *result) {
806
+ __m512d alpha_vec = _mm512_set1_pd(alpha);
807
+ __m512d beta_vec = _mm512_set1_pd(beta);
808
+ __m512d a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
809
+ __mmask8 mask = 0xFF;
810
+
811
+ simsimd_fma_f64_skylake_cycle:
812
+ if (n < 8) {
813
+ mask = (__mmask8)_bzhi_u32(0xFFFFFFFF, n);
814
+ a_vec = _mm512_maskz_loadu_pd(mask, a);
815
+ b_vec = _mm512_maskz_loadu_pd(mask, b);
816
+ c_vec = _mm512_maskz_loadu_pd(mask, c);
817
+ n = 0;
818
+ }
819
+ else {
820
+ a_vec = _mm512_loadu_pd(a);
821
+ b_vec = _mm512_loadu_pd(b);
822
+ c_vec = _mm512_loadu_pd(c);
823
+ a += 8, b += 8, c += 8, n -= 8;
824
+ }
825
+ ab_vec = _mm512_mul_pd(a_vec, b_vec);
826
+ ab_scaled_vec = _mm512_mul_pd(ab_vec, alpha_vec);
827
+ sum_vec = _mm512_fmadd_pd(c_vec, beta_vec, ab_scaled_vec);
828
+ _mm512_mask_storeu_pd(result, mask, sum_vec);
829
+ result += 8;
830
+ if (n) goto simsimd_fma_f64_skylake_cycle;
831
+ }
832
+
833
+ SIMSIMD_PUBLIC void simsimd_fma_f32_skylake( //
834
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, simsimd_size_t n, //
835
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
836
+ __m512 alpha_vec = _mm512_set1_ps(alpha);
837
+ __m512 beta_vec = _mm512_set1_ps(beta);
838
+ __m512 a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
839
+ __mmask16 mask = 0xFFFF;
840
+
841
+ simsimd_fma_f32_skylake_cycle:
842
+ if (n < 16) {
843
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
844
+ a_vec = _mm512_maskz_loadu_ps(mask, a);
845
+ b_vec = _mm512_maskz_loadu_ps(mask, b);
846
+ c_vec = _mm512_maskz_loadu_ps(mask, c);
847
+ n = 0;
848
+ }
849
+ else {
850
+ a_vec = _mm512_loadu_ps(a);
851
+ b_vec = _mm512_loadu_ps(b);
852
+ c_vec = _mm512_loadu_ps(c);
853
+ a += 16, b += 16, c += 16, n -= 16;
854
+ }
855
+ ab_vec = _mm512_mul_ps(a_vec, b_vec);
856
+ ab_scaled_vec = _mm512_mul_ps(ab_vec, alpha_vec);
857
+ sum_vec = _mm512_fmadd_ps(c_vec, beta_vec, ab_scaled_vec);
858
+ _mm512_mask_storeu_ps(result, mask, sum_vec);
859
+ result += 16;
860
+ if (n) goto simsimd_fma_f32_skylake_cycle;
861
+ }
862
+
863
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_skylake( //
864
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, simsimd_size_t n, //
865
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
866
+ __m512 alpha_vec = _mm512_set1_ps(alpha);
867
+ __m512 beta_vec = _mm512_set1_ps(beta);
868
+ __m256i a_bf16_vec, b_bf16_vec, c_bf16_vec, sum_bf16_vec;
869
+ __m512 a_vec, b_vec, c_vec, ab_vec, ab_scaled_vec, sum_vec;
870
+ __mmask16 mask = 0xFFFF;
871
+
872
+ simsimd_fma_bf16_skylake_cycle:
873
+ if (n < 16) {
874
+ mask = (__mmask16)_bzhi_u32(0xFFFFFFFF, n);
875
+ a_bf16_vec = _mm256_maskz_loadu_epi16(mask, a);
876
+ b_bf16_vec = _mm256_maskz_loadu_epi16(mask, b);
877
+ c_bf16_vec = _mm256_maskz_loadu_epi16(mask, c);
878
+ n = 0;
879
+ }
880
+ else {
881
+ a_bf16_vec = _mm256_loadu_epi16(a);
882
+ b_bf16_vec = _mm256_loadu_epi16(b);
883
+ c_bf16_vec = _mm256_loadu_epi16(c);
884
+ a += 16, b += 16, c += 16, n -= 16;
885
+ }
886
+ a_vec = _simsimd_bf16x16_to_f32x16_skylake(a_bf16_vec);
887
+ b_vec = _simsimd_bf16x16_to_f32x16_skylake(b_bf16_vec);
888
+ c_vec = _simsimd_bf16x16_to_f32x16_skylake(c_bf16_vec);
889
+ ab_vec = _mm512_mul_ps(a_vec, b_vec);
890
+ ab_scaled_vec = _mm512_mul_ps(ab_vec, alpha_vec);
891
+ sum_vec = _mm512_fmadd_ps(c_vec, beta_vec, ab_scaled_vec);
892
+ sum_bf16_vec = _simsimd_f32x16_to_bf16x16_skylake(sum_vec);
893
+ _mm256_mask_storeu_epi16(result, mask, sum_bf16_vec);
894
+ result += 16;
895
+ if (n) goto simsimd_fma_bf16_skylake_cycle;
896
+ }
897
+
898
+ #pragma clang attribute pop
899
+ #pragma GCC pop_options
900
+ #endif // SIMSIMD_TARGET_SKYLAKE
901
+
902
+ #if SIMSIMD_TARGET_SAPPHIRE
903
+ #pragma GCC push_options
904
+ #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512fp16")
905
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512fp16"))), \
906
+ apply_to = function)
907
+
908
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_sapphire( //
909
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
910
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
911
+
912
+ __mmask32 mask = 0xFFFFFFFF;
913
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
914
+ __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
915
+ __m512h a_f16_vec, b_f16_vec, c_f16_vec;
916
+ __m512h a_scaled_f16_vec, sum_f16_vec;
917
+
918
+ simsimd_wsum_f16_sapphire_cycle:
919
+ if (n < 32) {
920
+ mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
921
+ a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
922
+ b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
923
+ n = 0;
924
+ }
925
+ else {
926
+ a_f16_vec = _mm512_loadu_ph(a);
927
+ b_f16_vec = _mm512_loadu_ph(b);
928
+ a += 32, b += 32, n -= 32;
929
+ }
930
+ a_scaled_f16_vec = _mm512_mul_ph(a_f16_vec, alpha_vec);
931
+ sum_f16_vec = _mm512_fmadd_ph(b_f16_vec, beta_vec, a_scaled_f16_vec);
932
+ _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
933
+ result += 32;
934
+ if (n) goto simsimd_wsum_f16_sapphire_cycle;
935
+ }
936
+
937
+ SIMSIMD_PUBLIC void simsimd_fma_f16_sapphire( //
938
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, simsimd_size_t n, //
939
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
940
+
941
+ __mmask32 mask = 0xFFFFFFFF;
942
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
943
+ __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
944
+ __m512h a_f16_vec, b_f16_vec, c_f16_vec;
945
+ __m512h ab_f16_vec, ab_scaled_f16_vec, sum_f16_vec;
946
+
947
+ simsimd_fma_f16_sapphire_cycle:
948
+ if (n < 32) {
949
+ mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
950
+ a_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a));
951
+ b_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b));
952
+ c_f16_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, c));
953
+ n = 0;
954
+ }
955
+ else {
956
+ a_f16_vec = _mm512_loadu_ph(a);
957
+ b_f16_vec = _mm512_loadu_ph(b);
958
+ c_f16_vec = _mm512_loadu_ph(c);
959
+ a += 32, b += 32, c += 32, n -= 32;
960
+ }
961
+ ab_f16_vec = _mm512_mul_ph(a_f16_vec, b_f16_vec);
962
+ ab_scaled_f16_vec = _mm512_mul_ph(ab_f16_vec, alpha_vec);
963
+ sum_f16_vec = _mm512_fmadd_ph(c_f16_vec, beta_vec, ab_scaled_f16_vec);
964
+ _mm512_mask_storeu_epi16(result, mask, _mm512_castph_si512(sum_f16_vec));
965
+ result += 32;
966
+ if (n) goto simsimd_fma_f16_sapphire_cycle;
967
+ }
968
+
969
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_sapphire( //
970
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
971
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
972
+
973
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
974
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
975
+ __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
976
+ __m512i a_u8_vec, b_u8_vec, c_u8_vec, sum_u8_vec;
977
+ __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
978
+ __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
979
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
980
+
981
+ simsimd_wsum_u8_sapphire_cycle:
982
+ if (n < 64) {
983
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
984
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
985
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
986
+ n = 0;
987
+ }
988
+ else {
989
+ a_u8_vec = _mm512_loadu_epi8(a);
990
+ b_u8_vec = _mm512_loadu_epi8(b);
991
+ a += 64, b += 64, n -= 64;
992
+ }
993
+ // Upcast:
994
+ a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
995
+ a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
996
+ b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8_vec, _mm512_setzero_si512()));
997
+ b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8_vec, _mm512_setzero_si512()));
998
+ // Scale:
999
+ a_scaled_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1000
+ a_scaled_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1001
+ // Add:
1002
+ sum_f16_low_vec = _mm512_fmadd_ph(b_f16_low_vec, beta_vec, a_scaled_f16_low_vec);
1003
+ sum_f16_high_vec = _mm512_fmadd_ph(b_f16_high_vec, beta_vec, a_scaled_f16_high_vec);
1004
+ // Downcast:
1005
+ sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1006
+ sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1007
+ sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
1008
+ _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1009
+ result += 64;
1010
+ if (n) goto simsimd_wsum_u8_sapphire_cycle;
1011
+ }
1012
+
1013
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_sapphire( //
1014
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
1015
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1016
+
1017
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFFull;
1018
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1019
+ __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
1020
+ __m512i a_i8_vec, b_i8_vec, c_i8_vec, sum_i8_vec;
1021
+ __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1022
+ __m512h a_scaled_f16_low_vec, a_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1023
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
1024
+
1025
+ simsimd_wsum_i8_sapphire_cycle:
1026
+ if (n < 64) {
1027
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFFull, n);
1028
+ a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1029
+ b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
1030
+ n = 0;
1031
+ }
1032
+ else {
1033
+ a_i8_vec = _mm512_loadu_epi8(a);
1034
+ b_i8_vec = _mm512_loadu_epi8(b);
1035
+ a += 64, b += 64, n -= 64;
1036
+ }
1037
+ // Upcast:
1038
+ a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
1039
+ a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
1040
+ b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1041
+ b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1042
+ // Scale:
1043
+ a_scaled_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, alpha_vec);
1044
+ a_scaled_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, alpha_vec);
1045
+ // Add:
1046
+ sum_f16_low_vec = _mm512_fmadd_ph(b_f16_low_vec, beta_vec, a_scaled_f16_low_vec);
1047
+ sum_f16_high_vec = _mm512_fmadd_ph(b_f16_high_vec, beta_vec, a_scaled_f16_high_vec);
1048
+ // Downcast:
1049
+ sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1050
+ sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1051
+ sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
1052
+ _mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
1053
+ _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1054
+ result += 64;
1055
+ if (n) goto simsimd_wsum_i8_sapphire_cycle;
1056
+ }
1057
+
1058
+ SIMSIMD_PUBLIC void simsimd_fma_i8_sapphire( //
1059
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, simsimd_size_t n, //
1060
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1061
+
1062
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFF;
1063
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1064
+ __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
1065
+ __m512i a_i8_vec, b_i8_vec, c_i8_vec, sum_i8_vec;
1066
+ __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1067
+ __m512h c_f16_low_vec, c_f16_high_vec, ab_f16_low_vec, ab_f16_high_vec;
1068
+ __m512h ab_scaled_f16_low_vec, ab_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1069
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
1070
+
1071
+ simsimd_fma_i8_sapphire_cycle:
1072
+ if (n < 64) {
1073
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1074
+ a_i8_vec = _mm512_maskz_loadu_epi8(mask, a);
1075
+ b_i8_vec = _mm512_maskz_loadu_epi8(mask, b);
1076
+ c_i8_vec = _mm512_maskz_loadu_epi8(mask, c);
1077
+ n = 0;
1078
+ }
1079
+ else {
1080
+ a_i8_vec = _mm512_loadu_epi8(a);
1081
+ b_i8_vec = _mm512_loadu_epi8(b);
1082
+ c_i8_vec = _mm512_loadu_epi8(c);
1083
+ a += 64, b += 64, c += 64, n -= 64;
1084
+ }
1085
+ // Upcast:
1086
+ a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)));
1087
+ a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)));
1088
+ b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1089
+ b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1090
+ c_f16_low_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_castsi512_si256(c_i8_vec)));
1091
+ c_f16_high_vec = _mm512_cvtepi16_ph(_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(c_i8_vec, 1)));
1092
+ // Multiply:
1093
+ ab_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, b_f16_low_vec);
1094
+ ab_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, b_f16_high_vec);
1095
+ // Scale:
1096
+ ab_scaled_f16_low_vec = _mm512_mul_ph(ab_f16_low_vec, alpha_vec);
1097
+ ab_scaled_f16_high_vec = _mm512_mul_ph(ab_f16_high_vec, alpha_vec);
1098
+ // Add:
1099
+ sum_f16_low_vec = _mm512_fmadd_ph(c_f16_low_vec, beta_vec, ab_scaled_f16_low_vec);
1100
+ sum_f16_high_vec = _mm512_fmadd_ph(c_f16_high_vec, beta_vec, ab_scaled_f16_high_vec);
1101
+ // Downcast:
1102
+ sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1103
+ sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1104
+ sum_i8_vec = _mm512_inserti64x4(_mm512_castsi256_si512(_mm512_cvtsepi16_epi8(sum_i16_low_vec)),
1105
+ _mm512_cvtsepi16_epi8(sum_i16_high_vec), 1);
1106
+ _mm512_mask_storeu_epi8(result, mask, sum_i8_vec);
1107
+ result += 64;
1108
+ if (n) goto simsimd_fma_i8_sapphire_cycle;
1109
+ }
1110
+
1111
+ SIMSIMD_PUBLIC void simsimd_fma_u8_sapphire( //
1112
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, simsimd_size_t n, //
1113
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
1114
+
1115
+ __mmask64 mask = 0xFFFFFFFFFFFFFFFF;
1116
+ __m512h alpha_vec = _mm512_set1_ph((_Float16)alpha);
1117
+ __m512h beta_vec = _mm512_set1_ph((_Float16)beta);
1118
+ __m512i a_u8_vec, b_u8_vec, c_u8_vec, sum_u8_vec;
1119
+ __m512h a_f16_low_vec, a_f16_high_vec, b_f16_low_vec, b_f16_high_vec;
1120
+ __m512h c_f16_low_vec, c_f16_high_vec, ab_f16_low_vec, ab_f16_high_vec;
1121
+ __m512h ab_scaled_f16_low_vec, ab_scaled_f16_high_vec, sum_f16_low_vec, sum_f16_high_vec;
1122
+ __m512i sum_i16_low_vec, sum_i16_high_vec;
1123
+
1124
+ simsimd_fma_u8_sapphire_cycle:
1125
+ if (n < 64) {
1126
+ mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1127
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1128
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1129
+ c_u8_vec = _mm512_maskz_loadu_epi8(mask, c);
1130
+ n = 0;
1131
+ }
1132
+ else {
1133
+ a_u8_vec = _mm512_loadu_epi8(a);
1134
+ b_u8_vec = _mm512_loadu_epi8(b);
1135
+ c_u8_vec = _mm512_loadu_epi8(c);
1136
+ a += 64, b += 64, c += 64, n -= 64;
1137
+ }
1138
+ // Upcast:
1139
+ a_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(a_u8_vec, _mm512_setzero_si512()));
1140
+ a_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(a_u8_vec, _mm512_setzero_si512()));
1141
+ b_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(b_u8_vec, _mm512_setzero_si512()));
1142
+ b_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(b_u8_vec, _mm512_setzero_si512()));
1143
+ c_f16_low_vec = _mm512_cvtepi16_ph(_mm512_unpacklo_epi8(c_u8_vec, _mm512_setzero_si512()));
1144
+ c_f16_high_vec = _mm512_cvtepi16_ph(_mm512_unpackhi_epi8(c_u8_vec, _mm512_setzero_si512()));
1145
+ // Multiply:
1146
+ ab_f16_low_vec = _mm512_mul_ph(a_f16_low_vec, b_f16_low_vec);
1147
+ ab_f16_high_vec = _mm512_mul_ph(a_f16_high_vec, b_f16_high_vec);
1148
+ // Scale:
1149
+ ab_scaled_f16_low_vec = _mm512_mul_ph(ab_f16_low_vec, alpha_vec);
1150
+ ab_scaled_f16_high_vec = _mm512_mul_ph(ab_f16_high_vec, alpha_vec);
1151
+ // Add:
1152
+ sum_f16_low_vec = _mm512_fmadd_ph(c_f16_low_vec, beta_vec, ab_scaled_f16_low_vec);
1153
+ sum_f16_high_vec = _mm512_fmadd_ph(c_f16_high_vec, beta_vec, ab_scaled_f16_high_vec);
1154
+ // Downcast:
1155
+ sum_i16_low_vec = _mm512_cvtph_epi16(sum_f16_low_vec);
1156
+ sum_i16_high_vec = _mm512_cvtph_epi16(sum_f16_high_vec);
1157
+ sum_u8_vec = _mm512_packus_epi16(sum_i16_low_vec, sum_i16_high_vec);
1158
+ _mm512_mask_storeu_epi8(result, mask, sum_u8_vec);
1159
+ result += 64;
1160
+ if (n) goto simsimd_fma_u8_sapphire_cycle;
1161
+ }
1162
+
1163
+ #pragma clang attribute pop
1164
+ #pragma GCC pop_options
1165
+ #endif // SIMSIMD_TARGET_SAPPHIRE
1166
+ #endif // _SIMSIMD_TARGET_X86
1167
+
1168
+ #if _SIMSIMD_TARGET_ARM
1169
+ #if SIMSIMD_TARGET_NEON
1170
+ #pragma GCC push_options
1171
+ #pragma GCC target("arch=armv8.2-a+simd")
1172
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
1173
+
1174
+ SIMSIMD_PUBLIC void simsimd_wsum_f32_neon( //
1175
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n, //
1176
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1177
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1178
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1179
+
1180
+ // The main loop:
1181
+ simsimd_size_t i = 0;
1182
+ for (; i + 4 <= n; i += 4) {
1183
+ float32x4_t a_vec = vld1q_f32(a + i);
1184
+ float32x4_t b_vec = vld1q_f32(b + i);
1185
+ float32x4_t a_scaled_vec = vmulq_n_f32(a_vec, alpha_f32);
1186
+ float32x4_t b_scaled_vec = vmulq_n_f32(b_vec, beta_f32);
1187
+ float32x4_t sum_vec = vaddq_f32(a_scaled_vec, b_scaled_vec);
1188
+ vst1q_f32(result + i, sum_vec);
1189
+ }
1190
+
1191
+ // The tail:
1192
+ for (; i < n; ++i) result[i] = alpha_f32 * a[i] + beta_f32 * b[i];
1193
+ }
1194
+
1195
+ SIMSIMD_PUBLIC void simsimd_fma_f32_neon( //
1196
+ simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_f32_t const *c, //
1197
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f32_t *result) {
1198
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1199
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1200
+
1201
+ // The main loop:
1202
+ simsimd_size_t i = 0;
1203
+ for (; i + 4 <= n; i += 4) {
1204
+ float32x4_t a_vec = vld1q_f32(a + i);
1205
+ float32x4_t b_vec = vld1q_f32(b + i);
1206
+ float32x4_t c_vec = vld1q_f32(c + i);
1207
+ float32x4_t ab_vec = vmulq_f32(a_vec, b_vec);
1208
+ float32x4_t ab_scaled_vec = vmulq_n_f32(ab_vec, alpha_f32);
1209
+ float32x4_t sum_vec = vfmaq_n_f32(ab_scaled_vec, c_vec, beta_f32);
1210
+ vst1q_f32(result + i, sum_vec);
1211
+ }
1212
+
1213
+ // The tail:
1214
+ for (; i < n; ++i) result[i] = alpha_f32 * a[i] * b[i] + beta_f32 * c[i];
1215
+ }
1216
+
1217
+ #pragma clang attribute pop
1218
+ #pragma GCC pop_options
1219
+ #endif // SIMSIMD_TARGET_NEON
1220
+
1221
+ #if SIMSIMD_TARGET_NEON_BF16
1222
+ #pragma GCC push_options
1223
+ #pragma GCC target("arch=armv8.6-a+simd+bf16")
1224
+ #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
1225
+
1226
+ SIMSIMD_PUBLIC void simsimd_wsum_bf16_neon( //
1227
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n, //
1228
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
1229
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1230
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1231
+
1232
+ // The main loop:
1233
+ simsimd_size_t i = 0;
1234
+ for (; i + 4 <= n; i += 4) {
1235
+ float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
1236
+ float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
1237
+ float32x4_t a_scaled_vec = vmulq_n_f32(a_vec, alpha_f32);
1238
+ float32x4_t b_scaled_vec = vmulq_n_f32(b_vec, beta_f32);
1239
+ float32x4_t sum_vec = vaddq_f32(a_scaled_vec, b_scaled_vec);
1240
+ vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
1241
+ }
1242
+
1243
+ // The tail:
1244
+ for (; i < n; ++i)
1245
+ simsimd_f32_to_bf16(alpha_f32 * simsimd_bf16_to_f32(a + i) + beta_f32 * simsimd_bf16_to_f32(b + i), result + i);
1246
+ }
1247
+
1248
+ SIMSIMD_PUBLIC void simsimd_fma_bf16_neon( //
1249
+ simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_bf16_t const *c, //
1250
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_bf16_t *result) {
1251
+ simsimd_f32_t alpha_f32 = (simsimd_f32_t)alpha;
1252
+ simsimd_f32_t beta_f32 = (simsimd_f32_t)beta;
1253
+
1254
+ // The main loop:
1255
+ simsimd_size_t i = 0;
1256
+ for (; i + 4 <= n; i += 4) {
1257
+ float32x4_t a_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)a + i));
1258
+ float32x4_t b_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)b + i));
1259
+ float32x4_t c_vec = vcvt_f32_bf16(vld1_bf16((bfloat16_t const *)c + i));
1260
+ float32x4_t ab_vec = vmulq_f32(a_vec, b_vec);
1261
+ float32x4_t ab_scaled_vec = vmulq_n_f32(ab_vec, alpha_f32);
1262
+ float32x4_t sum_vec = vfmaq_n_f32(ab_scaled_vec, c_vec, beta_f32);
1263
+ vst1_bf16((bfloat16_t *)result + i, vcvt_bf16_f32(sum_vec));
1264
+ }
1265
+
1266
+ // The tail:
1267
+ for (; i < n; ++i)
1268
+ simsimd_f32_to_bf16(
1269
+ alpha_f32 * simsimd_bf16_to_f32(a + i) * simsimd_bf16_to_f32(b + i) + beta_f32 * simsimd_bf16_to_f32(c + i),
1270
+ result + i);
1271
+ }
1272
+
1273
+ #pragma clang attribute pop
1274
+ #pragma GCC pop_options
1275
+ #endif // SIMSIMD_TARGET_NEON_BF16
1276
+
1277
+ #if SIMSIMD_TARGET_NEON_F16
1278
+ #pragma GCC push_options
1279
+ #pragma GCC target("arch=armv8.2-a+simd+fp16")
1280
+ #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
1281
+
1282
+ SIMSIMD_PUBLIC void simsimd_wsum_f16_neon( //
1283
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n, //
1284
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
1285
+ float16_t alpha_f16 = (float16_t)alpha;
1286
+ float16_t beta_f16 = (float16_t)beta;
1287
+
1288
+ // The main loop:
1289
+ simsimd_size_t i = 0;
1290
+ for (; i + 8 <= n; i += 8) {
1291
+ float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
1292
+ float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
1293
+ float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
1294
+ float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
1295
+ float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
1296
+ vst1q_f16((float16_t *)result + i, sum_vec);
1297
+ }
1298
+
1299
+ // The tail:
1300
+ for (; i < n; ++i)
1301
+ ((float16_t *)result)[i] = alpha_f16 * ((float16_t const *)a)[i] + beta_f16 * ((float16_t const *)b)[i];
1302
+ }
1303
+
1304
+ SIMSIMD_PUBLIC void simsimd_fma_f16_neon( //
1305
+ simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_f16_t const *c, //
1306
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_f16_t *result) {
1307
+ float16_t alpha_f16 = (float16_t)alpha;
1308
+ float16_t beta_f16 = (float16_t)beta;
1309
+
1310
+ // The main loop:
1311
+ simsimd_size_t i = 0;
1312
+ for (; i + 8 <= n; i += 8) {
1313
+ float16x8_t a_vec = vld1q_f16((float16_t const *)a + i);
1314
+ float16x8_t b_vec = vld1q_f16((float16_t const *)b + i);
1315
+ float16x8_t c_vec = vld1q_f16((float16_t const *)c + i);
1316
+ float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
1317
+ float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
1318
+ float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
1319
+ vst1q_f16((float16_t *)result + i, sum_vec);
1320
+ }
1321
+
1322
+ // The tail:
1323
+ for (; i < n; ++i)
1324
+ ((float16_t *)result)[i] =
1325
+ alpha_f16 * ((float16_t const *)a)[i] * ((float16_t const *)b)[i] + beta_f16 * ((float16_t const *)c)[i];
1326
+ }
1327
+
1328
+ SIMSIMD_PUBLIC void simsimd_wsum_u8_neon( //
1329
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n, //
1330
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
1331
+ float16_t alpha_f16 = (float16_t)alpha;
1332
+ float16_t beta_f16 = (float16_t)beta;
1333
+
1334
+ // The main loop:
1335
+ simsimd_size_t i = 0;
1336
+ for (; i + 8 <= n; i += 8) {
1337
+ uint8x8_t a_u8_vec = vld1_u8(a + i);
1338
+ uint8x8_t b_u8_vec = vld1_u8(b + i);
1339
+ float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
1340
+ float16x8_t b_vec = vcvtq_f16_u16(vmovl_u8(b_u8_vec));
1341
+ float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
1342
+ float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
1343
+ float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
1344
+ uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
1345
+ vst1_u8(result + i, sum_u8_vec);
1346
+ }
1347
+
1348
+ // The tail:
1349
+ for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] + beta_f16 * b[i], result + i); }
1350
+ }
1351
+
1352
+ SIMSIMD_PUBLIC void simsimd_fma_u8_neon( //
1353
+ simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_u8_t const *c, //
1354
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_u8_t *result) {
1355
+ float16_t alpha_f16 = (float16_t)alpha;
1356
+ float16_t beta_f16 = (float16_t)beta;
1357
+
1358
+ // The main loop:
1359
+ simsimd_size_t i = 0;
1360
+ for (; i + 8 <= n; i += 8) {
1361
+ uint8x8_t a_u8_vec = vld1_u8(a + i);
1362
+ uint8x8_t b_u8_vec = vld1_u8(b + i);
1363
+ uint8x8_t c_u8_vec = vld1_u8(c + i);
1364
+ float16x8_t a_vec = vcvtq_f16_u16(vmovl_u8(a_u8_vec));
1365
+ float16x8_t b_vec = vcvtq_f16_u16(vmovl_u8(b_u8_vec));
1366
+ float16x8_t c_vec = vcvtq_f16_u16(vmovl_u8(c_u8_vec));
1367
+ float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
1368
+ float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
1369
+ float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
1370
+ uint8x8_t sum_u8_vec = vqmovn_u16(vcvtaq_u16_f16(sum_vec));
1371
+ vst1_u8(result + i, sum_u8_vec);
1372
+ }
1373
+
1374
+ // The tail:
1375
+ for (; i < n; ++i) { SIMSIMD_F32_TO_U8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
1376
+ }
1377
+
1378
+ SIMSIMD_PUBLIC void simsimd_wsum_i8_neon( //
1379
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n, //
1380
+ simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1381
+ float16_t alpha_f16 = (float16_t)alpha;
1382
+ float16_t beta_f16 = (float16_t)beta;
1383
+
1384
+ // The main loop:
1385
+ simsimd_size_t i = 0;
1386
+ for (; i + 8 <= n; i += 8) {
1387
+ int8x8_t a_i8_vec = vld1_s8(a + i);
1388
+ int8x8_t b_i8_vec = vld1_s8(b + i);
1389
+ float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
1390
+ float16x8_t b_vec = vcvtq_f16_s16(vmovl_s8(b_i8_vec));
1391
+ float16x8_t a_scaled_vec = vmulq_n_f16(a_vec, alpha_f16);
1392
+ float16x8_t b_scaled_vec = vmulq_n_f16(b_vec, beta_f16);
1393
+ float16x8_t sum_vec = vaddq_f16(a_scaled_vec, b_scaled_vec);
1394
+ int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
1395
+ vst1_s8(result + i, sum_i8_vec);
1396
+ }
1397
+
1398
+ // The tail:
1399
+ for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i] + beta_f16 * b[i], result + i); }
1400
+ }
1401
+
1402
+ SIMSIMD_PUBLIC void simsimd_fma_i8_neon( //
1403
+ simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_i8_t const *c, //
1404
+ simsimd_size_t n, simsimd_distance_t alpha, simsimd_distance_t beta, simsimd_i8_t *result) {
1405
+ float16_t alpha_f16 = (float16_t)alpha;
1406
+ float16_t beta_f16 = (float16_t)beta;
1407
+
1408
+ // The main loop:
1409
+ simsimd_size_t i = 0;
1410
+ for (; i + 8 <= n; i += 8) {
1411
+ int8x8_t a_i8_vec = vld1_s8(a + i);
1412
+ int8x8_t b_i8_vec = vld1_s8(b + i);
1413
+ int8x8_t c_i8_vec = vld1_s8(c + i);
1414
+ float16x8_t a_vec = vcvtq_f16_s16(vmovl_s8(a_i8_vec));
1415
+ float16x8_t b_vec = vcvtq_f16_s16(vmovl_s8(b_i8_vec));
1416
+ float16x8_t c_vec = vcvtq_f16_s16(vmovl_s8(c_i8_vec));
1417
+ float16x8_t ab_vec = vmulq_f16(a_vec, b_vec);
1418
+ float16x8_t ab_scaled_vec = vmulq_n_f16(ab_vec, alpha_f16);
1419
+ float16x8_t sum_vec = vfmaq_n_f16(ab_scaled_vec, c_vec, beta_f16);
1420
+ int8x8_t sum_i8_vec = vqmovn_s16(vcvtaq_s16_f16(sum_vec));
1421
+ vst1_s8(result + i, sum_i8_vec);
1422
+ }
1423
+
1424
+ // The tail:
1425
+ for (; i < n; ++i) { SIMSIMD_F32_TO_I8(alpha_f16 * a[i] * b[i] + beta_f16 * c[i], result + i); }
1426
+ }
1427
+
1428
+ #pragma clang attribute pop
1429
+ #pragma GCC pop_options
1430
+ #endif // SIMSIMD_TARGET_NEON_F16
1431
+ #endif // _SIMSIMD_TARGET_ARM
1432
+
1433
+ #ifdef __cplusplus
1434
+ }
1435
+ #endif
1436
+
1437
+ #endif