usearch 2.15.3 → 2.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +1 -1
  2. package/cpp/test.cpp +16 -0
  3. package/include/usearch/index.hpp +21 -5
  4. package/include/usearch/index_dense.hpp +6 -1
  5. package/include/usearch/index_plugins.hpp +345 -8
  6. package/javascript/lib.cpp +17 -5
  7. package/javascript/usearch.test.js +190 -21
  8. package/package.json +1 -1
  9. package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
  10. package/prebuilds/linux-arm64/usearch.node +0 -0
  11. package/prebuilds/linux-x64/usearch.node +0 -0
  12. package/simsimd/.git-blame-ignore-revs +2 -0
  13. package/simsimd/CONTRIBUTING.md +71 -23
  14. package/simsimd/Cargo.lock +1 -1
  15. package/simsimd/Cargo.toml +4 -4
  16. package/simsimd/MANIFEST.in +2 -1
  17. package/simsimd/Package.swift +15 -3
  18. package/simsimd/README.md +104 -35
  19. package/simsimd/binding.gyp +6 -2
  20. package/simsimd/build.rs +2 -0
  21. package/simsimd/c/lib.c +180 -99
  22. package/simsimd/include/simsimd/binary.h +311 -102
  23. package/simsimd/include/simsimd/curved.h +134 -135
  24. package/simsimd/include/simsimd/dot.h +448 -274
  25. package/simsimd/include/simsimd/elementwise.h +1437 -0
  26. package/simsimd/include/simsimd/geospatial.h +2 -2
  27. package/simsimd/include/simsimd/mesh.h +2 -2
  28. package/simsimd/include/simsimd/probability.h +88 -88
  29. package/simsimd/include/simsimd/simsimd.h +1157 -772
  30. package/simsimd/include/simsimd/sparse.h +768 -147
  31. package/simsimd/include/simsimd/spatial.h +955 -376
  32. package/simsimd/include/simsimd/types.h +145 -71
  33. package/simsimd/javascript/lib.c +1 -2
  34. package/simsimd/package.json +4 -4
  35. package/simsimd/rust/lib.rs +14 -1
  36. package/simsimd/scripts/bench.cxx +1070 -0
  37. package/simsimd/scripts/bench.ipynb +187 -0
  38. package/simsimd/{javascript → scripts}/bench.js +1 -1
  39. package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
  40. package/simsimd/scripts/bench_vectors.py +700 -0
  41. package/simsimd/scripts/bench_vectors_live.py +213 -0
  42. package/simsimd/scripts/test.c +155 -0
  43. package/simsimd/{javascript → scripts}/test.mjs +2 -2
  44. package/simsimd/scripts/test.py +1410 -0
  45. package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
  46. package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
  47. /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
  48. /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
@@ -13,12 +13,13 @@
13
13
  * - 32-bit IEEE floating point numbers
14
14
  * - 16-bit IEEE floating point numbers
15
15
  * - 16-bit brain floating point numbers
16
+ * - 8-bit unsigned integral numbers
16
17
  * - 8-bit signed integral numbers
17
18
  * - 4-bit signed integral numbers
18
19
  *
19
20
  * For hardware architectures:
20
- * - Arm (NEON, SVE)
21
- * - x86 (AVX2, AVX512)
21
+ * - Arm: NEON, SVE
22
+ * - x86: Haswell, Skylake, Ice Lake, Genoa, Sapphire
22
23
  *
23
24
  * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
24
25
  * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
@@ -55,6 +56,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf1
55
56
  SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
56
57
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
57
58
  SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
59
+ SIMSIMD_PUBLIC void simsimd_l2_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
60
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
61
+ SIMSIMD_PUBLIC void simsimd_cos_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
58
62
 
59
63
  /* Double-precision serial backends for all numeric types.
60
64
  * For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
@@ -67,10 +71,6 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f1
67
71
  SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
68
72
  SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
69
73
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
70
- SIMSIMD_PUBLIC void simsimd_cos_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
71
- SIMSIMD_PUBLIC void simsimd_l2_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
72
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
73
- SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
74
74
 
75
75
  /* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
76
76
  * By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
@@ -91,6 +91,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_
91
91
  SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
92
92
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
93
93
  SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
94
+ SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
95
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
96
+ SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
94
97
 
95
98
  /* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
96
99
  * Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
@@ -117,6 +120,9 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
117
120
  SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
118
121
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
119
122
  SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
123
+ SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
124
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
125
+ SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
120
126
  SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
121
127
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
122
128
  SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
@@ -126,6 +132,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf
126
132
  SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
127
133
  SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
128
134
  SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
135
+ SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
136
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
137
+ SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
129
138
 
130
139
  /* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
131
140
  * Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
@@ -142,9 +151,15 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_
142
151
  * Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
143
152
  * Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
144
153
  */
154
+ SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
155
+ SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
156
+ SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
145
157
  SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
146
158
  SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
147
159
  SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
160
+ SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
161
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
162
+ SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
148
163
  SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
149
164
  SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
150
165
  SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
@@ -152,49 +167,51 @@ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_
152
167
  SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
153
168
  SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
154
169
 
170
+ /* SIMD-powered backends for AVX-INT8-VNNI extensions on Xeon 6 CPUs, including Sierra Forest and Granite Rapids.
171
+ * The packs many "efficiency" cores into a single socket, avoiding heavy 512-bit operations, and focusing on 256-bit ones.
172
+ */
173
+ SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
155
174
  // clang-format on
156
175
 
157
- #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
158
- SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const* a, \
159
- simsimd_##input_type##_t const* b, simsimd_size_t n, \
160
- simsimd_distance_t* result) { \
161
- simsimd_##accumulator_type##_t d2 = 0; \
162
- for (simsimd_size_t i = 0; i != n; ++i) { \
163
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
164
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
165
- d2 += (ai - bi) * (ai - bi); \
166
- } \
167
- *result = d2; \
168
- }
169
-
170
- #define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
171
- SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const* a, \
172
- simsimd_##input_type##_t const* b, simsimd_size_t n, \
173
- simsimd_distance_t* result) { \
174
- simsimd_l2sq_##input_type##_##name(a, b, n, result); \
175
- *result = SIMSIMD_SQRT(*result); \
176
- }
177
-
178
- #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
179
- SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const* a, \
180
- simsimd_##input_type##_t const* b, simsimd_size_t n, \
181
- simsimd_distance_t* result) { \
182
- simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
183
- for (simsimd_size_t i = 0; i != n; ++i) { \
184
- simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
185
- simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
186
- ab += ai * bi; \
187
- a2 += ai * ai; \
188
- b2 += bi * bi; \
189
- } \
190
- if (a2 == 0 && b2 == 0) { \
191
- *result = 0; \
192
- } else if (ab == 0) { \
193
- *result = 1; \
194
- } else { \
195
- simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
196
- *result = unclipped_result > 0 ? unclipped_result : 0; \
197
- } \
176
+ #define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
177
+ SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const *a, \
178
+ simsimd_##input_type##_t const *b, simsimd_size_t n, \
179
+ simsimd_distance_t *result) { \
180
+ simsimd_##accumulator_type##_t d2 = 0; \
181
+ for (simsimd_size_t i = 0; i != n; ++i) { \
182
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
183
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
184
+ d2 += (ai - bi) * (ai - bi); \
185
+ } \
186
+ *result = d2; \
187
+ }
188
+
189
+ #define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
190
+ SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const *a, \
191
+ simsimd_##input_type##_t const *b, simsimd_size_t n, \
192
+ simsimd_distance_t *result) { \
193
+ simsimd_l2sq_##input_type##_##name(a, b, n, result); \
194
+ *result = SIMSIMD_SQRT(*result); \
195
+ }
196
+
197
+ #define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
198
+ SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const *a, \
199
+ simsimd_##input_type##_t const *b, simsimd_size_t n, \
200
+ simsimd_distance_t *result) { \
201
+ simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
202
+ for (simsimd_size_t i = 0; i != n; ++i) { \
203
+ simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
204
+ simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
205
+ ab += ai * bi; \
206
+ a2 += ai * ai; \
207
+ b2 += bi * bi; \
208
+ } \
209
+ if (a2 == 0 && b2 == 0) { *result = 0; } \
210
+ else if (ab == 0) { *result = 1; } \
211
+ else { \
212
+ simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
213
+ *result = unclipped_result > 0 ? unclipped_result : 0; \
214
+ } \
198
215
  }
199
216
 
200
217
  SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
@@ -217,6 +234,10 @@ SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
217
234
  SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
218
235
  SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
219
236
 
237
+ SIMSIMD_MAKE_COS(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_u8_serial
238
+ SIMSIMD_MAKE_L2SQ(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_u8_serial
239
+ SIMSIMD_MAKE_L2(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_u8_serial
240
+
220
241
  SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
221
242
  SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
222
243
  SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
@@ -229,11 +250,7 @@ SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_
229
250
  SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
230
251
  SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
231
252
 
232
- SIMSIMD_MAKE_COS(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_accurate
233
- SIMSIMD_MAKE_L2SQ(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_accurate
234
- SIMSIMD_MAKE_L2(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_accurate
235
-
236
- #if SIMSIMD_TARGET_ARM
253
+ #if _SIMSIMD_TARGET_ARM
237
254
  #if SIMSIMD_TARGET_NEON
238
255
  #pragma GCC push_options
239
256
  #pragma GCC target("arch=armv8.2-a+simd")
@@ -247,10 +264,8 @@ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
247
264
  }
248
265
  SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
249
266
  simsimd_f32_t b2) {
250
- if (a2 == 0 && b2 == 0)
251
- return 0;
252
- if (ab == 0)
253
- return 1;
267
+ if (a2 == 0 && b2 == 0) return 0;
268
+ if (ab == 0) return 1;
254
269
  simsimd_f32_t squares_arr[2] = {a2, b2};
255
270
  float32x2_t squares = vld1_f32(squares_arr);
256
271
  // Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
@@ -270,10 +285,8 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_
270
285
 
271
286
  SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
272
287
  simsimd_f64_t b2) {
273
- if (a2 == 0 && b2 == 0)
274
- return 0;
275
- if (ab == 0)
276
- return 1;
288
+ if (a2 == 0 && b2 == 0) return 0;
289
+ if (ab == 0) return 1;
277
290
  simsimd_f64_t squares_arr[2] = {a2, b2};
278
291
  float64x2_t squares = vld1q_f64(squares_arr);
279
292
 
@@ -292,13 +305,13 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_
292
305
  return result > 0 ? result : 0;
293
306
  }
294
307
 
295
- SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
296
- simsimd_distance_t* result) {
308
+ SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
309
+ simsimd_distance_t *result) {
297
310
  simsimd_l2sq_f32_neon(a, b, n, result);
298
311
  *result = _simsimd_sqrt_f64_neon(*result);
299
312
  }
300
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
301
- simsimd_distance_t* result) {
313
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
314
+ simsimd_distance_t *result) {
302
315
  float32x4_t sum_vec = vdupq_n_f32(0);
303
316
  simsimd_size_t i = 0;
304
317
  for (; i + 4 <= n; i += 4) {
@@ -315,8 +328,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t
315
328
  *result = sum;
316
329
  }
317
330
 
318
- SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
319
- simsimd_distance_t* result) {
331
+ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
332
+ simsimd_distance_t *result) {
320
333
  float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
321
334
  simsimd_size_t i = 0;
322
335
  for (; i + 4 <= n; i += 4) {
@@ -335,13 +348,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
335
348
  *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
336
349
  }
337
350
 
338
- SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
339
- simsimd_distance_t* result) {
351
+ SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
352
+ simsimd_distance_t *result) {
340
353
  simsimd_l2sq_f64_neon(a, b, n, result);
341
354
  *result = _simsimd_sqrt_f64_neon(*result);
342
355
  }
343
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
344
- simsimd_distance_t* result) {
356
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
357
+ simsimd_distance_t *result) {
345
358
  float64x2_t sum_vec = vdupq_n_f64(0);
346
359
  simsimd_size_t i = 0;
347
360
  for (; i + 2 <= n; i += 2) {
@@ -358,8 +371,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t
358
371
  *result = sum;
359
372
  }
360
373
 
361
- SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
362
- simsimd_distance_t* result) {
374
+ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
375
+ simsimd_distance_t *result) {
363
376
  float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
364
377
  simsimd_size_t i = 0;
365
378
  for (; i + 2 <= n; i += 2) {
@@ -387,13 +400,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t c
387
400
  #pragma GCC target("arch=armv8.2-a+simd+fp16")
388
401
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
389
402
 
390
- SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
391
- simsimd_distance_t* result) {
403
+ SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
404
+ simsimd_distance_t *result) {
392
405
  simsimd_l2sq_f16_neon(a, b, n, result);
393
406
  *result = _simsimd_sqrt_f32_neon(*result);
394
407
  }
395
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
396
- simsimd_distance_t* result) {
408
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
409
+ simsimd_distance_t *result) {
397
410
  float32x4_t a_vec, b_vec;
398
411
  float32x4_t sum_vec = vdupq_n_f32(0);
399
412
 
@@ -402,21 +415,21 @@ simsimd_l2sq_f16_neon_cycle:
402
415
  a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
403
416
  b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
404
417
  n = 0;
405
- } else {
406
- a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
407
- b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
418
+ }
419
+ else {
420
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
421
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
408
422
  n -= 4, a += 4, b += 4;
409
423
  }
410
424
  float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
411
425
  sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
412
- if (n)
413
- goto simsimd_l2sq_f16_neon_cycle;
426
+ if (n) goto simsimd_l2sq_f16_neon_cycle;
414
427
 
415
428
  *result = vaddvq_f32(sum_vec);
416
429
  }
417
430
 
418
- SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
419
- simsimd_distance_t* result) {
431
+ SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
432
+ simsimd_distance_t *result) {
420
433
  float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
421
434
  float32x4_t a_vec, b_vec;
422
435
 
@@ -425,16 +438,16 @@ simsimd_cos_f16_neon_cycle:
425
438
  a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
426
439
  b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
427
440
  n = 0;
428
- } else {
429
- a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a));
430
- b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b));
441
+ }
442
+ else {
443
+ a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
444
+ b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
431
445
  n -= 4, a += 4, b += 4;
432
446
  }
433
447
  ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
434
448
  a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
435
449
  b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
436
- if (n)
437
- goto simsimd_cos_f16_neon_cycle;
450
+ if (n) goto simsimd_cos_f16_neon_cycle;
438
451
 
439
452
  simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
440
453
  *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
@@ -449,8 +462,8 @@ simsimd_cos_f16_neon_cycle:
449
462
  #pragma GCC target("arch=armv8.6-a+simd+bf16")
450
463
  #pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
451
464
 
452
- SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
453
- simsimd_distance_t* result) {
465
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
466
+ simsimd_distance_t *result) {
454
467
 
455
468
  // Similar to `simsimd_cos_i8_neon`, we can use the `BFMMLA` instruction through
456
469
  // the `vbfmmlaq_f32` intrinsic to compute matrix products and later drop 1/4 of values.
@@ -501,29 +514,29 @@ simsimd_cos_bf16_neon_cycle:
501
514
  a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
502
515
  b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
503
516
  n = 0;
504
- } else {
505
- a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
506
- b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
517
+ }
518
+ else {
519
+ a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
520
+ b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
507
521
  n -= 8, a += 8, b += 8;
508
522
  }
509
523
  ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
510
524
  a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
511
525
  b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
512
- if (n)
513
- goto simsimd_cos_bf16_neon_cycle;
526
+ if (n) goto simsimd_cos_bf16_neon_cycle;
514
527
 
515
528
  // Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
516
529
  simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
517
530
  *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
518
531
  }
519
532
 
520
- SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
521
- simsimd_distance_t* result) {
533
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
534
+ simsimd_distance_t *result) {
522
535
  simsimd_l2sq_bf16_neon(a, b, n, result);
523
536
  *result = _simsimd_sqrt_f64_neon(*result);
524
537
  }
525
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
526
- simsimd_distance_t* result) {
538
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
539
+ simsimd_distance_t *result) {
527
540
  float32x4_t diff_high_vec, diff_low_vec;
528
541
  float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
529
542
 
@@ -534,17 +547,17 @@ simsimd_l2sq_bf16_neon_cycle:
534
547
  diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
535
548
  diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
536
549
  n = 0;
537
- } else {
538
- bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)a);
539
- bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const*)b);
550
+ }
551
+ else {
552
+ bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
553
+ bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
540
554
  diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
541
555
  diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
542
556
  n -= 8, a += 8, b += 8;
543
557
  }
544
558
  sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
545
559
  sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
546
- if (n)
547
- goto simsimd_l2sq_bf16_neon_cycle;
560
+ if (n) goto simsimd_l2sq_bf16_neon_cycle;
548
561
 
549
562
  *result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
550
563
  }
@@ -558,35 +571,40 @@ simsimd_l2sq_bf16_neon_cycle:
558
571
  #pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
559
572
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
560
573
 
561
- SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
562
- simsimd_distance_t* result) {
574
+ SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
575
+ simsimd_distance_t *result) {
563
576
  simsimd_l2sq_i8_neon(a, b, n, result);
564
577
  *result = _simsimd_sqrt_f32_neon(*result);
565
578
  }
566
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
567
- simsimd_distance_t* result) {
568
- int32x4_t d2_vec = vdupq_n_s32(0);
579
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
580
+ simsimd_distance_t *result) {
581
+
582
+ // The naive approach is to upcast 8-bit signed integers into 16-bit signed integers
583
+ // for subtraction, then multiply within 16-bit integers and accumulate the results
584
+ // into 32-bit integers. This approach is slow on modern Arm CPUs. On Graviton 4,
585
+ // that approach results in 17 GB/s of throughput, compared to 39 GB/s for `i8`
586
+ // dot-products.
587
+ //
588
+ // Luckily we can use the `vabdq_s8` which technically returns `i8` values, but it's a
589
+ // matter of reinterpret-casting! That approach boosts us to 33 GB/s of throughput.
590
+ uint32x4_t d2_vec = vdupq_n_u32(0);
569
591
  simsimd_size_t i = 0;
570
- for (; i + 8 <= n; i += 8) {
571
- int8x8_t a_vec = vld1_s8(a + i);
572
- int8x8_t b_vec = vld1_s8(b + i);
573
- int16x8_t a_vec16 = vmovl_s8(a_vec);
574
- int16x8_t b_vec16 = vmovl_s8(b_vec);
575
- int16x8_t d_vec = vsubq_s16(a_vec16, b_vec16);
576
- int32x4_t d_low = vmull_s16(vget_low_s16(d_vec), vget_low_s16(d_vec));
577
- int32x4_t d_high = vmull_s16(vget_high_s16(d_vec), vget_high_s16(d_vec));
578
- d2_vec = vaddq_s32(d2_vec, vaddq_s32(d_low, d_high));
579
- }
580
- int32_t d2 = vaddvq_s32(d2_vec);
592
+ for (; i + 16 <= n; i += 16) {
593
+ int8x16_t a_vec = vld1q_s8(a + i);
594
+ int8x16_t b_vec = vld1q_s8(b + i);
595
+ uint8x16_t d_vec = vreinterpretq_u8_s8(vabdq_s8(a_vec, b_vec));
596
+ d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
597
+ }
598
+ uint32_t d2 = vaddvq_u32(d2_vec);
581
599
  for (; i < n; ++i) {
582
600
  int32_t n = (int32_t)a[i] - b[i];
583
- d2 += n * n;
601
+ d2 += (uint32_t)(n * n);
584
602
  }
585
603
  *result = d2;
586
604
  }
587
605
 
588
- SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
589
- simsimd_distance_t* result) {
606
+ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
607
+ simsimd_distance_t *result) {
590
608
 
591
609
  simsimd_size_t i = 0;
592
610
 
@@ -704,6 +722,56 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
704
722
  *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
705
723
  }
706
724
 
725
+ SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
726
+ simsimd_distance_t *result) {
727
+ simsimd_l2sq_u8_neon(a, b, n, result);
728
+ *result = _simsimd_sqrt_f32_neon(*result);
729
+ }
730
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
731
+ simsimd_distance_t *result) {
732
+ uint32x4_t d2_vec = vdupq_n_u32(0);
733
+ simsimd_size_t i = 0;
734
+ for (; i + 16 <= n; i += 16) {
735
+ uint8x16_t a_vec = vld1q_u8(a + i);
736
+ uint8x16_t b_vec = vld1q_u8(b + i);
737
+ uint8x16_t d_vec = vabdq_u8(a_vec, b_vec);
738
+ d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
739
+ }
740
+ uint32_t d2 = vaddvq_u32(d2_vec);
741
+ for (; i < n; ++i) {
742
+ int32_t n = (int32_t)a[i] - b[i];
743
+ d2 += (uint32_t)(n * n);
744
+ }
745
+ *result = d2;
746
+ }
747
+
748
+ SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
749
+ simsimd_distance_t *result) {
750
+
751
+ simsimd_size_t i = 0;
752
+ uint32x4_t ab_vec = vdupq_n_u32(0);
753
+ uint32x4_t a2_vec = vdupq_n_u32(0);
754
+ uint32x4_t b2_vec = vdupq_n_u32(0);
755
+ for (; i + 16 <= n; i += 16) {
756
+ uint8x16_t a_vec = vld1q_u8(a + i);
757
+ uint8x16_t b_vec = vld1q_u8(b + i);
758
+ ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
759
+ a2_vec = vdotq_u32(a2_vec, a_vec, a_vec);
760
+ b2_vec = vdotq_u32(b2_vec, b_vec, b_vec);
761
+ }
762
+ uint32_t ab = vaddvq_u32(ab_vec);
763
+ uint32_t a2 = vaddvq_u32(a2_vec);
764
+ uint32_t b2 = vaddvq_u32(b2_vec);
765
+
766
+ // Take care of the tail:
767
+ for (; i < n; ++i) {
768
+ uint32_t ai = a[i], bi = b[i];
769
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
770
+ }
771
+
772
+ *result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
773
+ }
774
+
707
775
  #pragma clang attribute pop
708
776
  #pragma GCC pop_options
709
777
  #endif // SIMSIMD_TARGET_NEON_I8
@@ -713,13 +781,13 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
713
781
  #pragma GCC target("arch=armv8.2-a+sve")
714
782
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
715
783
 
716
- SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
717
- simsimd_distance_t* result) {
784
+ SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
785
+ simsimd_distance_t *result) {
718
786
  simsimd_l2sq_f32_sve(a, b, n, result);
719
787
  *result = _simsimd_sqrt_f64_neon(*result);
720
788
  }
721
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
722
- simsimd_distance_t* result) {
789
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
790
+ simsimd_distance_t *result) {
723
791
  simsimd_size_t i = 0;
724
792
  svfloat32_t d2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
725
793
  do {
@@ -734,8 +802,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t c
734
802
  *result = d2;
735
803
  }
736
804
 
737
- SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
738
- simsimd_distance_t* result) {
805
+ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
806
+ simsimd_distance_t *result) {
739
807
  simsimd_size_t i = 0;
740
808
  svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
741
809
  svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
@@ -756,13 +824,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
756
824
  *result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
757
825
  }
758
826
 
759
- SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
760
- simsimd_distance_t* result) {
827
+ SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
828
+ simsimd_distance_t *result) {
761
829
  simsimd_l2sq_f64_sve(a, b, n, result);
762
830
  *result = _simsimd_sqrt_f64_neon(*result);
763
831
  }
764
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
765
- simsimd_distance_t* result) {
832
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
833
+ simsimd_distance_t *result) {
766
834
  simsimd_size_t i = 0;
767
835
  svfloat64_t d2_vec = svdupq_n_f64(0.0, 0.0);
768
836
  do {
@@ -777,8 +845,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t c
777
845
  *result = d2;
778
846
  }
779
847
 
780
- SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
781
- simsimd_distance_t* result) {
848
+ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
849
+ simsimd_distance_t *result) {
782
850
  simsimd_size_t i = 0;
783
851
  svfloat64_t ab_vec = svdupq_n_f64(0.0, 0.0);
784
852
  svfloat64_t a2_vec = svdupq_n_f64(0.0, 0.0);
@@ -808,17 +876,17 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
808
876
  #pragma GCC target("arch=armv8.2-a+sve+fp16")
809
877
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
810
878
 
811
- SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
812
- simsimd_distance_t* result) {
879
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
880
+ simsimd_distance_t *result) {
813
881
  simsimd_l2sq_f16_sve(a, b, n, result);
814
882
  *result = _simsimd_sqrt_f32_neon(*result);
815
883
  }
816
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
817
- simsimd_distance_t* result) {
884
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
885
+ simsimd_distance_t *result) {
818
886
  simsimd_size_t i = 0;
819
887
  svfloat16_t d2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
820
- simsimd_f16_for_arm_simd_t const* a = (simsimd_f16_for_arm_simd_t const*)(a_enum);
821
- simsimd_f16_for_arm_simd_t const* b = (simsimd_f16_for_arm_simd_t const*)(b_enum);
888
+ simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
889
+ simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
822
890
  do {
823
891
  svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
824
892
  svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
@@ -831,14 +899,14 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f1
831
899
  *result = d2_f16;
832
900
  }
833
901
 
834
- SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, simsimd_size_t n,
835
- simsimd_distance_t* result) {
902
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
903
+ simsimd_distance_t *result) {
836
904
  simsimd_size_t i = 0;
837
905
  svfloat16_t ab_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
838
906
  svfloat16_t a2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
839
907
  svfloat16_t b2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
840
- simsimd_f16_for_arm_simd_t const* a = (simsimd_f16_for_arm_simd_t const*)(a_enum);
841
- simsimd_f16_for_arm_simd_t const* b = (simsimd_f16_for_arm_simd_t const*)(b_enum);
908
+ simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
909
+ simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
842
910
  do {
843
911
  svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
844
912
  svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
@@ -864,18 +932,18 @@ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
864
932
  #pragma GCC target("arch=armv8.2-a+sve+bf16")
865
933
  #pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
866
934
 
867
- SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
868
- simsimd_distance_t* result) {
935
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
936
+ simsimd_distance_t *result) {
869
937
  simsimd_l2sq_bf16_sve(a, b, n, result);
870
938
  *result = _simsimd_sqrt_f32_neon(*result);
871
939
  }
872
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_bf16_t const* b_enum, simsimd_size_t n,
873
- simsimd_distance_t* result) {
940
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
941
+ simsimd_distance_t *result) {
874
942
  simsimd_size_t i = 0;
875
943
  svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
876
944
  svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
877
- simsimd_u16_t const* a = (simsimd_u16_t const*)(a_enum);
878
- simsimd_u16_t const* b = (simsimd_u16_t const*)(b_enum);
945
+ simsimd_u16_t const *a = (simsimd_u16_t const *)(a_enum);
946
+ simsimd_u16_t const *b = (simsimd_u16_t const *)(b_enum);
879
947
  do {
880
948
  svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
881
949
  svuint16_t a_vec = svld1_u16(pg_vec, a + i);
@@ -899,14 +967,14 @@ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_
899
967
  *result = d2;
900
968
  }
901
969
 
902
- SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_bf16_t const* b_enum, simsimd_size_t n,
903
- simsimd_distance_t* result) {
970
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
971
+ simsimd_distance_t *result) {
904
972
  simsimd_size_t i = 0;
905
973
  svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
906
974
  svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
907
975
  svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
908
- simsimd_bf16_for_arm_simd_t const* a = (simsimd_bf16_for_arm_simd_t const*)(a_enum);
909
- simsimd_bf16_for_arm_simd_t const* b = (simsimd_bf16_for_arm_simd_t const*)(b_enum);
976
+ simsimd_bf16_for_arm_simd_t const *a = (simsimd_bf16_for_arm_simd_t const *)(a_enum);
977
+ simsimd_bf16_for_arm_simd_t const *b = (simsimd_bf16_for_arm_simd_t const *)(b_enum);
910
978
  do {
911
979
  svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
912
980
  svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
@@ -926,9 +994,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_b
926
994
  #pragma clang attribute pop
927
995
  #pragma GCC pop_options
928
996
  #endif // SIMSIMD_TARGET_SVE_BF16
929
- #endif // SIMSIMD_TARGET_ARM
997
+ #endif // _SIMSIMD_TARGET_ARM
930
998
 
931
- #if SIMSIMD_TARGET_X86
999
+ #if _SIMSIMD_TARGET_X86
932
1000
  #if SIMSIMD_TARGET_HASWELL
933
1001
  #pragma GCC push_options
934
1002
  #pragma GCC target("avx2")
@@ -945,8 +1013,7 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f
945
1013
  simsimd_f64_t b2) {
946
1014
 
947
1015
  // If both vectors have magnitude 0, the distance is 0.
948
- if (a2 == 0 && b2 == 0)
949
- return 0;
1016
+ if (a2 == 0 && b2 == 0) return 0;
950
1017
  // If any one of the vectors is 0, the square root of the product is 0,
951
1018
  // the division is illformed, and the result is 1.
952
1019
  else if (ab == 0)
@@ -974,8 +1041,7 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f
974
1041
  simsimd_f32_t b2) {
975
1042
 
976
1043
  // If both vectors have magnitude 0, the distance is 0.
977
- if (a2 == 0.0f && b2 == 0.0f)
978
- return 0.0f;
1044
+ if (a2 == 0.0f && b2 == 0.0f) return 0.0f;
979
1045
  // If any one of the vectors is 0, the square root of the product is 0,
980
1046
  // the division is illformed, and the result is 1.
981
1047
  else if (ab == 0.0f)
@@ -1006,21 +1072,21 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f
1006
1072
  #pragma clang attribute pop
1007
1073
  #pragma GCC pop_options
1008
1074
  #endif // SIMSIMD_TARGET_HASWELL
1009
- #endif // SIMSIMD_TARGET_X86
1075
+ #endif // _SIMSIMD_TARGET_X86
1010
1076
 
1011
- #if SIMSIMD_TARGET_X86
1077
+ #if _SIMSIMD_TARGET_X86
1012
1078
  #if SIMSIMD_TARGET_HASWELL
1013
1079
  #pragma GCC push_options
1014
1080
  #pragma GCC target("avx2", "f16c", "fma")
1015
1081
  #pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
1016
1082
 
1017
- SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1018
- simsimd_distance_t* result) {
1083
+ SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1084
+ simsimd_distance_t *result) {
1019
1085
  simsimd_l2sq_f16_haswell(a, b, n, result);
1020
1086
  *result = _simsimd_sqrt_f32_haswell(*result);
1021
1087
  }
1022
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1023
- simsimd_distance_t* result) {
1088
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1089
+ simsimd_distance_t *result) {
1024
1090
  __m256 a_vec, b_vec;
1025
1091
  __m256 d2_vec = _mm256_setzero_ps();
1026
1092
 
@@ -1029,21 +1095,21 @@ simsimd_l2sq_f16_haswell_cycle:
1029
1095
  a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1030
1096
  b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1031
1097
  n = 0;
1032
- } else {
1033
- a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
1034
- b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
1098
+ }
1099
+ else {
1100
+ a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
1101
+ b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
1035
1102
  n -= 8, a += 8, b += 8;
1036
1103
  }
1037
1104
  __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1038
1105
  d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1039
- if (n)
1040
- goto simsimd_l2sq_f16_haswell_cycle;
1106
+ if (n) goto simsimd_l2sq_f16_haswell_cycle;
1041
1107
 
1042
1108
  *result = _simsimd_reduce_f32x8_haswell(d2_vec);
1043
1109
  }
1044
1110
 
1045
- SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1046
- simsimd_distance_t* result) {
1111
+ SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1112
+ simsimd_distance_t *result) {
1047
1113
  __m256 a_vec, b_vec;
1048
1114
  __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
1049
1115
 
@@ -1052,16 +1118,16 @@ simsimd_cos_f16_haswell_cycle:
1052
1118
  a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
1053
1119
  b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
1054
1120
  n = 0;
1055
- } else {
1056
- a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)a));
1057
- b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)b));
1121
+ }
1122
+ else {
1123
+ a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
1124
+ b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
1058
1125
  n -= 8, a += 8, b += 8;
1059
1126
  }
1060
1127
  ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1061
1128
  a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1062
1129
  b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1063
- if (n)
1064
- goto simsimd_cos_f16_haswell_cycle;
1130
+ if (n) goto simsimd_cos_f16_haswell_cycle;
1065
1131
 
1066
1132
  simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1067
1133
  simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
@@ -1069,13 +1135,13 @@ simsimd_cos_f16_haswell_cycle:
1069
1135
  *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1070
1136
  }
1071
1137
 
1072
- SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1073
- simsimd_distance_t* result) {
1138
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1139
+ simsimd_distance_t *result) {
1074
1140
  simsimd_l2sq_bf16_haswell(a, b, n, result);
1075
1141
  *result = _simsimd_sqrt_f32_haswell(*result);
1076
1142
  }
1077
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1078
- simsimd_distance_t* result) {
1143
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1144
+ simsimd_distance_t *result) {
1079
1145
  __m256 a_vec, b_vec;
1080
1146
  __m256 d2_vec = _mm256_setzero_ps();
1081
1147
 
@@ -1084,21 +1150,21 @@ simsimd_l2sq_bf16_haswell_cycle:
1084
1150
  a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1085
1151
  b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1086
1152
  n = 0;
1087
- } else {
1088
- a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)a));
1089
- b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)b));
1153
+ }
1154
+ else {
1155
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
1156
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
1090
1157
  n -= 8, a += 8, b += 8;
1091
1158
  }
1092
1159
  __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
1093
1160
  d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
1094
- if (n)
1095
- goto simsimd_l2sq_bf16_haswell_cycle;
1161
+ if (n) goto simsimd_l2sq_bf16_haswell_cycle;
1096
1162
 
1097
1163
  *result = _simsimd_reduce_f32x8_haswell(d2_vec);
1098
1164
  }
1099
1165
 
1100
- SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1101
- simsimd_distance_t* result) {
1166
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1167
+ simsimd_distance_t *result) {
1102
1168
  __m256 a_vec, b_vec;
1103
1169
  __m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
1104
1170
 
@@ -1107,16 +1173,16 @@ simsimd_cos_bf16_haswell_cycle:
1107
1173
  a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
1108
1174
  b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
1109
1175
  n = 0;
1110
- } else {
1111
- a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)a));
1112
- b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_loadu_si128((__m128i const*)b));
1176
+ }
1177
+ else {
1178
+ a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
1179
+ b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
1113
1180
  n -= 8, a += 8, b += 8;
1114
1181
  }
1115
1182
  ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
1116
1183
  a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
1117
1184
  b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
1118
- if (n)
1119
- goto simsimd_cos_bf16_haswell_cycle;
1185
+ if (n) goto simsimd_cos_bf16_haswell_cycle;
1120
1186
 
1121
1187
  simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
1122
1188
  simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
@@ -1124,105 +1190,197 @@ simsimd_cos_bf16_haswell_cycle:
1124
1190
  *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1125
1191
  }
1126
1192
 
1127
- SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1128
- simsimd_distance_t* result) {
1193
+ SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1194
+ simsimd_distance_t *result) {
1129
1195
  simsimd_l2sq_i8_haswell(a, b, n, result);
1130
1196
  *result = _simsimd_sqrt_f32_haswell(*result);
1131
1197
  }
1132
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1133
- simsimd_distance_t* result) {
1198
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1199
+ simsimd_distance_t *result) {
1134
1200
 
1135
- __m256i d2_high_vec = _mm256_setzero_si256();
1136
- __m256i d2_low_vec = _mm256_setzero_si256();
1201
+ __m256i d2_i32_low_vec = _mm256_setzero_si256();
1202
+ __m256i d2_i32_high_vec = _mm256_setzero_si256();
1137
1203
 
1138
1204
  simsimd_size_t i = 0;
1139
1205
  for (; i + 32 <= n; i += 32) {
1140
- __m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
1141
- __m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
1206
+ __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1207
+ __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1208
+
1209
+ // Sign extend `i8` to `i16`
1210
+ __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_i8_vec));
1211
+ __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1212
+ __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_i8_vec));
1213
+ __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1214
+
1215
+ // Subtract
1216
+ // After this we will be squaring the values. The sign will be dropped
1217
+ // and each difference will be in the range [0, 255].
1218
+ __m256i d_i16_low_vec = _mm256_sub_epi16(a_i16_low_vec, b_i16_low_vec);
1219
+ __m256i d_i16_high_vec = _mm256_sub_epi16(a_i16_high_vec, b_i16_high_vec);
1220
+
1221
+ // Accumulate into `i32` vectors
1222
+ d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
1223
+ d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
1224
+ }
1142
1225
 
1143
- // Sign extend int8 to int16
1144
- __m256i a_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_vec));
1145
- __m256i a_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
1146
- __m256i b_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_vec));
1147
- __m256i b_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
1226
+ // Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
1227
+ int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
1148
1228
 
1149
- // Subtract and multiply
1150
- __m256i d_low = _mm256_sub_epi16(a_low, b_low);
1151
- __m256i d_high = _mm256_sub_epi16(a_high, b_high);
1152
- __m256i d2_low_part = _mm256_madd_epi16(d_low, d_low);
1153
- __m256i d2_high_part = _mm256_madd_epi16(d_high, d_high);
1229
+ // Take care of the tail:
1230
+ for (; i < n; ++i) {
1231
+ int n = (int)(a[i]) - b[i];
1232
+ d2 += n * n;
1233
+ }
1154
1234
 
1155
- // Accumulate into int32 vectors
1156
- d2_low_vec = _mm256_add_epi32(d2_low_vec, d2_low_part);
1157
- d2_high_vec = _mm256_add_epi32(d2_high_vec, d2_high_part);
1235
+ *result = (simsimd_f64_t)d2;
1236
+ }
1237
+
1238
+ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1239
+ simsimd_distance_t *result) {
1240
+
1241
+ __m256i ab_i32_low_vec = _mm256_setzero_si256();
1242
+ __m256i ab_i32_high_vec = _mm256_setzero_si256();
1243
+ __m256i a2_i32_low_vec = _mm256_setzero_si256();
1244
+ __m256i a2_i32_high_vec = _mm256_setzero_si256();
1245
+ __m256i b2_i32_low_vec = _mm256_setzero_si256();
1246
+ __m256i b2_i32_high_vec = _mm256_setzero_si256();
1247
+
1248
+ // AVX2 has no instructions for 8-bit signed integer dot-products,
1249
+ // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1250
+ // So we need to normalize the first vector to its absolute value,
1251
+ // and shift the product sign into the second vector.
1252
+ //
1253
+ // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1254
+ // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1255
+ // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1256
+ //
1257
+ // The problem with this approach, however, is the `-128` value in the second vector.
1258
+ // Flipping it's sign will do nothing, and the result will be incorrect.
1259
+ // This can easily lead to noticeable numerical errors in the final result.
1260
+ simsimd_size_t i = 0;
1261
+ for (; i + 32 <= n; i += 32) {
1262
+ __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1263
+ __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1264
+
1265
+ // Unpack `int8` to `int16`
1266
+ __m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
1267
+ __m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
1268
+ __m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
1269
+ __m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
1270
+
1271
+ // Multiply and accumulate as `int16`, accumulate products as `int32`:
1272
+ ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1273
+ ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1274
+ a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
1275
+ a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
1276
+ b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
1277
+ b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
1158
1278
  }
1159
1279
 
1160
- // Accumulate the 32-bit integers from `d2_high_vec` and `d2_low_vec`
1161
- __m256i d2_vec = _mm256_add_epi32(d2_low_vec, d2_high_vec);
1162
- __m128i d2_sum = _mm_add_epi32(_mm256_extracti128_si256(d2_vec, 0), _mm256_extracti128_si256(d2_vec, 1));
1163
- d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
1164
- d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
1165
- int d2 = _mm_extract_epi32(d2_sum, 0);
1280
+ // Further reduce to a single sum for each vector
1281
+ int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1282
+ int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
1283
+ int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
1166
1284
 
1167
1285
  // Take care of the tail:
1168
1286
  for (; i < n; ++i) {
1169
- int n = a[i] - b[i];
1287
+ int ai = a[i], bi = b[i];
1288
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1289
+ }
1290
+
1291
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1292
+ }
1293
+
1294
+ SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1295
+ simsimd_distance_t *result) {
1296
+ simsimd_l2sq_u8_haswell(a, b, n, result);
1297
+ *result = _simsimd_sqrt_f32_haswell(*result);
1298
+ }
1299
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1300
+ simsimd_distance_t *result) {
1301
+
1302
+ __m256i d2_i32_low_vec = _mm256_setzero_si256();
1303
+ __m256i d2_i32_high_vec = _mm256_setzero_si256();
1304
+ __m256i const zeros_vec = _mm256_setzero_si256();
1305
+
1306
+ simsimd_size_t i = 0;
1307
+ for (; i + 32 <= n; i += 32) {
1308
+ __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1309
+ __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1310
+
1311
+ // Substracting unsigned vectors in AVX2 is done by saturating subtraction:
1312
+ __m256i d_u8_vec = _mm256_or_si256(_mm256_subs_epu8(a_u8_vec, b_u8_vec), _mm256_subs_epu8(b_u8_vec, a_u8_vec));
1313
+
1314
+ // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1315
+ // instructions instead of extracts, as they are much faster and more efficient.
1316
+ __m256i d_i16_low_vec = _mm256_unpacklo_epi8(d_u8_vec, zeros_vec);
1317
+ __m256i d_i16_high_vec = _mm256_unpackhi_epi8(d_u8_vec, zeros_vec);
1318
+
1319
+ // Multiply and accumulate at `int16` level, accumulate at `int32` level:
1320
+ d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
1321
+ d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
1322
+ }
1323
+
1324
+ // Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
1325
+ int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
1326
+
1327
+ // Take care of the tail:
1328
+ for (; i < n; ++i) {
1329
+ int n = (int)(a[i]) - b[i];
1170
1330
  d2 += n * n;
1171
1331
  }
1172
1332
 
1173
1333
  *result = (simsimd_f64_t)d2;
1174
1334
  }
1175
1335
 
1176
- SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1177
- simsimd_distance_t* result) {
1336
+ SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1337
+ simsimd_distance_t *result) {
1178
1338
 
1179
- __m256i ab_low_vec = _mm256_setzero_si256();
1180
- __m256i ab_high_vec = _mm256_setzero_si256();
1181
- __m256i a2_low_vec = _mm256_setzero_si256();
1182
- __m256i a2_high_vec = _mm256_setzero_si256();
1183
- __m256i b2_low_vec = _mm256_setzero_si256();
1184
- __m256i b2_high_vec = _mm256_setzero_si256();
1339
+ __m256i ab_i32_low_vec = _mm256_setzero_si256();
1340
+ __m256i ab_i32_high_vec = _mm256_setzero_si256();
1341
+ __m256i a2_i32_low_vec = _mm256_setzero_si256();
1342
+ __m256i a2_i32_high_vec = _mm256_setzero_si256();
1343
+ __m256i b2_i32_low_vec = _mm256_setzero_si256();
1344
+ __m256i b2_i32_high_vec = _mm256_setzero_si256();
1345
+ __m256i const zeros_vec = _mm256_setzero_si256();
1185
1346
 
1347
+ // AVX2 has no instructions for 8-bit signed integer dot-products,
1348
+ // but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
1349
+ // So we need to normalize the first vector to its absolute value,
1350
+ // and shift the product sign into the second vector.
1351
+ //
1352
+ // __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
1353
+ // __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
1354
+ // __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
1355
+ //
1356
+ // The problem with this approach, however, is the `-128` value in the second vector.
1357
+ // Flipping it's sign will do nothing, and the result will be incorrect.
1358
+ // This can easily lead to noticeable numerical errors in the final result.
1186
1359
  simsimd_size_t i = 0;
1187
1360
  for (; i + 32 <= n; i += 32) {
1188
- __m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
1189
- __m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
1361
+ __m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
1362
+ __m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
1190
1363
 
1191
- // Unpack `int8` to `int16`
1192
- __m256i a_low_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 0));
1193
- __m256i a_high_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
1194
- __m256i b_low_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 0));
1195
- __m256i b_high_16 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
1364
+ // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
1365
+ // instructions instead of extracts, as they are much faster and more efficient.
1366
+ __m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
1367
+ __m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
1368
+ __m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
1369
+ __m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
1196
1370
 
1197
1371
  // Multiply and accumulate as `int16`, accumulate products as `int32`
1198
- ab_low_vec = _mm256_add_epi32(ab_low_vec, _mm256_madd_epi16(a_low_16, b_low_16));
1199
- ab_high_vec = _mm256_add_epi32(ab_high_vec, _mm256_madd_epi16(a_high_16, b_high_16));
1200
- a2_low_vec = _mm256_add_epi32(a2_low_vec, _mm256_madd_epi16(a_low_16, a_low_16));
1201
- a2_high_vec = _mm256_add_epi32(a2_high_vec, _mm256_madd_epi16(a_high_16, a_high_16));
1202
- b2_low_vec = _mm256_add_epi32(b2_low_vec, _mm256_madd_epi16(b_low_16, b_low_16));
1203
- b2_high_vec = _mm256_add_epi32(b2_high_vec, _mm256_madd_epi16(b_high_16, b_high_16));
1204
- }
1205
-
1206
- // Horizontal sum across the 256-bit register
1207
- __m256i ab_vec = _mm256_add_epi32(ab_low_vec, ab_high_vec);
1208
- __m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
1209
- ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
1210
- ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
1211
-
1212
- __m256i a2_vec = _mm256_add_epi32(a2_low_vec, a2_high_vec);
1213
- __m128i a2_sum = _mm_add_epi32(_mm256_extracti128_si256(a2_vec, 0), _mm256_extracti128_si256(a2_vec, 1));
1214
- a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
1215
- a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
1216
-
1217
- __m256i b2_vec = _mm256_add_epi32(b2_low_vec, b2_high_vec);
1218
- __m128i b2_sum = _mm_add_epi32(_mm256_extracti128_si256(b2_vec, 0), _mm256_extracti128_si256(b2_vec, 1));
1219
- b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
1220
- b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
1372
+ ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
1373
+ ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
1374
+ a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
1375
+ a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
1376
+ b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
1377
+ b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
1378
+ }
1221
1379
 
1222
1380
  // Further reduce to a single sum for each vector
1223
- int ab = _mm_extract_epi32(ab_sum, 0);
1224
- int a2 = _mm_extract_epi32(a2_sum, 0);
1225
- int b2 = _mm_extract_epi32(b2_sum, 0);
1381
+ int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
1382
+ int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
1383
+ int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
1226
1384
 
1227
1385
  // Take care of the tail:
1228
1386
  for (; i < n; ++i) {
@@ -1233,13 +1391,13 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
1233
1391
  *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1234
1392
  }
1235
1393
 
1236
- SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1237
- simsimd_distance_t* result) {
1394
+ SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1395
+ simsimd_distance_t *result) {
1238
1396
  simsimd_l2sq_f32_haswell(a, b, n, result);
1239
1397
  *result = _simsimd_sqrt_f32_haswell(*result);
1240
1398
  }
1241
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1242
- simsimd_distance_t* result) {
1399
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1400
+ simsimd_distance_t *result) {
1243
1401
 
1244
1402
  __m256 d2_vec = _mm256_setzero_ps();
1245
1403
  simsimd_size_t i = 0;
@@ -1259,8 +1417,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
1259
1417
  *result = d2;
1260
1418
  }
1261
1419
 
1262
- SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1263
- simsimd_distance_t* result) {
1420
+ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1421
+ simsimd_distance_t *result) {
1264
1422
 
1265
1423
  __m256 ab_vec = _mm256_setzero_ps();
1266
1424
  __m256 a2_vec = _mm256_setzero_ps();
@@ -1284,6 +1442,57 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
1284
1442
  *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1285
1443
  }
1286
1444
 
1445
+ SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1446
+ simsimd_distance_t *result) {
1447
+ simsimd_l2sq_f64_haswell(a, b, n, result);
1448
+ *result = _simsimd_sqrt_f64_haswell(*result);
1449
+ }
1450
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1451
+ simsimd_distance_t *result) {
1452
+
1453
+ __m256d d2_vec = _mm256_setzero_pd();
1454
+ simsimd_size_t i = 0;
1455
+ for (; i + 4 <= n; i += 4) {
1456
+ __m256d a_vec = _mm256_loadu_pd(a + i);
1457
+ __m256d b_vec = _mm256_loadu_pd(b + i);
1458
+ __m256d d_vec = _mm256_sub_pd(a_vec, b_vec);
1459
+ d2_vec = _mm256_fmadd_pd(d_vec, d_vec, d2_vec);
1460
+ }
1461
+
1462
+ simsimd_f64_t d2 = _simsimd_reduce_f64x4_haswell(d2_vec);
1463
+ for (; i < n; ++i) {
1464
+ simsimd_f64_t d = a[i] - b[i];
1465
+ d2 += d * d;
1466
+ }
1467
+
1468
+ *result = d2;
1469
+ }
1470
+
1471
+ SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1472
+ simsimd_distance_t *result) {
1473
+
1474
+ __m256d ab_vec = _mm256_setzero_pd();
1475
+ __m256d a2_vec = _mm256_setzero_pd();
1476
+ __m256d b2_vec = _mm256_setzero_pd();
1477
+ simsimd_size_t i = 0;
1478
+ for (; i + 4 <= n; i += 4) {
1479
+ __m256d a_vec = _mm256_loadu_pd(a + i);
1480
+ __m256d b_vec = _mm256_loadu_pd(b + i);
1481
+ ab_vec = _mm256_fmadd_pd(a_vec, b_vec, ab_vec);
1482
+ a2_vec = _mm256_fmadd_pd(a_vec, a_vec, a2_vec);
1483
+ b2_vec = _mm256_fmadd_pd(b_vec, b_vec, b2_vec);
1484
+ }
1485
+
1486
+ simsimd_f64_t ab = _simsimd_reduce_f64x4_haswell(ab_vec);
1487
+ simsimd_f64_t a2 = _simsimd_reduce_f64x4_haswell(a2_vec);
1488
+ simsimd_f64_t b2 = _simsimd_reduce_f64x4_haswell(b2_vec);
1489
+ for (; i < n; ++i) {
1490
+ simsimd_f64_t ai = a[i], bi = b[i];
1491
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
1492
+ }
1493
+ *result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
1494
+ }
1495
+
1287
1496
  #pragma clang attribute pop
1288
1497
  #pragma GCC pop_options
1289
1498
  #endif // SIMSIMD_TARGET_HASWELL
@@ -1293,13 +1502,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
1293
1502
  #pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
1294
1503
  #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
1295
1504
 
1296
- SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1297
- simsimd_distance_t* result) {
1505
+ SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1506
+ simsimd_distance_t *result) {
1298
1507
  simsimd_l2sq_f32_skylake(a, b, n, result);
1299
1508
  *result = _simsimd_sqrt_f64_haswell(*result);
1300
1509
  }
1301
- SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1302
- simsimd_distance_t* result) {
1510
+ SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1511
+ simsimd_distance_t *result) {
1303
1512
  __m512 d2_vec = _mm512_setzero();
1304
1513
  __m512 a_vec, b_vec;
1305
1514
 
@@ -1309,15 +1518,15 @@ simsimd_l2sq_f32_skylake_cycle:
1309
1518
  a_vec = _mm512_maskz_loadu_ps(mask, a);
1310
1519
  b_vec = _mm512_maskz_loadu_ps(mask, b);
1311
1520
  n = 0;
1312
- } else {
1521
+ }
1522
+ else {
1313
1523
  a_vec = _mm512_loadu_ps(a);
1314
1524
  b_vec = _mm512_loadu_ps(b);
1315
1525
  a += 16, b += 16, n -= 16;
1316
1526
  }
1317
1527
  __m512 d_vec = _mm512_sub_ps(a_vec, b_vec);
1318
1528
  d2_vec = _mm512_fmadd_ps(d_vec, d_vec, d2_vec);
1319
- if (n)
1320
- goto simsimd_l2sq_f32_skylake_cycle;
1529
+ if (n) goto simsimd_l2sq_f32_skylake_cycle;
1321
1530
 
1322
1531
  *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1323
1532
  }
@@ -1326,8 +1535,7 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f
1326
1535
  simsimd_f64_t b2) {
1327
1536
 
1328
1537
  // If both vectors have magnitude 0, the distance is 0.
1329
- if (a2 == 0 && b2 == 0)
1330
- return 0;
1538
+ if (a2 == 0 && b2 == 0) return 0;
1331
1539
  // If any one of the vectors is 0, the square root of the product is 0,
1332
1540
  // the division is illformed, and the result is 1.
1333
1541
  else if (ab == 0)
@@ -1366,8 +1574,8 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f
1366
1574
  return result > 0 ? result : 0;
1367
1575
  }
1368
1576
 
1369
- SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n,
1370
- simsimd_distance_t* result) {
1577
+ SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
1578
+ simsimd_distance_t *result) {
1371
1579
  __m512 ab_vec = _mm512_setzero();
1372
1580
  __m512 a2_vec = _mm512_setzero();
1373
1581
  __m512 b2_vec = _mm512_setzero();
@@ -1379,7 +1587,8 @@ simsimd_cos_f32_skylake_cycle:
1379
1587
  a_vec = _mm512_maskz_loadu_ps(mask, a);
1380
1588
  b_vec = _mm512_maskz_loadu_ps(mask, b);
1381
1589
  n = 0;
1382
- } else {
1590
+ }
1591
+ else {
1383
1592
  a_vec = _mm512_loadu_ps(a);
1384
1593
  b_vec = _mm512_loadu_ps(b);
1385
1594
  a += 16, b += 16, n -= 16;
@@ -1387,8 +1596,7 @@ simsimd_cos_f32_skylake_cycle:
1387
1596
  ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
1388
1597
  a2_vec = _mm512_fmadd_ps(a_vec, a_vec, a2_vec);
1389
1598
  b2_vec = _mm512_fmadd_ps(b_vec, b_vec, b2_vec);
1390
- if (n)
1391
- goto simsimd_cos_f32_skylake_cycle;
1599
+ if (n) goto simsimd_cos_f32_skylake_cycle;
1392
1600
 
1393
1601
  simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1394
1602
  simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
@@ -1396,13 +1604,13 @@ simsimd_cos_f32_skylake_cycle:
1396
1604
  *result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
1397
1605
  }
1398
1606
 
1399
- SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1400
- simsimd_distance_t* result) {
1607
+ SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1608
+ simsimd_distance_t *result) {
1401
1609
  simsimd_l2sq_f64_skylake(a, b, n, result);
1402
1610
  *result = _simsimd_sqrt_f64_haswell(*result);
1403
1611
  }
1404
- SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1405
- simsimd_distance_t* result) {
1612
+ SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1613
+ simsimd_distance_t *result) {
1406
1614
  __m512d d2_vec = _mm512_setzero_pd();
1407
1615
  __m512d a_vec, b_vec;
1408
1616
 
@@ -1412,21 +1620,21 @@ simsimd_l2sq_f64_skylake_cycle:
1412
1620
  a_vec = _mm512_maskz_loadu_pd(mask, a);
1413
1621
  b_vec = _mm512_maskz_loadu_pd(mask, b);
1414
1622
  n = 0;
1415
- } else {
1623
+ }
1624
+ else {
1416
1625
  a_vec = _mm512_loadu_pd(a);
1417
1626
  b_vec = _mm512_loadu_pd(b);
1418
1627
  a += 8, b += 8, n -= 8;
1419
1628
  }
1420
1629
  __m512d d_vec = _mm512_sub_pd(a_vec, b_vec);
1421
1630
  d2_vec = _mm512_fmadd_pd(d_vec, d_vec, d2_vec);
1422
- if (n)
1423
- goto simsimd_l2sq_f64_skylake_cycle;
1631
+ if (n) goto simsimd_l2sq_f64_skylake_cycle;
1424
1632
 
1425
1633
  *result = _mm512_reduce_add_pd(d2_vec);
1426
1634
  }
1427
1635
 
1428
- SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n,
1429
- simsimd_distance_t* result) {
1636
+ SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
1637
+ simsimd_distance_t *result) {
1430
1638
  __m512d ab_vec = _mm512_setzero_pd();
1431
1639
  __m512d a2_vec = _mm512_setzero_pd();
1432
1640
  __m512d b2_vec = _mm512_setzero_pd();
@@ -1438,7 +1646,8 @@ simsimd_cos_f64_skylake_cycle:
1438
1646
  a_vec = _mm512_maskz_loadu_pd(mask, a);
1439
1647
  b_vec = _mm512_maskz_loadu_pd(mask, b);
1440
1648
  n = 0;
1441
- } else {
1649
+ }
1650
+ else {
1442
1651
  a_vec = _mm512_loadu_pd(a);
1443
1652
  b_vec = _mm512_loadu_pd(b);
1444
1653
  a += 8, b += 8, n -= 8;
@@ -1446,8 +1655,7 @@ simsimd_cos_f64_skylake_cycle:
1446
1655
  ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
1447
1656
  a2_vec = _mm512_fmadd_pd(a_vec, a_vec, a2_vec);
1448
1657
  b2_vec = _mm512_fmadd_pd(b_vec, b_vec, b2_vec);
1449
- if (n)
1450
- goto simsimd_cos_f64_skylake_cycle;
1658
+ if (n) goto simsimd_cos_f64_skylake_cycle;
1451
1659
 
1452
1660
  simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
1453
1661
  simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
@@ -1462,7 +1670,7 @@ simsimd_cos_f64_skylake_cycle:
1462
1670
  #if SIMSIMD_TARGET_GENOA
1463
1671
  #pragma GCC push_options
1464
1672
  #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
1465
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1673
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
1466
1674
  apply_to = function)
1467
1675
 
1468
1676
  SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
@@ -1517,13 +1725,13 @@ SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i
1517
1725
  return d.ivec;
1518
1726
  }
1519
1727
 
1520
- SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1521
- simsimd_distance_t* result) {
1728
+ SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1729
+ simsimd_distance_t *result) {
1522
1730
  simsimd_l2sq_bf16_genoa(a, b, n, result);
1523
1731
  *result = _simsimd_sqrt_f32_haswell(*result);
1524
1732
  }
1525
- SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1526
- simsimd_distance_t* result) {
1733
+ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1734
+ simsimd_distance_t *result) {
1527
1735
  __m512 d2_vec = _mm512_setzero_ps();
1528
1736
  __m512i a_i16_vec, b_i16_vec, d_i16_vec;
1529
1737
 
@@ -1533,21 +1741,21 @@ simsimd_l2sq_bf16_genoa_cycle:
1533
1741
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1534
1742
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1535
1743
  n = 0;
1536
- } else {
1744
+ }
1745
+ else {
1537
1746
  a_i16_vec = _mm512_loadu_epi16(a);
1538
1747
  b_i16_vec = _mm512_loadu_epi16(b);
1539
1748
  a += 32, b += 32, n -= 32;
1540
1749
  }
1541
1750
  d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
1542
1751
  d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
1543
- if (n)
1544
- goto simsimd_l2sq_bf16_genoa_cycle;
1752
+ if (n) goto simsimd_l2sq_bf16_genoa_cycle;
1545
1753
 
1546
1754
  *result = _simsimd_reduce_f32x16_skylake(d2_vec);
1547
1755
  }
1548
1756
 
1549
- SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
1550
- simsimd_distance_t* result) {
1757
+ SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
1758
+ simsimd_distance_t *result) {
1551
1759
  __m512 ab_vec = _mm512_setzero_ps();
1552
1760
  __m512 a2_vec = _mm512_setzero_ps();
1553
1761
  __m512 b2_vec = _mm512_setzero_ps();
@@ -1559,7 +1767,8 @@ simsimd_cos_bf16_genoa_cycle:
1559
1767
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1560
1768
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1561
1769
  n = 0;
1562
- } else {
1770
+ }
1771
+ else {
1563
1772
  a_i16_vec = _mm512_loadu_epi16(a);
1564
1773
  b_i16_vec = _mm512_loadu_epi16(b);
1565
1774
  a += 32, b += 32, n -= 32;
@@ -1567,8 +1776,7 @@ simsimd_cos_bf16_genoa_cycle:
1567
1776
  ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
1568
1777
  a2_vec = _mm512_dpbf16_ps(a2_vec, (__m512bh)(a_i16_vec), (__m512bh)(a_i16_vec));
1569
1778
  b2_vec = _mm512_dpbf16_ps(b2_vec, (__m512bh)(b_i16_vec), (__m512bh)(b_i16_vec));
1570
- if (n)
1571
- goto simsimd_cos_bf16_genoa_cycle;
1779
+ if (n) goto simsimd_cos_bf16_genoa_cycle;
1572
1780
 
1573
1781
  simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
1574
1782
  simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
@@ -1585,13 +1793,13 @@ simsimd_cos_bf16_genoa_cycle:
1585
1793
  #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
1586
1794
  #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
1587
1795
 
1588
- SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1589
- simsimd_distance_t* result) {
1796
+ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1797
+ simsimd_distance_t *result) {
1590
1798
  simsimd_l2sq_f16_sapphire(a, b, n, result);
1591
1799
  *result = _simsimd_sqrt_f32_haswell(*result);
1592
1800
  }
1593
- SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1594
- simsimd_distance_t* result) {
1801
+ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1802
+ simsimd_distance_t *result) {
1595
1803
  __m512h d2_vec = _mm512_setzero_ph();
1596
1804
  __m512i a_i16_vec, b_i16_vec;
1597
1805
 
@@ -1601,21 +1809,21 @@ simsimd_l2sq_f16_sapphire_cycle:
1601
1809
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1602
1810
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1603
1811
  n = 0;
1604
- } else {
1812
+ }
1813
+ else {
1605
1814
  a_i16_vec = _mm512_loadu_epi16(a);
1606
1815
  b_i16_vec = _mm512_loadu_epi16(b);
1607
1816
  a += 32, b += 32, n -= 32;
1608
1817
  }
1609
1818
  __m512h d_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec));
1610
1819
  d2_vec = _mm512_fmadd_ph(d_vec, d_vec, d2_vec);
1611
- if (n)
1612
- goto simsimd_l2sq_f16_sapphire_cycle;
1820
+ if (n) goto simsimd_l2sq_f16_sapphire_cycle;
1613
1821
 
1614
1822
  *result = _mm512_reduce_add_ph(d2_vec);
1615
1823
  }
1616
1824
 
1617
- SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n,
1618
- simsimd_distance_t* result) {
1825
+ SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
1826
+ simsimd_distance_t *result) {
1619
1827
  __m512h ab_vec = _mm512_setzero_ph();
1620
1828
  __m512h a2_vec = _mm512_setzero_ph();
1621
1829
  __m512h b2_vec = _mm512_setzero_ph();
@@ -1627,7 +1835,8 @@ simsimd_cos_f16_sapphire_cycle:
1627
1835
  a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
1628
1836
  b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
1629
1837
  n = 0;
1630
- } else {
1838
+ }
1839
+ else {
1631
1840
  a_i16_vec = _mm512_loadu_epi16(a);
1632
1841
  b_i16_vec = _mm512_loadu_epi16(b);
1633
1842
  a += 32, b += 32, n -= 32;
@@ -1635,8 +1844,7 @@ simsimd_cos_f16_sapphire_cycle:
1635
1844
  ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
1636
1845
  a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(a_i16_vec), a2_vec);
1637
1846
  b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_i16_vec), _mm512_castsi512_ph(b_i16_vec), b2_vec);
1638
- if (n)
1639
- goto simsimd_cos_f16_sapphire_cycle;
1847
+ if (n) goto simsimd_cos_f16_sapphire_cycle;
1640
1848
 
1641
1849
  simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
1642
1850
  simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
@@ -1651,92 +1859,463 @@ simsimd_cos_f16_sapphire_cycle:
1651
1859
  #if SIMSIMD_TARGET_ICE
1652
1860
  #pragma GCC push_options
1653
1861
  #pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
1654
- #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1862
+ #pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
1655
1863
  apply_to = function)
1656
1864
 
1657
- SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1658
- simsimd_distance_t* result) {
1865
+ SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1866
+ simsimd_distance_t *result) {
1659
1867
  simsimd_l2sq_i8_ice(a, b, n, result);
1660
1868
  *result = _simsimd_sqrt_f32_haswell(*result);
1661
1869
  }
1662
- SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1663
- simsimd_distance_t* result) {
1664
- __m512i d2_i32s_vec = _mm512_setzero_si512();
1665
- __m512i a_vec, b_vec, d_i16s_vec;
1870
+ SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1871
+ simsimd_distance_t *result) {
1872
+ __m512i d2_i32_vec = _mm512_setzero_si512();
1873
+ __m512i a_i16_vec, b_i16_vec, d_i16s_vec;
1666
1874
 
1667
1875
  simsimd_l2sq_i8_ice_cycle:
1668
- if (n < 32) { // TODO: Avoid easrly i16 upcast to step through 64 values at a time
1876
+ if (n < 32) { // TODO: Avoid eaarly i16 upcast to step through 64 values at a time
1669
1877
  __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1670
- a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1671
- b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1878
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1879
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1672
1880
  n = 0;
1673
- } else {
1674
- a_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(a));
1675
- b_vec = _mm512_cvtepi8_epi16(_mm256_loadu_epi8(b));
1881
+ }
1882
+ else {
1883
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
1884
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
1676
1885
  a += 32, b += 32, n -= 32;
1677
1886
  }
1678
- d_i16s_vec = _mm512_sub_epi16(a_vec, b_vec);
1679
- d2_i32s_vec = _mm512_dpwssd_epi32(d2_i32s_vec, d_i16s_vec, d_i16s_vec);
1680
- if (n)
1681
- goto simsimd_l2sq_i8_ice_cycle;
1887
+ d_i16s_vec = _mm512_sub_epi16(a_i16_vec, b_i16_vec);
1888
+ d2_i32_vec = _mm512_dpwssd_epi32(d2_i32_vec, d_i16s_vec, d_i16s_vec);
1889
+ if (n) goto simsimd_l2sq_i8_ice_cycle;
1682
1890
 
1683
- *result = _mm512_reduce_add_epi32(d2_i32s_vec);
1891
+ *result = _mm512_reduce_add_epi32(d2_i32_vec);
1684
1892
  }
1685
1893
 
1686
- SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n,
1687
- simsimd_distance_t* result) {
1688
- __m512i ab_low_i32s_vec = _mm512_setzero_si512();
1689
- __m512i ab_high_i32s_vec = _mm512_setzero_si512();
1690
- __m512i a2_i32s_vec = _mm512_setzero_si512();
1691
- __m512i b2_i32s_vec = _mm512_setzero_si512();
1692
- __m512i a_vec, b_vec;
1693
- __m512i a_abs_vec, b_abs_vec;
1894
+ SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
1895
+ simsimd_distance_t *result) {
1694
1896
 
1897
+ __m512i ab_i32_vec = _mm512_setzero_si512();
1898
+ __m512i a2_i32_vec = _mm512_setzero_si512();
1899
+ __m512i b2_i32_vec = _mm512_setzero_si512();
1900
+ __m512i a_i16_vec, b_i16_vec;
1695
1901
  simsimd_cos_i8_ice_cycle:
1696
- if (n < 64) {
1697
- __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1698
- a_vec = _mm512_maskz_loadu_epi8(mask, a);
1699
- b_vec = _mm512_maskz_loadu_epi8(mask, b);
1902
+ if (n < 32) {
1903
+ __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
1904
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
1905
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
1700
1906
  n = 0;
1701
- } else {
1702
- a_vec = _mm512_loadu_epi8(a);
1703
- b_vec = _mm512_loadu_epi8(b);
1704
- a += 64, b += 64, n -= 64;
1907
+ }
1908
+ else {
1909
+ a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
1910
+ b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
1911
+ a += 32, b += 32, n -= 32;
1705
1912
  }
1706
1913
 
1707
1914
  // We can't directly use the `_mm512_dpbusd_epi32` intrinsic everywhere,
1708
1915
  // as it's asymmetric with respect to the sign of the input arguments:
1916
+ //
1709
1917
  // Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
1710
- // Luckily to compute the squares, we just drop the sign bit of the second argument.
1711
- a_abs_vec = _mm512_abs_epi8(a_vec);
1712
- b_abs_vec = _mm512_abs_epi8(b_vec);
1713
- a2_i32s_vec = _mm512_dpbusds_epi32(a2_i32s_vec, a_abs_vec, a_abs_vec);
1714
- b2_i32s_vec = _mm512_dpbusds_epi32(b2_i32s_vec, b_abs_vec, b_abs_vec);
1918
+ //
1919
+ // To compute the squares, we could just drop the sign bit of the second argument.
1920
+ // But this would lead to big-big problems on values like `-128`!
1921
+ // For dot-products we don't have the luxury of optimizing the sign bit away.
1922
+ // Assuming this is an approximate kernel (with reciprocal square root approximations)
1923
+ // in the end, we can allow clamping the value to [-127, 127] range.
1924
+ //
1925
+ // On Ice Lake:
1926
+ //
1927
+ // 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can only execute on port 0, with 5 cycle latency.
1928
+ // 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also only execute on port 0, with 5 cycle latency.
1929
+ // 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 5, with 5 cycle latency.
1930
+ //
1931
+ // On Zen4 Genoa:
1932
+ //
1933
+ // 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 4 cycle latency.
1934
+ // 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also execute on ports 0 and 1, with 4 cycle latency.
1935
+ // 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 3 cycle latency.
1936
+ //
1937
+ // The old solution was complex replied on 1. and 2.:
1938
+ //
1939
+ // a_i8_abs_vec = _mm512_abs_epi8(a_i8_vec);
1940
+ // b_i8_abs_vec = _mm512_abs_epi8(b_i8_vec);
1941
+ // a2_i32_vec = _mm512_dpbusds_epi32(a2_i32_vec, a_i8_abs_vec, a_i8_abs_vec);
1942
+ // b2_i32_vec = _mm512_dpbusds_epi32(b2_i32_vec, b_i8_abs_vec, b_i8_abs_vec);
1943
+ // ab_i32_low_vec = _mm512_dpwssds_epi32( //
1944
+ // ab_i32_low_vec, //
1945
+ // _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)), //
1946
+ // _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
1947
+ // ab_i32_high_vec = _mm512_dpwssds_epi32( //
1948
+ // ab_i32_high_vec, //
1949
+ // _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)), //
1950
+ // _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
1951
+ //
1952
+ // The new solution is simpler and relies on 3.:
1953
+ ab_i32_vec = _mm512_add_epi32(ab_i32_vec, _mm512_madd_epi16(a_i16_vec, b_i16_vec));
1954
+ a2_i32_vec = _mm512_add_epi32(a2_i32_vec, _mm512_madd_epi16(a_i16_vec, a_i16_vec));
1955
+ b2_i32_vec = _mm512_add_epi32(b2_i32_vec, _mm512_madd_epi16(b_i16_vec, b_i16_vec));
1956
+ if (n) goto simsimd_cos_i8_ice_cycle;
1957
+
1958
+ int ab = _mm512_reduce_add_epi32(ab_i32_vec);
1959
+ int a2 = _mm512_reduce_add_epi32(a2_i32_vec);
1960
+ int b2 = _mm512_reduce_add_epi32(b2_i32_vec);
1961
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1962
+ }
1963
+ SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1964
+ simsimd_distance_t *result) {
1965
+ simsimd_l2sq_u8_ice(a, b, n, result);
1966
+ *result = _simsimd_sqrt_f32_haswell(*result);
1967
+ }
1968
+ SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
1969
+ simsimd_distance_t *result) {
1970
+ __m512i d2_i32_low_vec = _mm512_setzero_si512();
1971
+ __m512i d2_i32_high_vec = _mm512_setzero_si512();
1972
+ __m512i const zeros_vec = _mm512_setzero_si512();
1973
+ __m512i d_i16_low_vec, d_i16_high_vec;
1974
+ __m512i a_u8_vec, b_u8_vec, d_u8_vec;
1975
+
1976
+ simsimd_l2sq_u8_ice_cycle:
1977
+ if (n < 64) {
1978
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
1979
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
1980
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
1981
+ n = 0;
1982
+ }
1983
+ else {
1984
+ a_u8_vec = _mm512_loadu_si512(a);
1985
+ b_u8_vec = _mm512_loadu_si512(b);
1986
+ a += 64, b += 64, n -= 64;
1987
+ }
1988
+
1989
+ // Substracting unsigned vectors in AVX-512 is done by saturating subtraction:
1990
+ d_u8_vec = _mm512_or_si512(_mm512_subs_epu8(a_u8_vec, b_u8_vec), _mm512_subs_epu8(b_u8_vec, a_u8_vec));
1991
+ d_i16_low_vec = _mm512_unpacklo_epi8(d_u8_vec, zeros_vec);
1992
+ d_i16_high_vec = _mm512_unpackhi_epi8(d_u8_vec, zeros_vec);
1993
+
1994
+ // Multiply and accumulate at `int16` level, accumulate at `int32` level:
1995
+ d2_i32_low_vec = _mm512_dpwssd_epi32(d2_i32_low_vec, d_i16_low_vec, d_i16_low_vec);
1996
+ d2_i32_high_vec = _mm512_dpwssd_epi32(d2_i32_high_vec, d_i16_high_vec, d_i16_high_vec);
1997
+ if (n) goto simsimd_l2sq_u8_ice_cycle;
1998
+
1999
+ *result = _mm512_reduce_add_epi32(_mm512_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
2000
+ }
2001
+
2002
+ SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
2003
+ simsimd_distance_t *result) {
2004
+
2005
+ __m512i ab_i32_low_vec = _mm512_setzero_si512();
2006
+ __m512i ab_i32_high_vec = _mm512_setzero_si512();
2007
+ __m512i a2_i32_low_vec = _mm512_setzero_si512();
2008
+ __m512i a2_i32_high_vec = _mm512_setzero_si512();
2009
+ __m512i b2_i32_low_vec = _mm512_setzero_si512();
2010
+ __m512i b2_i32_high_vec = _mm512_setzero_si512();
2011
+ __m512i const zeros_vec = _mm512_setzero_si512();
2012
+ __m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
2013
+ __m512i a_u8_vec, b_u8_vec;
2014
+
2015
+ simsimd_cos_u8_ice_cycle:
2016
+ if (n < 64) {
2017
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
2018
+ a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
2019
+ b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
2020
+ n = 0;
2021
+ }
2022
+ else {
2023
+ a_u8_vec = _mm512_loadu_si512(a);
2024
+ b_u8_vec = _mm512_loadu_si512(b);
2025
+ a += 64, b += 64, n -= 64;
2026
+ }
2027
+
2028
+ // Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
2029
+ // instructions instead of extracts, as they are much faster and more efficient.
2030
+ a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
2031
+ a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
2032
+ b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
2033
+ b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
2034
+
2035
+ // Multiply and accumulate as `int16`, accumulate products as `int32`:
2036
+ ab_i32_low_vec = _mm512_dpwssds_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
2037
+ ab_i32_high_vec = _mm512_dpwssds_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
2038
+ a2_i32_low_vec = _mm512_dpwssds_epi32(a2_i32_low_vec, a_i16_low_vec, a_i16_low_vec);
2039
+ a2_i32_high_vec = _mm512_dpwssds_epi32(a2_i32_high_vec, a_i16_high_vec, a_i16_high_vec);
2040
+ b2_i32_low_vec = _mm512_dpwssds_epi32(b2_i32_low_vec, b_i16_low_vec, b_i16_low_vec);
2041
+ b2_i32_high_vec = _mm512_dpwssds_epi32(b2_i32_high_vec, b_i16_high_vec, b_i16_high_vec);
2042
+ if (n) goto simsimd_cos_u8_ice_cycle;
2043
+
2044
+ int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
2045
+ int a2 = _mm512_reduce_add_epi32(_mm512_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
2046
+ int b2 = _mm512_reduce_add_epi32(_mm512_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
2047
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2048
+ }
2049
+
2050
+ SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2051
+ simsimd_distance_t *result) {
2052
+ simsimd_l2sq_i4x2_ice(a, b, n_words, result);
2053
+ *result = _simsimd_sqrt_f32_haswell(*result);
2054
+ }
2055
+ SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2056
+ simsimd_distance_t *result) {
2057
+
2058
+ // While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
2059
+ // The absolute difference between two 4-bit integers is at most 15 and it is always a `uint4_t` value!
2060
+ // Moreover, it's square is at most 225, which fits into `uint8_t` and can be computed with a single
2061
+ // lookup table. Accumulating those values is similar to checksumming, a piece of cake for SIMD!
2062
+ __m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
2063
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2064
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2065
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2066
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
2067
+ __m512i const u4_squares_lookup_vec = _mm512_set_epi8( //
2068
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2069
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2070
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2071
+ (char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0);
2072
+
2073
+ /// The mask used to take the low nibble of each byte.
2074
+ __m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
2075
+
2076
+ // Temporaries:
2077
+ __m512i a_i4x2_vec, b_i4x2_vec;
2078
+ __m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
2079
+ __m512i d_u8_low_vec, d_u8_high_vec; //! Only the low 4 bits are actually used
2080
+ __m512i d2_u8_low_vec, d2_u8_high_vec;
2081
+ __m512i d2_u16_low_vec, d2_u16_high_vec;
2082
+
2083
+ // Accumulators:
2084
+ __m512i d2_u32_vec = _mm512_setzero_si512();
2085
+
2086
+ simsimd_l2sq_i4x2_ice_cycle:
2087
+ if (n_words < 64) {
2088
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
2089
+ a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
2090
+ b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
2091
+ n_words = 0;
2092
+ }
2093
+ else {
2094
+ a_i4x2_vec = _mm512_loadu_epi8(a);
2095
+ b_i4x2_vec = _mm512_loadu_epi8(b);
2096
+ a += 64, b += 64, n_words -= 64;
2097
+ }
2098
+
2099
+ // Unpack the 4-bit values into 8-bit values with an empty top nibble.
2100
+ a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
2101
+ a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
2102
+ b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
2103
+ b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
2104
+ a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
2105
+ a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
2106
+ b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
2107
+ b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
2108
+
2109
+ // We can implement subtraction with a lookup table, or using `_mm512_sub_epi8`.
2110
+ d_u8_low_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_low_vec, b_i8_low_vec));
2111
+ d_u8_high_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_high_vec, b_i8_high_vec));
2112
+
2113
+ // Now we can use the lookup table to compute the squares of the 4-bit unsigned integers
2114
+ // in the low nibbles of the `d_u8_low_vec` and `d_u8_high_vec` vectors.
2115
+ d2_u8_low_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_low_vec);
2116
+ d2_u8_high_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_high_vec);
2117
+
2118
+ // Aggregating into 16-bit integers, we need to first upcast our 8-bit values to 16 bits.
2119
+ // After that, we will perform one more operation, upcasting further into 32-bit integers.
2120
+ d2_u16_low_vec = //
2121
+ _mm512_add_epi16( //
2122
+ _mm512_unpacklo_epi8(d2_u8_low_vec, _mm512_setzero_si512()),
2123
+ _mm512_unpackhi_epi8(d2_u8_low_vec, _mm512_setzero_si512()));
2124
+ d2_u16_high_vec = //
2125
+ _mm512_add_epi16( //
2126
+ _mm512_unpacklo_epi8(d2_u8_high_vec, _mm512_setzero_si512()),
2127
+ _mm512_unpackhi_epi8(d2_u8_high_vec, _mm512_setzero_si512()));
2128
+ d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_low_vec, _mm512_setzero_si512()));
2129
+ d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_high_vec, _mm512_setzero_si512()));
2130
+ if (n_words) goto simsimd_l2sq_i4x2_ice_cycle;
2131
+
2132
+ // Finally, we can reduce the 16-bit integers to 32-bit integers and sum them up.
2133
+ int d2 = _mm512_reduce_add_epi32(d2_u32_vec);
2134
+ *result = d2;
2135
+ }
2136
+ SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
2137
+ simsimd_distance_t *result) {
2138
+
2139
+ // We need to compose a lookup table for all the scalar products of 4-bit integers.
2140
+ // While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
2141
+ // Practically speaking, the product of two 4-bit signed integers is a 7-bit integer,
2142
+ // as the maximum absolute value of the product is `abs(-8 * -8) == 64`.
2143
+ //
2144
+ // To store 128 possible values of 2^7 bits we only need 128 single-byte scalars,
2145
+ // or just 2x ZMM registers. In that case our lookup will only take `vpermi2b` instruction,
2146
+ // easily inokable with `_mm512_permutex2var_epi8` intrinsic with latency of 6 on Sapphire Rapids.
2147
+ // The problem is converting 2d indices of our symmetric matrix into 1d offsets in the dense array.
2148
+ //
2149
+ // Alternatively, we can take the entire symmetric (16 x 16) matrix of products,
2150
+ // put into 4x ZMM registers, and use it with `_mm512_shuffle_epi8`, remembering
2151
+ // that it can only lookup with 128-bit lanes (16x 8-bit values).
2152
+ // That intrinsic has latency 1, but will need to be repeated and combined with
2153
+ // multiple iterations of `_mm512_shuffle_i64x2` that has latency 3.
2154
+ //
2155
+ // Altenatively, we can get down to 3 cycles per lookup with `vpermb` and `_mm512_permutexvar_epi8` intrinsics.
2156
+ // For that we can split our (16 x 16) matrix into 4x (8 x 8) submatrices, and use 4x ZMM registers.
2157
+ //
2158
+ // Still, all of those solutions are quite heavy compared to two parallel calls to `_mm512_dpbusds_epi32`
2159
+ // for the dot product. But we can still use the `_mm512_permutexvar_epi8` to compute the squares of the
2160
+ // 16 possible `int4_t` values faster.
2161
+ //
2162
+ // Here is how our `int4_t` range looks:
2163
+ //
2164
+ // dec: 0 1 2 3 4 5 6 7 -8 -7 -6 -5 -4 -3 -2 -1
2165
+ // hex: 0 1 2 3 4 5 6 7 8 9 A B C D E F
2166
+ //
2167
+ // Squared:
2168
+ //
2169
+ // dec2: 0 1 4 9 16 25 36 49 64 49 36 25 16 9 4 1
2170
+ // hex2: 0 1 4 9 10 19 24 31 40 31 24 19 10 9 4 1
2171
+ //
2172
+ // Broadcast it to every lane, so that: `square(x) == _mm512_shuffle_epi8(i4_squares_lookup_vec, x)`.
2173
+ __m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
2174
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2175
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2176
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
2177
+ -1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
2178
+ __m512i const i4_squares_lookup_vec = _mm512_set_epi8( //
2179
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2180
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2181
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
2182
+ 1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0);
2183
+
2184
+ /// The mask used to take the low nibble of each byte.
2185
+ __m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
2186
+
2187
+ // Temporaries:
2188
+ __m512i a_i4x2_vec, b_i4x2_vec;
2189
+ __m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
2190
+ __m512i a2_u8_vec, b2_u8_vec;
2191
+
2192
+ // Accumulators:
2193
+ __m512i a2_u16_low_vec = _mm512_setzero_si512();
2194
+ __m512i a2_u16_high_vec = _mm512_setzero_si512();
2195
+ __m512i b2_u16_low_vec = _mm512_setzero_si512();
2196
+ __m512i b2_u16_high_vec = _mm512_setzero_si512();
2197
+ __m512i ab_i32_low_vec = _mm512_setzero_si512();
2198
+ __m512i ab_i32_high_vec = _mm512_setzero_si512();
2199
+
2200
+ simsimd_cos_i4x2_ice_cycle:
2201
+ if (n_words < 64) {
2202
+ __mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
2203
+ a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
2204
+ b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
2205
+ n_words = 0;
2206
+ }
2207
+ else {
2208
+ a_i4x2_vec = _mm512_loadu_epi8(a);
2209
+ b_i4x2_vec = _mm512_loadu_epi8(b);
2210
+ a += 64, b += 64, n_words -= 64;
2211
+ }
2212
+
2213
+ // Unpack the 4-bit values into 8-bit values with an empty top nibble.
2214
+ // For now, they are not really 8-bit integers, as they are not sign-extended.
2215
+ // That part will come later, using the `i4_to_i8_lookup_vec` lookup.
2216
+ a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
2217
+ a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
2218
+ b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
2219
+ b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
2220
+
2221
+ // Compute the squares of the 4-bit integers.
2222
+ // For symmetry we could have used 4 registers, aka "a2_i8_low_vec", "a2_i8_high_vec", "b2_i8_low_vec",
2223
+ // "b2_i8_high_vec". But the largest square value is just 64, so we can safely aggregate into 8-bit unsigned values.
2224
+ a2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_low_vec),
2225
+ _mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_high_vec));
2226
+ b2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_low_vec),
2227
+ _mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_high_vec));
2228
+
2229
+ // We can safely aggregate into just 16-bit sums without overflow, if the vectors have less than:
2230
+ // (2 scalars / byte) * (64 bytes / register) * (256 non-overflowing 8-bit additions in 16-bit intesgers)
2231
+ // = 32'768 dimensions.
2232
+ //
2233
+ // We use saturated addition here to clearly inform in case of overflow.
2234
+ a2_u16_low_vec = _mm512_adds_epu16(a2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
2235
+ a2_u16_high_vec = _mm512_adds_epu16(a2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
2236
+ b2_u16_low_vec = _mm512_adds_epu16(b2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
2237
+ b2_u16_high_vec = _mm512_adds_epu16(b2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
2238
+
2239
+ // Time to perform the proper sign extension of the 4-bit integers to 8-bit integers.
2240
+ a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
2241
+ a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
2242
+ b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
2243
+ b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
1715
2244
 
1716
2245
  // The same trick won't work for the primary dot-product, as the signs vector
1717
- // components may differ significantly. So we have to use two `_mm512_dpwssd_epi32`
2246
+ // components may differ significantly. So we have to use two `_mm512_dpwssds_epi32`
1718
2247
  // intrinsics instead, upcasting four chunks to 16-bit integers beforehand!
1719
- ab_low_i32s_vec = _mm512_dpwssds_epi32( //
1720
- ab_low_i32s_vec, //
1721
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_vec)), //
1722
- _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_vec)));
1723
- ab_high_i32s_vec = _mm512_dpwssds_epi32( //
1724
- ab_high_i32s_vec, //
1725
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_vec, 1)), //
1726
- _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_vec, 1)));
1727
- if (n)
1728
- goto simsimd_cos_i8_ice_cycle;
1729
-
1730
- int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_low_i32s_vec, ab_high_i32s_vec));
1731
- int a2 = _mm512_reduce_add_epi32(a2_i32s_vec);
1732
- int b2 = _mm512_reduce_add_epi32(b2_i32s_vec);
2248
+ // Alternatively, we can flip the signs of the second argument and use `_mm512_dpbusds_epi32`,
2249
+ // but it ends up taking more instructions.
2250
+ ab_i32_low_vec = _mm512_dpwssds_epi32( //
2251
+ ab_i32_low_vec, //
2252
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_low_vec)), //
2253
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_low_vec)));
2254
+ ab_i32_low_vec = _mm512_dpwssds_epi32( //
2255
+ ab_i32_low_vec, //
2256
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_low_vec, 1)), //
2257
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_low_vec, 1)));
2258
+ ab_i32_high_vec = _mm512_dpwssds_epi32( //
2259
+ ab_i32_high_vec, //
2260
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_high_vec)), //
2261
+ _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_high_vec)));
2262
+ ab_i32_high_vec = _mm512_dpwssds_epi32( //
2263
+ ab_i32_high_vec, //
2264
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_high_vec, 1)), //
2265
+ _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_high_vec, 1)));
2266
+ if (n_words) goto simsimd_cos_i4x2_ice_cycle;
2267
+
2268
+ int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
2269
+ unsigned short a2_u16[32], b2_u16[32];
2270
+ _mm512_storeu_si512(a2_u16, _mm512_add_epi16(a2_u16_low_vec, a2_u16_high_vec));
2271
+ unsigned int a2 = 0, b2 = 0;
2272
+ for (int i = 0; i < 32; ++i) a2 += a2_u16[i], b2 += b2_u16[i];
1733
2273
  *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
1734
2274
  }
1735
2275
 
1736
2276
  #pragma clang attribute pop
1737
2277
  #pragma GCC pop_options
1738
2278
  #endif // SIMSIMD_TARGET_ICE
1739
- #endif // SIMSIMD_TARGET_X86
2279
+
2280
+ #if SIMSIMD_TARGET_SIERRA
2281
+ #pragma GCC push_options
2282
+ #pragma GCC target("avx2", "bmi2", "avx2vnni")
2283
+ #pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
2284
+
2285
+ SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
2286
+ simsimd_distance_t *result) {
2287
+
2288
+ __m256i ab_i32_vec = _mm256_setzero_si256();
2289
+ __m256i a2_i32_vec = _mm256_setzero_si256();
2290
+ __m256i b2_i32_vec = _mm256_setzero_si256();
2291
+
2292
+ simsimd_size_t i = 0;
2293
+ for (; i + 32 <= n; i += 32) {
2294
+ __m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
2295
+ __m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
2296
+ ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
2297
+ a2_i32_vec = _mm256_dpbssds_epi32(a2_i32_vec, a_i8_vec, a_i8_vec);
2298
+ b2_i32_vec = _mm256_dpbssds_epi32(b2_i32_vec, b_i8_vec, b_i8_vec);
2299
+ }
2300
+
2301
+ // Further reduce to a single sum for each vector
2302
+ int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
2303
+ int a2 = _simsimd_reduce_i32x8_haswell(a2_i32_vec);
2304
+ int b2 = _simsimd_reduce_i32x8_haswell(b2_i32_vec);
2305
+
2306
+ // Take care of the tail:
2307
+ for (; i < n; ++i) {
2308
+ int ai = a[i], bi = b[i];
2309
+ ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
2310
+ }
2311
+
2312
+ *result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
2313
+ }
2314
+
2315
+ #pragma clang attribute pop
2316
+ #pragma GCC pop_options
2317
+ #endif // SIMSIMD_TARGET_SIERRA
2318
+ #endif // _SIMSIMD_TARGET_X86
1740
2319
 
1741
2320
  #ifdef __cplusplus
1742
2321
  }