usearch 2.15.3 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/cpp/test.cpp +16 -0
- package/include/usearch/index.hpp +21 -5
- package/include/usearch/index_dense.hpp +6 -1
- package/include/usearch/index_plugins.hpp +345 -8
- package/javascript/lib.cpp +17 -5
- package/javascript/usearch.test.js +190 -21
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.git-blame-ignore-revs +2 -0
- package/simsimd/CONTRIBUTING.md +71 -23
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +4 -4
- package/simsimd/MANIFEST.in +2 -1
- package/simsimd/Package.swift +15 -3
- package/simsimd/README.md +104 -35
- package/simsimd/binding.gyp +6 -2
- package/simsimd/build.rs +2 -0
- package/simsimd/c/lib.c +180 -99
- package/simsimd/include/simsimd/binary.h +311 -102
- package/simsimd/include/simsimd/curved.h +134 -135
- package/simsimd/include/simsimd/dot.h +448 -274
- package/simsimd/include/simsimd/elementwise.h +1437 -0
- package/simsimd/include/simsimd/geospatial.h +2 -2
- package/simsimd/include/simsimd/mesh.h +2 -2
- package/simsimd/include/simsimd/probability.h +88 -88
- package/simsimd/include/simsimd/simsimd.h +1157 -772
- package/simsimd/include/simsimd/sparse.h +768 -147
- package/simsimd/include/simsimd/spatial.h +955 -376
- package/simsimd/include/simsimd/types.h +145 -71
- package/simsimd/javascript/lib.c +1 -2
- package/simsimd/package.json +4 -4
- package/simsimd/rust/lib.rs +14 -1
- package/simsimd/scripts/bench.cxx +1070 -0
- package/simsimd/scripts/bench.ipynb +187 -0
- package/simsimd/{javascript → scripts}/bench.js +1 -1
- package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
- package/simsimd/scripts/bench_vectors.py +700 -0
- package/simsimd/scripts/bench_vectors_live.py +213 -0
- package/simsimd/scripts/test.c +155 -0
- package/simsimd/{javascript → scripts}/test.mjs +2 -2
- package/simsimd/scripts/test.py +1410 -0
- package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
- package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
- /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
- /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
|
@@ -13,12 +13,13 @@
|
|
|
13
13
|
* - 32-bit IEEE floating point numbers
|
|
14
14
|
* - 16-bit IEEE floating point numbers
|
|
15
15
|
* - 16-bit brain floating point numbers
|
|
16
|
+
* - 8-bit unsigned integral numbers
|
|
16
17
|
* - 8-bit signed integral numbers
|
|
17
18
|
* - 4-bit signed integral numbers
|
|
18
19
|
*
|
|
19
20
|
* For hardware architectures:
|
|
20
|
-
* - Arm
|
|
21
|
-
* - x86
|
|
21
|
+
* - Arm: NEON, SVE
|
|
22
|
+
* - x86: Haswell, Skylake, Ice Lake, Genoa, Sapphire
|
|
22
23
|
*
|
|
23
24
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
24
25
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -55,6 +56,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf1
|
|
|
55
56
|
SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
56
57
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
57
58
|
SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
59
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
60
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
61
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
58
62
|
|
|
59
63
|
/* Double-precision serial backends for all numeric types.
|
|
60
64
|
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
@@ -67,10 +71,6 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f1
|
|
|
67
71
|
SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
68
72
|
SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
69
73
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
70
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
71
|
-
SIMSIMD_PUBLIC void simsimd_l2_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
72
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
73
|
-
SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
74
74
|
|
|
75
75
|
/* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
|
|
76
76
|
* By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
|
|
@@ -91,6 +91,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_
|
|
|
91
91
|
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
92
92
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
93
93
|
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
94
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
95
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
96
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
94
97
|
|
|
95
98
|
/* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
|
|
96
99
|
* Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
|
|
@@ -117,6 +120,9 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
117
120
|
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
118
121
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
119
122
|
SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
123
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
124
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
125
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
120
126
|
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
121
127
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
122
128
|
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
@@ -126,6 +132,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf
|
|
|
126
132
|
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
127
133
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
128
134
|
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
135
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
136
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
137
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
129
138
|
|
|
130
139
|
/* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
|
|
131
140
|
* Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
|
|
@@ -142,9 +151,15 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_
|
|
|
142
151
|
* Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
|
|
143
152
|
* Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
|
|
144
153
|
*/
|
|
154
|
+
SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
155
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
156
|
+
SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
145
157
|
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
146
158
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
147
159
|
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
160
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
161
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
162
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
148
163
|
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
149
164
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
150
165
|
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
@@ -152,49 +167,51 @@ SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_
|
|
|
152
167
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
153
168
|
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
154
169
|
|
|
170
|
+
/* SIMD-powered backends for AVX-INT8-VNNI extensions on Xeon 6 CPUs, including Sierra Forest and Granite Rapids.
|
|
171
|
+
* The packs many "efficiency" cores into a single socket, avoiding heavy 512-bit operations, and focusing on 256-bit ones.
|
|
172
|
+
*/
|
|
173
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
155
174
|
// clang-format on
|
|
156
175
|
|
|
157
|
-
#define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert)
|
|
158
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const*
|
|
159
|
-
simsimd_##input_type##_t const*
|
|
160
|
-
simsimd_distance_t*
|
|
161
|
-
simsimd_##accumulator_type##_t d2 = 0;
|
|
162
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
163
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i);
|
|
164
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i);
|
|
165
|
-
d2 += (ai - bi) * (ai - bi);
|
|
166
|
-
}
|
|
167
|
-
*result = d2;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
#define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert)
|
|
171
|
-
SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const*
|
|
172
|
-
simsimd_##input_type##_t const*
|
|
173
|
-
simsimd_distance_t*
|
|
174
|
-
simsimd_l2sq_##input_type##_##name(a, b, n, result);
|
|
175
|
-
*result = SIMSIMD_SQRT(*result);
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
#define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert)
|
|
179
|
-
SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const*
|
|
180
|
-
simsimd_##input_type##_t const*
|
|
181
|
-
simsimd_distance_t*
|
|
182
|
-
simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0;
|
|
183
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
184
|
-
simsimd_##accumulator_type##_t ai = load_and_convert(a + i);
|
|
185
|
-
simsimd_##accumulator_type##_t bi = load_and_convert(b + i);
|
|
186
|
-
ab += ai * bi;
|
|
187
|
-
a2 += ai * ai;
|
|
188
|
-
b2 += bi * bi;
|
|
189
|
-
}
|
|
190
|
-
if (a2 == 0 && b2 == 0) {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
*result = unclipped_result > 0 ? unclipped_result : 0; \
|
|
197
|
-
} \
|
|
176
|
+
#define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
|
|
177
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
178
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
179
|
+
simsimd_distance_t *result) { \
|
|
180
|
+
simsimd_##accumulator_type##_t d2 = 0; \
|
|
181
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
182
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
183
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
184
|
+
d2 += (ai - bi) * (ai - bi); \
|
|
185
|
+
} \
|
|
186
|
+
*result = d2; \
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
#define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
|
|
190
|
+
SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
191
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
192
|
+
simsimd_distance_t *result) { \
|
|
193
|
+
simsimd_l2sq_##input_type##_##name(a, b, n, result); \
|
|
194
|
+
*result = SIMSIMD_SQRT(*result); \
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
#define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
|
|
198
|
+
SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
199
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
200
|
+
simsimd_distance_t *result) { \
|
|
201
|
+
simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
|
|
202
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
203
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
204
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
205
|
+
ab += ai * bi; \
|
|
206
|
+
a2 += ai * ai; \
|
|
207
|
+
b2 += bi * bi; \
|
|
208
|
+
} \
|
|
209
|
+
if (a2 == 0 && b2 == 0) { *result = 0; } \
|
|
210
|
+
else if (ab == 0) { *result = 1; } \
|
|
211
|
+
else { \
|
|
212
|
+
simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
|
|
213
|
+
*result = unclipped_result > 0 ? unclipped_result : 0; \
|
|
214
|
+
} \
|
|
198
215
|
}
|
|
199
216
|
|
|
200
217
|
SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
|
|
@@ -217,6 +234,10 @@ SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
|
|
|
217
234
|
SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
|
|
218
235
|
SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
|
|
219
236
|
|
|
237
|
+
SIMSIMD_MAKE_COS(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_u8_serial
|
|
238
|
+
SIMSIMD_MAKE_L2SQ(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_u8_serial
|
|
239
|
+
SIMSIMD_MAKE_L2(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_u8_serial
|
|
240
|
+
|
|
220
241
|
SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
|
|
221
242
|
SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
|
|
222
243
|
SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
|
|
@@ -229,11 +250,7 @@ SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_
|
|
|
229
250
|
SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
|
|
230
251
|
SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
|
|
231
252
|
|
|
232
|
-
|
|
233
|
-
SIMSIMD_MAKE_L2SQ(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_accurate
|
|
234
|
-
SIMSIMD_MAKE_L2(accurate, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_accurate
|
|
235
|
-
|
|
236
|
-
#if SIMSIMD_TARGET_ARM
|
|
253
|
+
#if _SIMSIMD_TARGET_ARM
|
|
237
254
|
#if SIMSIMD_TARGET_NEON
|
|
238
255
|
#pragma GCC push_options
|
|
239
256
|
#pragma GCC target("arch=armv8.2-a+simd")
|
|
@@ -247,10 +264,8 @@ SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
|
|
|
247
264
|
}
|
|
248
265
|
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
|
|
249
266
|
simsimd_f32_t b2) {
|
|
250
|
-
if (a2 == 0 && b2 == 0)
|
|
251
|
-
|
|
252
|
-
if (ab == 0)
|
|
253
|
-
return 1;
|
|
267
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
268
|
+
if (ab == 0) return 1;
|
|
254
269
|
simsimd_f32_t squares_arr[2] = {a2, b2};
|
|
255
270
|
float32x2_t squares = vld1_f32(squares_arr);
|
|
256
271
|
// Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
|
|
@@ -270,10 +285,8 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_
|
|
|
270
285
|
|
|
271
286
|
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
272
287
|
simsimd_f64_t b2) {
|
|
273
|
-
if (a2 == 0 && b2 == 0)
|
|
274
|
-
|
|
275
|
-
if (ab == 0)
|
|
276
|
-
return 1;
|
|
288
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
289
|
+
if (ab == 0) return 1;
|
|
277
290
|
simsimd_f64_t squares_arr[2] = {a2, b2};
|
|
278
291
|
float64x2_t squares = vld1q_f64(squares_arr);
|
|
279
292
|
|
|
@@ -292,13 +305,13 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_
|
|
|
292
305
|
return result > 0 ? result : 0;
|
|
293
306
|
}
|
|
294
307
|
|
|
295
|
-
SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const*
|
|
296
|
-
simsimd_distance_t*
|
|
308
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
309
|
+
simsimd_distance_t *result) {
|
|
297
310
|
simsimd_l2sq_f32_neon(a, b, n, result);
|
|
298
311
|
*result = _simsimd_sqrt_f64_neon(*result);
|
|
299
312
|
}
|
|
300
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const*
|
|
301
|
-
simsimd_distance_t*
|
|
313
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
314
|
+
simsimd_distance_t *result) {
|
|
302
315
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
303
316
|
simsimd_size_t i = 0;
|
|
304
317
|
for (; i + 4 <= n; i += 4) {
|
|
@@ -315,8 +328,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
315
328
|
*result = sum;
|
|
316
329
|
}
|
|
317
330
|
|
|
318
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const*
|
|
319
|
-
simsimd_distance_t*
|
|
331
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
332
|
+
simsimd_distance_t *result) {
|
|
320
333
|
float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
|
|
321
334
|
simsimd_size_t i = 0;
|
|
322
335
|
for (; i + 4 <= n; i += 4) {
|
|
@@ -335,13 +348,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
335
348
|
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
336
349
|
}
|
|
337
350
|
|
|
338
|
-
SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const*
|
|
339
|
-
simsimd_distance_t*
|
|
351
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
352
|
+
simsimd_distance_t *result) {
|
|
340
353
|
simsimd_l2sq_f64_neon(a, b, n, result);
|
|
341
354
|
*result = _simsimd_sqrt_f64_neon(*result);
|
|
342
355
|
}
|
|
343
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const*
|
|
344
|
-
simsimd_distance_t*
|
|
356
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
357
|
+
simsimd_distance_t *result) {
|
|
345
358
|
float64x2_t sum_vec = vdupq_n_f64(0);
|
|
346
359
|
simsimd_size_t i = 0;
|
|
347
360
|
for (; i + 2 <= n; i += 2) {
|
|
@@ -358,8 +371,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t
|
|
|
358
371
|
*result = sum;
|
|
359
372
|
}
|
|
360
373
|
|
|
361
|
-
SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const*
|
|
362
|
-
simsimd_distance_t*
|
|
374
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
375
|
+
simsimd_distance_t *result) {
|
|
363
376
|
float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
|
|
364
377
|
simsimd_size_t i = 0;
|
|
365
378
|
for (; i + 2 <= n; i += 2) {
|
|
@@ -387,13 +400,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t c
|
|
|
387
400
|
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
388
401
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
389
402
|
|
|
390
|
-
SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const*
|
|
391
|
-
simsimd_distance_t*
|
|
403
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
404
|
+
simsimd_distance_t *result) {
|
|
392
405
|
simsimd_l2sq_f16_neon(a, b, n, result);
|
|
393
406
|
*result = _simsimd_sqrt_f32_neon(*result);
|
|
394
407
|
}
|
|
395
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const*
|
|
396
|
-
simsimd_distance_t*
|
|
408
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
409
|
+
simsimd_distance_t *result) {
|
|
397
410
|
float32x4_t a_vec, b_vec;
|
|
398
411
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
399
412
|
|
|
@@ -402,21 +415,21 @@ simsimd_l2sq_f16_neon_cycle:
|
|
|
402
415
|
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
403
416
|
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
404
417
|
n = 0;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
421
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
408
422
|
n -= 4, a += 4, b += 4;
|
|
409
423
|
}
|
|
410
424
|
float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
|
|
411
425
|
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
412
|
-
if (n)
|
|
413
|
-
goto simsimd_l2sq_f16_neon_cycle;
|
|
426
|
+
if (n) goto simsimd_l2sq_f16_neon_cycle;
|
|
414
427
|
|
|
415
428
|
*result = vaddvq_f32(sum_vec);
|
|
416
429
|
}
|
|
417
430
|
|
|
418
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const*
|
|
419
|
-
simsimd_distance_t*
|
|
431
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
432
|
+
simsimd_distance_t *result) {
|
|
420
433
|
float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
|
|
421
434
|
float32x4_t a_vec, b_vec;
|
|
422
435
|
|
|
@@ -425,16 +438,16 @@ simsimd_cos_f16_neon_cycle:
|
|
|
425
438
|
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
426
439
|
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
427
440
|
n = 0;
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
|
|
441
|
+
}
|
|
442
|
+
else {
|
|
443
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
444
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
431
445
|
n -= 4, a += 4, b += 4;
|
|
432
446
|
}
|
|
433
447
|
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
434
448
|
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
435
449
|
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
436
|
-
if (n)
|
|
437
|
-
goto simsimd_cos_f16_neon_cycle;
|
|
450
|
+
if (n) goto simsimd_cos_f16_neon_cycle;
|
|
438
451
|
|
|
439
452
|
simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
|
|
440
453
|
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
@@ -449,8 +462,8 @@ simsimd_cos_f16_neon_cycle:
|
|
|
449
462
|
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
450
463
|
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
451
464
|
|
|
452
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const*
|
|
453
|
-
simsimd_distance_t*
|
|
465
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
466
|
+
simsimd_distance_t *result) {
|
|
454
467
|
|
|
455
468
|
// Similar to `simsimd_cos_i8_neon`, we can use the `BFMMLA` instruction through
|
|
456
469
|
// the `vbfmmlaq_f32` intrinsic to compute matrix products and later drop 1/4 of values.
|
|
@@ -501,29 +514,29 @@ simsimd_cos_bf16_neon_cycle:
|
|
|
501
514
|
a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
502
515
|
b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
503
516
|
n = 0;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
|
|
517
|
+
}
|
|
518
|
+
else {
|
|
519
|
+
a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
|
|
520
|
+
b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
|
|
507
521
|
n -= 8, a += 8, b += 8;
|
|
508
522
|
}
|
|
509
523
|
ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
|
|
510
524
|
a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
|
|
511
525
|
b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
|
|
512
|
-
if (n)
|
|
513
|
-
goto simsimd_cos_bf16_neon_cycle;
|
|
526
|
+
if (n) goto simsimd_cos_bf16_neon_cycle;
|
|
514
527
|
|
|
515
528
|
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
516
529
|
simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
|
|
517
530
|
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
518
531
|
}
|
|
519
532
|
|
|
520
|
-
SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const*
|
|
521
|
-
simsimd_distance_t*
|
|
533
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
534
|
+
simsimd_distance_t *result) {
|
|
522
535
|
simsimd_l2sq_bf16_neon(a, b, n, result);
|
|
523
536
|
*result = _simsimd_sqrt_f64_neon(*result);
|
|
524
537
|
}
|
|
525
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const*
|
|
526
|
-
simsimd_distance_t*
|
|
538
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
539
|
+
simsimd_distance_t *result) {
|
|
527
540
|
float32x4_t diff_high_vec, diff_low_vec;
|
|
528
541
|
float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
|
|
529
542
|
|
|
@@ -534,17 +547,17 @@ simsimd_l2sq_bf16_neon_cycle:
|
|
|
534
547
|
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
|
|
535
548
|
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
|
|
536
549
|
n = 0;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
bfloat16x8_t
|
|
550
|
+
}
|
|
551
|
+
else {
|
|
552
|
+
bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
|
|
553
|
+
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
|
|
540
554
|
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
|
|
541
555
|
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
|
|
542
556
|
n -= 8, a += 8, b += 8;
|
|
543
557
|
}
|
|
544
558
|
sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
|
|
545
559
|
sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
|
|
546
|
-
if (n)
|
|
547
|
-
goto simsimd_l2sq_bf16_neon_cycle;
|
|
560
|
+
if (n) goto simsimd_l2sq_bf16_neon_cycle;
|
|
548
561
|
|
|
549
562
|
*result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
|
|
550
563
|
}
|
|
@@ -558,35 +571,40 @@ simsimd_l2sq_bf16_neon_cycle:
|
|
|
558
571
|
#pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
|
|
559
572
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
|
|
560
573
|
|
|
561
|
-
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const*
|
|
562
|
-
simsimd_distance_t*
|
|
574
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
575
|
+
simsimd_distance_t *result) {
|
|
563
576
|
simsimd_l2sq_i8_neon(a, b, n, result);
|
|
564
577
|
*result = _simsimd_sqrt_f32_neon(*result);
|
|
565
578
|
}
|
|
566
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const*
|
|
567
|
-
simsimd_distance_t*
|
|
568
|
-
|
|
579
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
580
|
+
simsimd_distance_t *result) {
|
|
581
|
+
|
|
582
|
+
// The naive approach is to upcast 8-bit signed integers into 16-bit signed integers
|
|
583
|
+
// for subtraction, then multiply within 16-bit integers and accumulate the results
|
|
584
|
+
// into 32-bit integers. This approach is slow on modern Arm CPUs. On Graviton 4,
|
|
585
|
+
// that approach results in 17 GB/s of throughput, compared to 39 GB/s for `i8`
|
|
586
|
+
// dot-products.
|
|
587
|
+
//
|
|
588
|
+
// Luckily we can use the `vabdq_s8` which technically returns `i8` values, but it's a
|
|
589
|
+
// matter of reinterpret-casting! That approach boosts us to 33 GB/s of throughput.
|
|
590
|
+
uint32x4_t d2_vec = vdupq_n_u32(0);
|
|
569
591
|
simsimd_size_t i = 0;
|
|
570
|
-
for (; i +
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
int32x4_t d_high = vmull_s16(vget_high_s16(d_vec), vget_high_s16(d_vec));
|
|
578
|
-
d2_vec = vaddq_s32(d2_vec, vaddq_s32(d_low, d_high));
|
|
579
|
-
}
|
|
580
|
-
int32_t d2 = vaddvq_s32(d2_vec);
|
|
592
|
+
for (; i + 16 <= n; i += 16) {
|
|
593
|
+
int8x16_t a_vec = vld1q_s8(a + i);
|
|
594
|
+
int8x16_t b_vec = vld1q_s8(b + i);
|
|
595
|
+
uint8x16_t d_vec = vreinterpretq_u8_s8(vabdq_s8(a_vec, b_vec));
|
|
596
|
+
d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
|
|
597
|
+
}
|
|
598
|
+
uint32_t d2 = vaddvq_u32(d2_vec);
|
|
581
599
|
for (; i < n; ++i) {
|
|
582
600
|
int32_t n = (int32_t)a[i] - b[i];
|
|
583
|
-
d2 += n * n;
|
|
601
|
+
d2 += (uint32_t)(n * n);
|
|
584
602
|
}
|
|
585
603
|
*result = d2;
|
|
586
604
|
}
|
|
587
605
|
|
|
588
|
-
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const*
|
|
589
|
-
simsimd_distance_t*
|
|
606
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
607
|
+
simsimd_distance_t *result) {
|
|
590
608
|
|
|
591
609
|
simsimd_size_t i = 0;
|
|
592
610
|
|
|
@@ -704,6 +722,56 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
704
722
|
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
705
723
|
}
|
|
706
724
|
|
|
725
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
726
|
+
simsimd_distance_t *result) {
|
|
727
|
+
simsimd_l2sq_u8_neon(a, b, n, result);
|
|
728
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
729
|
+
}
|
|
730
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
731
|
+
simsimd_distance_t *result) {
|
|
732
|
+
uint32x4_t d2_vec = vdupq_n_u32(0);
|
|
733
|
+
simsimd_size_t i = 0;
|
|
734
|
+
for (; i + 16 <= n; i += 16) {
|
|
735
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
736
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
737
|
+
uint8x16_t d_vec = vabdq_u8(a_vec, b_vec);
|
|
738
|
+
d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
|
|
739
|
+
}
|
|
740
|
+
uint32_t d2 = vaddvq_u32(d2_vec);
|
|
741
|
+
for (; i < n; ++i) {
|
|
742
|
+
int32_t n = (int32_t)a[i] - b[i];
|
|
743
|
+
d2 += (uint32_t)(n * n);
|
|
744
|
+
}
|
|
745
|
+
*result = d2;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
749
|
+
simsimd_distance_t *result) {
|
|
750
|
+
|
|
751
|
+
simsimd_size_t i = 0;
|
|
752
|
+
uint32x4_t ab_vec = vdupq_n_u32(0);
|
|
753
|
+
uint32x4_t a2_vec = vdupq_n_u32(0);
|
|
754
|
+
uint32x4_t b2_vec = vdupq_n_u32(0);
|
|
755
|
+
for (; i + 16 <= n; i += 16) {
|
|
756
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
757
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
758
|
+
ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
|
|
759
|
+
a2_vec = vdotq_u32(a2_vec, a_vec, a_vec);
|
|
760
|
+
b2_vec = vdotq_u32(b2_vec, b_vec, b_vec);
|
|
761
|
+
}
|
|
762
|
+
uint32_t ab = vaddvq_u32(ab_vec);
|
|
763
|
+
uint32_t a2 = vaddvq_u32(a2_vec);
|
|
764
|
+
uint32_t b2 = vaddvq_u32(b2_vec);
|
|
765
|
+
|
|
766
|
+
// Take care of the tail:
|
|
767
|
+
for (; i < n; ++i) {
|
|
768
|
+
uint32_t ai = a[i], bi = b[i];
|
|
769
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
773
|
+
}
|
|
774
|
+
|
|
707
775
|
#pragma clang attribute pop
|
|
708
776
|
#pragma GCC pop_options
|
|
709
777
|
#endif // SIMSIMD_TARGET_NEON_I8
|
|
@@ -713,13 +781,13 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
713
781
|
#pragma GCC target("arch=armv8.2-a+sve")
|
|
714
782
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
|
|
715
783
|
|
|
716
|
-
SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const*
|
|
717
|
-
simsimd_distance_t*
|
|
784
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
785
|
+
simsimd_distance_t *result) {
|
|
718
786
|
simsimd_l2sq_f32_sve(a, b, n, result);
|
|
719
787
|
*result = _simsimd_sqrt_f64_neon(*result);
|
|
720
788
|
}
|
|
721
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const*
|
|
722
|
-
simsimd_distance_t*
|
|
789
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
790
|
+
simsimd_distance_t *result) {
|
|
723
791
|
simsimd_size_t i = 0;
|
|
724
792
|
svfloat32_t d2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
725
793
|
do {
|
|
@@ -734,8 +802,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
734
802
|
*result = d2;
|
|
735
803
|
}
|
|
736
804
|
|
|
737
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const*
|
|
738
|
-
simsimd_distance_t*
|
|
805
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
806
|
+
simsimd_distance_t *result) {
|
|
739
807
|
simsimd_size_t i = 0;
|
|
740
808
|
svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
741
809
|
svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
@@ -756,13 +824,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
|
|
|
756
824
|
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
757
825
|
}
|
|
758
826
|
|
|
759
|
-
SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const*
|
|
760
|
-
simsimd_distance_t*
|
|
827
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
828
|
+
simsimd_distance_t *result) {
|
|
761
829
|
simsimd_l2sq_f64_sve(a, b, n, result);
|
|
762
830
|
*result = _simsimd_sqrt_f64_neon(*result);
|
|
763
831
|
}
|
|
764
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const*
|
|
765
|
-
simsimd_distance_t*
|
|
832
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
833
|
+
simsimd_distance_t *result) {
|
|
766
834
|
simsimd_size_t i = 0;
|
|
767
835
|
svfloat64_t d2_vec = svdupq_n_f64(0.0, 0.0);
|
|
768
836
|
do {
|
|
@@ -777,8 +845,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t c
|
|
|
777
845
|
*result = d2;
|
|
778
846
|
}
|
|
779
847
|
|
|
780
|
-
SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const*
|
|
781
|
-
simsimd_distance_t*
|
|
848
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
849
|
+
simsimd_distance_t *result) {
|
|
782
850
|
simsimd_size_t i = 0;
|
|
783
851
|
svfloat64_t ab_vec = svdupq_n_f64(0.0, 0.0);
|
|
784
852
|
svfloat64_t a2_vec = svdupq_n_f64(0.0, 0.0);
|
|
@@ -808,17 +876,17 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
808
876
|
#pragma GCC target("arch=armv8.2-a+sve+fp16")
|
|
809
877
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
|
|
810
878
|
|
|
811
|
-
SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const*
|
|
812
|
-
simsimd_distance_t*
|
|
879
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
880
|
+
simsimd_distance_t *result) {
|
|
813
881
|
simsimd_l2sq_f16_sve(a, b, n, result);
|
|
814
882
|
*result = _simsimd_sqrt_f32_neon(*result);
|
|
815
883
|
}
|
|
816
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const*
|
|
817
|
-
simsimd_distance_t*
|
|
884
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
|
|
885
|
+
simsimd_distance_t *result) {
|
|
818
886
|
simsimd_size_t i = 0;
|
|
819
887
|
svfloat16_t d2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
820
|
-
simsimd_f16_for_arm_simd_t const*
|
|
821
|
-
simsimd_f16_for_arm_simd_t const*
|
|
888
|
+
simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
|
|
889
|
+
simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
|
|
822
890
|
do {
|
|
823
891
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
824
892
|
svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
|
|
@@ -831,14 +899,14 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f1
|
|
|
831
899
|
*result = d2_f16;
|
|
832
900
|
}
|
|
833
901
|
|
|
834
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const*
|
|
835
|
-
simsimd_distance_t*
|
|
902
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
|
|
903
|
+
simsimd_distance_t *result) {
|
|
836
904
|
simsimd_size_t i = 0;
|
|
837
905
|
svfloat16_t ab_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
838
906
|
svfloat16_t a2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
839
907
|
svfloat16_t b2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
840
|
-
simsimd_f16_for_arm_simd_t const*
|
|
841
|
-
simsimd_f16_for_arm_simd_t const*
|
|
908
|
+
simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
|
|
909
|
+
simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
|
|
842
910
|
do {
|
|
843
911
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
844
912
|
svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
|
|
@@ -864,18 +932,18 @@ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
|
|
|
864
932
|
#pragma GCC target("arch=armv8.2-a+sve+bf16")
|
|
865
933
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
|
|
866
934
|
|
|
867
|
-
SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const*
|
|
868
|
-
simsimd_distance_t*
|
|
935
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
936
|
+
simsimd_distance_t *result) {
|
|
869
937
|
simsimd_l2sq_bf16_sve(a, b, n, result);
|
|
870
938
|
*result = _simsimd_sqrt_f32_neon(*result);
|
|
871
939
|
}
|
|
872
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const*
|
|
873
|
-
simsimd_distance_t*
|
|
940
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
|
|
941
|
+
simsimd_distance_t *result) {
|
|
874
942
|
simsimd_size_t i = 0;
|
|
875
943
|
svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
876
944
|
svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
877
|
-
simsimd_u16_t const*
|
|
878
|
-
simsimd_u16_t const*
|
|
945
|
+
simsimd_u16_t const *a = (simsimd_u16_t const *)(a_enum);
|
|
946
|
+
simsimd_u16_t const *b = (simsimd_u16_t const *)(b_enum);
|
|
879
947
|
do {
|
|
880
948
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
881
949
|
svuint16_t a_vec = svld1_u16(pg_vec, a + i);
|
|
@@ -899,14 +967,14 @@ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_
|
|
|
899
967
|
*result = d2;
|
|
900
968
|
}
|
|
901
969
|
|
|
902
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const*
|
|
903
|
-
simsimd_distance_t*
|
|
970
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
|
|
971
|
+
simsimd_distance_t *result) {
|
|
904
972
|
simsimd_size_t i = 0;
|
|
905
973
|
svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
906
974
|
svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
907
975
|
svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
908
|
-
simsimd_bf16_for_arm_simd_t const*
|
|
909
|
-
simsimd_bf16_for_arm_simd_t const*
|
|
976
|
+
simsimd_bf16_for_arm_simd_t const *a = (simsimd_bf16_for_arm_simd_t const *)(a_enum);
|
|
977
|
+
simsimd_bf16_for_arm_simd_t const *b = (simsimd_bf16_for_arm_simd_t const *)(b_enum);
|
|
910
978
|
do {
|
|
911
979
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
912
980
|
svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
|
|
@@ -926,9 +994,9 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a_enum, simsimd_b
|
|
|
926
994
|
#pragma clang attribute pop
|
|
927
995
|
#pragma GCC pop_options
|
|
928
996
|
#endif // SIMSIMD_TARGET_SVE_BF16
|
|
929
|
-
#endif //
|
|
997
|
+
#endif // _SIMSIMD_TARGET_ARM
|
|
930
998
|
|
|
931
|
-
#if
|
|
999
|
+
#if _SIMSIMD_TARGET_X86
|
|
932
1000
|
#if SIMSIMD_TARGET_HASWELL
|
|
933
1001
|
#pragma GCC push_options
|
|
934
1002
|
#pragma GCC target("avx2")
|
|
@@ -945,8 +1013,7 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f
|
|
|
945
1013
|
simsimd_f64_t b2) {
|
|
946
1014
|
|
|
947
1015
|
// If both vectors have magnitude 0, the distance is 0.
|
|
948
|
-
if (a2 == 0 && b2 == 0)
|
|
949
|
-
return 0;
|
|
1016
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
950
1017
|
// If any one of the vectors is 0, the square root of the product is 0,
|
|
951
1018
|
// the division is illformed, and the result is 1.
|
|
952
1019
|
else if (ab == 0)
|
|
@@ -974,8 +1041,7 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f
|
|
|
974
1041
|
simsimd_f32_t b2) {
|
|
975
1042
|
|
|
976
1043
|
// If both vectors have magnitude 0, the distance is 0.
|
|
977
|
-
if (a2 == 0.0f && b2 == 0.0f)
|
|
978
|
-
return 0.0f;
|
|
1044
|
+
if (a2 == 0.0f && b2 == 0.0f) return 0.0f;
|
|
979
1045
|
// If any one of the vectors is 0, the square root of the product is 0,
|
|
980
1046
|
// the division is illformed, and the result is 1.
|
|
981
1047
|
else if (ab == 0.0f)
|
|
@@ -1006,21 +1072,21 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f
|
|
|
1006
1072
|
#pragma clang attribute pop
|
|
1007
1073
|
#pragma GCC pop_options
|
|
1008
1074
|
#endif // SIMSIMD_TARGET_HASWELL
|
|
1009
|
-
#endif //
|
|
1075
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
1010
1076
|
|
|
1011
|
-
#if
|
|
1077
|
+
#if _SIMSIMD_TARGET_X86
|
|
1012
1078
|
#if SIMSIMD_TARGET_HASWELL
|
|
1013
1079
|
#pragma GCC push_options
|
|
1014
1080
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
1015
1081
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
1016
1082
|
|
|
1017
|
-
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const*
|
|
1018
|
-
simsimd_distance_t*
|
|
1083
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1084
|
+
simsimd_distance_t *result) {
|
|
1019
1085
|
simsimd_l2sq_f16_haswell(a, b, n, result);
|
|
1020
1086
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1021
1087
|
}
|
|
1022
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const*
|
|
1023
|
-
simsimd_distance_t*
|
|
1088
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1089
|
+
simsimd_distance_t *result) {
|
|
1024
1090
|
__m256 a_vec, b_vec;
|
|
1025
1091
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
1026
1092
|
|
|
@@ -1029,21 +1095,21 @@ simsimd_l2sq_f16_haswell_cycle:
|
|
|
1029
1095
|
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1030
1096
|
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1031
1097
|
n = 0;
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
|
|
1098
|
+
}
|
|
1099
|
+
else {
|
|
1100
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1101
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1035
1102
|
n -= 8, a += 8, b += 8;
|
|
1036
1103
|
}
|
|
1037
1104
|
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
1038
1105
|
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1039
|
-
if (n)
|
|
1040
|
-
goto simsimd_l2sq_f16_haswell_cycle;
|
|
1106
|
+
if (n) goto simsimd_l2sq_f16_haswell_cycle;
|
|
1041
1107
|
|
|
1042
1108
|
*result = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
1043
1109
|
}
|
|
1044
1110
|
|
|
1045
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const*
|
|
1046
|
-
simsimd_distance_t*
|
|
1111
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1112
|
+
simsimd_distance_t *result) {
|
|
1047
1113
|
__m256 a_vec, b_vec;
|
|
1048
1114
|
__m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
|
|
1049
1115
|
|
|
@@ -1052,16 +1118,16 @@ simsimd_cos_f16_haswell_cycle:
|
|
|
1052
1118
|
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1053
1119
|
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1054
1120
|
n = 0;
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
|
|
1121
|
+
}
|
|
1122
|
+
else {
|
|
1123
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1124
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1058
1125
|
n -= 8, a += 8, b += 8;
|
|
1059
1126
|
}
|
|
1060
1127
|
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1061
1128
|
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1062
1129
|
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1063
|
-
if (n)
|
|
1064
|
-
goto simsimd_cos_f16_haswell_cycle;
|
|
1130
|
+
if (n) goto simsimd_cos_f16_haswell_cycle;
|
|
1065
1131
|
|
|
1066
1132
|
simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1067
1133
|
simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
@@ -1069,13 +1135,13 @@ simsimd_cos_f16_haswell_cycle:
|
|
|
1069
1135
|
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1070
1136
|
}
|
|
1071
1137
|
|
|
1072
|
-
SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const*
|
|
1073
|
-
simsimd_distance_t*
|
|
1138
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1139
|
+
simsimd_distance_t *result) {
|
|
1074
1140
|
simsimd_l2sq_bf16_haswell(a, b, n, result);
|
|
1075
1141
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1076
1142
|
}
|
|
1077
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const*
|
|
1078
|
-
simsimd_distance_t*
|
|
1143
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1144
|
+
simsimd_distance_t *result) {
|
|
1079
1145
|
__m256 a_vec, b_vec;
|
|
1080
1146
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
1081
1147
|
|
|
@@ -1084,21 +1150,21 @@ simsimd_l2sq_bf16_haswell_cycle:
|
|
|
1084
1150
|
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
|
|
1085
1151
|
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
|
|
1086
1152
|
n = 0;
|
|
1087
|
-
}
|
|
1088
|
-
|
|
1089
|
-
|
|
1153
|
+
}
|
|
1154
|
+
else {
|
|
1155
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
|
|
1156
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
|
|
1090
1157
|
n -= 8, a += 8, b += 8;
|
|
1091
1158
|
}
|
|
1092
1159
|
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
1093
1160
|
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1094
|
-
if (n)
|
|
1095
|
-
goto simsimd_l2sq_bf16_haswell_cycle;
|
|
1161
|
+
if (n) goto simsimd_l2sq_bf16_haswell_cycle;
|
|
1096
1162
|
|
|
1097
1163
|
*result = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
1098
1164
|
}
|
|
1099
1165
|
|
|
1100
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const*
|
|
1101
|
-
simsimd_distance_t*
|
|
1166
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1167
|
+
simsimd_distance_t *result) {
|
|
1102
1168
|
__m256 a_vec, b_vec;
|
|
1103
1169
|
__m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
|
|
1104
1170
|
|
|
@@ -1107,16 +1173,16 @@ simsimd_cos_bf16_haswell_cycle:
|
|
|
1107
1173
|
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
|
|
1108
1174
|
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
|
|
1109
1175
|
n = 0;
|
|
1110
|
-
}
|
|
1111
|
-
|
|
1112
|
-
|
|
1176
|
+
}
|
|
1177
|
+
else {
|
|
1178
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
|
|
1179
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
|
|
1113
1180
|
n -= 8, a += 8, b += 8;
|
|
1114
1181
|
}
|
|
1115
1182
|
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1116
1183
|
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1117
1184
|
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1118
|
-
if (n)
|
|
1119
|
-
goto simsimd_cos_bf16_haswell_cycle;
|
|
1185
|
+
if (n) goto simsimd_cos_bf16_haswell_cycle;
|
|
1120
1186
|
|
|
1121
1187
|
simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1122
1188
|
simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
@@ -1124,105 +1190,197 @@ simsimd_cos_bf16_haswell_cycle:
|
|
|
1124
1190
|
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1125
1191
|
}
|
|
1126
1192
|
|
|
1127
|
-
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const*
|
|
1128
|
-
simsimd_distance_t*
|
|
1193
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1194
|
+
simsimd_distance_t *result) {
|
|
1129
1195
|
simsimd_l2sq_i8_haswell(a, b, n, result);
|
|
1130
1196
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1131
1197
|
}
|
|
1132
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const*
|
|
1133
|
-
simsimd_distance_t*
|
|
1198
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1199
|
+
simsimd_distance_t *result) {
|
|
1134
1200
|
|
|
1135
|
-
__m256i
|
|
1136
|
-
__m256i
|
|
1201
|
+
__m256i d2_i32_low_vec = _mm256_setzero_si256();
|
|
1202
|
+
__m256i d2_i32_high_vec = _mm256_setzero_si256();
|
|
1137
1203
|
|
|
1138
1204
|
simsimd_size_t i = 0;
|
|
1139
1205
|
for (; i + 32 <= n; i += 32) {
|
|
1140
|
-
__m256i
|
|
1141
|
-
__m256i
|
|
1206
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1207
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1208
|
+
|
|
1209
|
+
// Sign extend `i8` to `i16`
|
|
1210
|
+
__m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_i8_vec));
|
|
1211
|
+
__m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
|
|
1212
|
+
__m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_i8_vec));
|
|
1213
|
+
__m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
|
|
1214
|
+
|
|
1215
|
+
// Subtract
|
|
1216
|
+
// After this we will be squaring the values. The sign will be dropped
|
|
1217
|
+
// and each difference will be in the range [0, 255].
|
|
1218
|
+
__m256i d_i16_low_vec = _mm256_sub_epi16(a_i16_low_vec, b_i16_low_vec);
|
|
1219
|
+
__m256i d_i16_high_vec = _mm256_sub_epi16(a_i16_high_vec, b_i16_high_vec);
|
|
1220
|
+
|
|
1221
|
+
// Accumulate into `i32` vectors
|
|
1222
|
+
d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
|
|
1223
|
+
d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
|
|
1224
|
+
}
|
|
1142
1225
|
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
__m256i a_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
|
|
1146
|
-
__m256i b_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_vec));
|
|
1147
|
-
__m256i b_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
|
|
1226
|
+
// Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
|
|
1227
|
+
int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
|
|
1148
1228
|
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1229
|
+
// Take care of the tail:
|
|
1230
|
+
for (; i < n; ++i) {
|
|
1231
|
+
int n = (int)(a[i]) - b[i];
|
|
1232
|
+
d2 += n * n;
|
|
1233
|
+
}
|
|
1154
1234
|
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1235
|
+
*result = (simsimd_f64_t)d2;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1239
|
+
simsimd_distance_t *result) {
|
|
1240
|
+
|
|
1241
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1242
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1243
|
+
__m256i a2_i32_low_vec = _mm256_setzero_si256();
|
|
1244
|
+
__m256i a2_i32_high_vec = _mm256_setzero_si256();
|
|
1245
|
+
__m256i b2_i32_low_vec = _mm256_setzero_si256();
|
|
1246
|
+
__m256i b2_i32_high_vec = _mm256_setzero_si256();
|
|
1247
|
+
|
|
1248
|
+
// AVX2 has no instructions for 8-bit signed integer dot-products,
|
|
1249
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1250
|
+
// So we need to normalize the first vector to its absolute value,
|
|
1251
|
+
// and shift the product sign into the second vector.
|
|
1252
|
+
//
|
|
1253
|
+
// __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
|
|
1254
|
+
// __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
|
|
1255
|
+
// __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
|
|
1256
|
+
//
|
|
1257
|
+
// The problem with this approach, however, is the `-128` value in the second vector.
|
|
1258
|
+
// Flipping it's sign will do nothing, and the result will be incorrect.
|
|
1259
|
+
// This can easily lead to noticeable numerical errors in the final result.
|
|
1260
|
+
simsimd_size_t i = 0;
|
|
1261
|
+
for (; i + 32 <= n; i += 32) {
|
|
1262
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1263
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1264
|
+
|
|
1265
|
+
// Unpack `int8` to `int16`
|
|
1266
|
+
__m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
|
|
1267
|
+
__m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
|
|
1268
|
+
__m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
|
|
1269
|
+
__m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
|
|
1270
|
+
|
|
1271
|
+
// Multiply and accumulate as `int16`, accumulate products as `int32`:
|
|
1272
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1273
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1274
|
+
a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
|
|
1275
|
+
a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
|
|
1276
|
+
b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
|
|
1277
|
+
b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
|
|
1158
1278
|
}
|
|
1159
1279
|
|
|
1160
|
-
//
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
|
|
1165
|
-
int d2 = _mm_extract_epi32(d2_sum, 0);
|
|
1280
|
+
// Further reduce to a single sum for each vector
|
|
1281
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1282
|
+
int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
|
|
1283
|
+
int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
|
|
1166
1284
|
|
|
1167
1285
|
// Take care of the tail:
|
|
1168
1286
|
for (; i < n; ++i) {
|
|
1169
|
-
int
|
|
1287
|
+
int ai = a[i], bi = b[i];
|
|
1288
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1295
|
+
simsimd_distance_t *result) {
|
|
1296
|
+
simsimd_l2sq_u8_haswell(a, b, n, result);
|
|
1297
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1298
|
+
}
|
|
1299
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1300
|
+
simsimd_distance_t *result) {
|
|
1301
|
+
|
|
1302
|
+
__m256i d2_i32_low_vec = _mm256_setzero_si256();
|
|
1303
|
+
__m256i d2_i32_high_vec = _mm256_setzero_si256();
|
|
1304
|
+
__m256i const zeros_vec = _mm256_setzero_si256();
|
|
1305
|
+
|
|
1306
|
+
simsimd_size_t i = 0;
|
|
1307
|
+
for (; i + 32 <= n; i += 32) {
|
|
1308
|
+
__m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1309
|
+
__m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1310
|
+
|
|
1311
|
+
// Substracting unsigned vectors in AVX2 is done by saturating subtraction:
|
|
1312
|
+
__m256i d_u8_vec = _mm256_or_si256(_mm256_subs_epu8(a_u8_vec, b_u8_vec), _mm256_subs_epu8(b_u8_vec, a_u8_vec));
|
|
1313
|
+
|
|
1314
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1315
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1316
|
+
__m256i d_i16_low_vec = _mm256_unpacklo_epi8(d_u8_vec, zeros_vec);
|
|
1317
|
+
__m256i d_i16_high_vec = _mm256_unpackhi_epi8(d_u8_vec, zeros_vec);
|
|
1318
|
+
|
|
1319
|
+
// Multiply and accumulate at `int16` level, accumulate at `int32` level:
|
|
1320
|
+
d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
|
|
1321
|
+
d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
|
|
1325
|
+
int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
|
|
1326
|
+
|
|
1327
|
+
// Take care of the tail:
|
|
1328
|
+
for (; i < n; ++i) {
|
|
1329
|
+
int n = (int)(a[i]) - b[i];
|
|
1170
1330
|
d2 += n * n;
|
|
1171
1331
|
}
|
|
1172
1332
|
|
|
1173
1333
|
*result = (simsimd_f64_t)d2;
|
|
1174
1334
|
}
|
|
1175
1335
|
|
|
1176
|
-
SIMSIMD_PUBLIC void
|
|
1177
|
-
simsimd_distance_t*
|
|
1336
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1337
|
+
simsimd_distance_t *result) {
|
|
1178
1338
|
|
|
1179
|
-
__m256i
|
|
1180
|
-
__m256i
|
|
1181
|
-
__m256i
|
|
1182
|
-
__m256i
|
|
1183
|
-
__m256i
|
|
1184
|
-
__m256i
|
|
1339
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1340
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1341
|
+
__m256i a2_i32_low_vec = _mm256_setzero_si256();
|
|
1342
|
+
__m256i a2_i32_high_vec = _mm256_setzero_si256();
|
|
1343
|
+
__m256i b2_i32_low_vec = _mm256_setzero_si256();
|
|
1344
|
+
__m256i b2_i32_high_vec = _mm256_setzero_si256();
|
|
1345
|
+
__m256i const zeros_vec = _mm256_setzero_si256();
|
|
1185
1346
|
|
|
1347
|
+
// AVX2 has no instructions for 8-bit signed integer dot-products,
|
|
1348
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1349
|
+
// So we need to normalize the first vector to its absolute value,
|
|
1350
|
+
// and shift the product sign into the second vector.
|
|
1351
|
+
//
|
|
1352
|
+
// __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
|
|
1353
|
+
// __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
|
|
1354
|
+
// __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
|
|
1355
|
+
//
|
|
1356
|
+
// The problem with this approach, however, is the `-128` value in the second vector.
|
|
1357
|
+
// Flipping it's sign will do nothing, and the result will be incorrect.
|
|
1358
|
+
// This can easily lead to noticeable numerical errors in the final result.
|
|
1186
1359
|
simsimd_size_t i = 0;
|
|
1187
1360
|
for (; i + 32 <= n; i += 32) {
|
|
1188
|
-
__m256i
|
|
1189
|
-
__m256i
|
|
1361
|
+
__m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1362
|
+
__m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1190
1363
|
|
|
1191
|
-
//
|
|
1192
|
-
|
|
1193
|
-
__m256i
|
|
1194
|
-
__m256i
|
|
1195
|
-
__m256i
|
|
1364
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1365
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1366
|
+
__m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
1367
|
+
__m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
1368
|
+
__m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
1369
|
+
__m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
1196
1370
|
|
|
1197
1371
|
// Multiply and accumulate as `int16`, accumulate products as `int32`
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
}
|
|
1205
|
-
|
|
1206
|
-
// Horizontal sum across the 256-bit register
|
|
1207
|
-
__m256i ab_vec = _mm256_add_epi32(ab_low_vec, ab_high_vec);
|
|
1208
|
-
__m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
|
|
1209
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1210
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1211
|
-
|
|
1212
|
-
__m256i a2_vec = _mm256_add_epi32(a2_low_vec, a2_high_vec);
|
|
1213
|
-
__m128i a2_sum = _mm_add_epi32(_mm256_extracti128_si256(a2_vec, 0), _mm256_extracti128_si256(a2_vec, 1));
|
|
1214
|
-
a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
|
|
1215
|
-
a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
|
|
1216
|
-
|
|
1217
|
-
__m256i b2_vec = _mm256_add_epi32(b2_low_vec, b2_high_vec);
|
|
1218
|
-
__m128i b2_sum = _mm_add_epi32(_mm256_extracti128_si256(b2_vec, 0), _mm256_extracti128_si256(b2_vec, 1));
|
|
1219
|
-
b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
|
|
1220
|
-
b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
|
|
1372
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1373
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1374
|
+
a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
|
|
1375
|
+
a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
|
|
1376
|
+
b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
|
|
1377
|
+
b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
|
|
1378
|
+
}
|
|
1221
1379
|
|
|
1222
1380
|
// Further reduce to a single sum for each vector
|
|
1223
|
-
int ab =
|
|
1224
|
-
int a2 =
|
|
1225
|
-
int b2 =
|
|
1381
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1382
|
+
int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
|
|
1383
|
+
int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
|
|
1226
1384
|
|
|
1227
1385
|
// Take care of the tail:
|
|
1228
1386
|
for (; i < n; ++i) {
|
|
@@ -1233,13 +1391,13 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
|
|
|
1233
1391
|
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1234
1392
|
}
|
|
1235
1393
|
|
|
1236
|
-
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const*
|
|
1237
|
-
simsimd_distance_t*
|
|
1394
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1395
|
+
simsimd_distance_t *result) {
|
|
1238
1396
|
simsimd_l2sq_f32_haswell(a, b, n, result);
|
|
1239
1397
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1240
1398
|
}
|
|
1241
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const*
|
|
1242
|
-
simsimd_distance_t*
|
|
1399
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1400
|
+
simsimd_distance_t *result) {
|
|
1243
1401
|
|
|
1244
1402
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
1245
1403
|
simsimd_size_t i = 0;
|
|
@@ -1259,8 +1417,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
1259
1417
|
*result = d2;
|
|
1260
1418
|
}
|
|
1261
1419
|
|
|
1262
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const*
|
|
1263
|
-
simsimd_distance_t*
|
|
1420
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1421
|
+
simsimd_distance_t *result) {
|
|
1264
1422
|
|
|
1265
1423
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
1266
1424
|
__m256 a2_vec = _mm256_setzero_ps();
|
|
@@ -1284,6 +1442,57 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
1284
1442
|
*result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
|
|
1285
1443
|
}
|
|
1286
1444
|
|
|
1445
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1446
|
+
simsimd_distance_t *result) {
|
|
1447
|
+
simsimd_l2sq_f64_haswell(a, b, n, result);
|
|
1448
|
+
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1449
|
+
}
|
|
1450
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1451
|
+
simsimd_distance_t *result) {
|
|
1452
|
+
|
|
1453
|
+
__m256d d2_vec = _mm256_setzero_pd();
|
|
1454
|
+
simsimd_size_t i = 0;
|
|
1455
|
+
for (; i + 4 <= n; i += 4) {
|
|
1456
|
+
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
1457
|
+
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
1458
|
+
__m256d d_vec = _mm256_sub_pd(a_vec, b_vec);
|
|
1459
|
+
d2_vec = _mm256_fmadd_pd(d_vec, d_vec, d2_vec);
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
simsimd_f64_t d2 = _simsimd_reduce_f64x4_haswell(d2_vec);
|
|
1463
|
+
for (; i < n; ++i) {
|
|
1464
|
+
simsimd_f64_t d = a[i] - b[i];
|
|
1465
|
+
d2 += d * d;
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
*result = d2;
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1472
|
+
simsimd_distance_t *result) {
|
|
1473
|
+
|
|
1474
|
+
__m256d ab_vec = _mm256_setzero_pd();
|
|
1475
|
+
__m256d a2_vec = _mm256_setzero_pd();
|
|
1476
|
+
__m256d b2_vec = _mm256_setzero_pd();
|
|
1477
|
+
simsimd_size_t i = 0;
|
|
1478
|
+
for (; i + 4 <= n; i += 4) {
|
|
1479
|
+
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
1480
|
+
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
1481
|
+
ab_vec = _mm256_fmadd_pd(a_vec, b_vec, ab_vec);
|
|
1482
|
+
a2_vec = _mm256_fmadd_pd(a_vec, a_vec, a2_vec);
|
|
1483
|
+
b2_vec = _mm256_fmadd_pd(b_vec, b_vec, b2_vec);
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
simsimd_f64_t ab = _simsimd_reduce_f64x4_haswell(ab_vec);
|
|
1487
|
+
simsimd_f64_t a2 = _simsimd_reduce_f64x4_haswell(a2_vec);
|
|
1488
|
+
simsimd_f64_t b2 = _simsimd_reduce_f64x4_haswell(b2_vec);
|
|
1489
|
+
for (; i < n; ++i) {
|
|
1490
|
+
simsimd_f64_t ai = a[i], bi = b[i];
|
|
1491
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1492
|
+
}
|
|
1493
|
+
*result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1287
1496
|
#pragma clang attribute pop
|
|
1288
1497
|
#pragma GCC pop_options
|
|
1289
1498
|
#endif // SIMSIMD_TARGET_HASWELL
|
|
@@ -1293,13 +1502,13 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
1293
1502
|
#pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
|
|
1294
1503
|
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
|
|
1295
1504
|
|
|
1296
|
-
SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const*
|
|
1297
|
-
simsimd_distance_t*
|
|
1505
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1506
|
+
simsimd_distance_t *result) {
|
|
1298
1507
|
simsimd_l2sq_f32_skylake(a, b, n, result);
|
|
1299
1508
|
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1300
1509
|
}
|
|
1301
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const*
|
|
1302
|
-
simsimd_distance_t*
|
|
1510
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1511
|
+
simsimd_distance_t *result) {
|
|
1303
1512
|
__m512 d2_vec = _mm512_setzero();
|
|
1304
1513
|
__m512 a_vec, b_vec;
|
|
1305
1514
|
|
|
@@ -1309,15 +1518,15 @@ simsimd_l2sq_f32_skylake_cycle:
|
|
|
1309
1518
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1310
1519
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1311
1520
|
n = 0;
|
|
1312
|
-
}
|
|
1521
|
+
}
|
|
1522
|
+
else {
|
|
1313
1523
|
a_vec = _mm512_loadu_ps(a);
|
|
1314
1524
|
b_vec = _mm512_loadu_ps(b);
|
|
1315
1525
|
a += 16, b += 16, n -= 16;
|
|
1316
1526
|
}
|
|
1317
1527
|
__m512 d_vec = _mm512_sub_ps(a_vec, b_vec);
|
|
1318
1528
|
d2_vec = _mm512_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1319
|
-
if (n)
|
|
1320
|
-
goto simsimd_l2sq_f32_skylake_cycle;
|
|
1529
|
+
if (n) goto simsimd_l2sq_f32_skylake_cycle;
|
|
1321
1530
|
|
|
1322
1531
|
*result = _simsimd_reduce_f32x16_skylake(d2_vec);
|
|
1323
1532
|
}
|
|
@@ -1326,8 +1535,7 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f
|
|
|
1326
1535
|
simsimd_f64_t b2) {
|
|
1327
1536
|
|
|
1328
1537
|
// If both vectors have magnitude 0, the distance is 0.
|
|
1329
|
-
if (a2 == 0 && b2 == 0)
|
|
1330
|
-
return 0;
|
|
1538
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
1331
1539
|
// If any one of the vectors is 0, the square root of the product is 0,
|
|
1332
1540
|
// the division is illformed, and the result is 1.
|
|
1333
1541
|
else if (ab == 0)
|
|
@@ -1366,8 +1574,8 @@ SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f
|
|
|
1366
1574
|
return result > 0 ? result : 0;
|
|
1367
1575
|
}
|
|
1368
1576
|
|
|
1369
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const*
|
|
1370
|
-
simsimd_distance_t*
|
|
1577
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1578
|
+
simsimd_distance_t *result) {
|
|
1371
1579
|
__m512 ab_vec = _mm512_setzero();
|
|
1372
1580
|
__m512 a2_vec = _mm512_setzero();
|
|
1373
1581
|
__m512 b2_vec = _mm512_setzero();
|
|
@@ -1379,7 +1587,8 @@ simsimd_cos_f32_skylake_cycle:
|
|
|
1379
1587
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1380
1588
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1381
1589
|
n = 0;
|
|
1382
|
-
}
|
|
1590
|
+
}
|
|
1591
|
+
else {
|
|
1383
1592
|
a_vec = _mm512_loadu_ps(a);
|
|
1384
1593
|
b_vec = _mm512_loadu_ps(b);
|
|
1385
1594
|
a += 16, b += 16, n -= 16;
|
|
@@ -1387,8 +1596,7 @@ simsimd_cos_f32_skylake_cycle:
|
|
|
1387
1596
|
ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1388
1597
|
a2_vec = _mm512_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1389
1598
|
b2_vec = _mm512_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1390
|
-
if (n)
|
|
1391
|
-
goto simsimd_cos_f32_skylake_cycle;
|
|
1599
|
+
if (n) goto simsimd_cos_f32_skylake_cycle;
|
|
1392
1600
|
|
|
1393
1601
|
simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1394
1602
|
simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
|
|
@@ -1396,13 +1604,13 @@ simsimd_cos_f32_skylake_cycle:
|
|
|
1396
1604
|
*result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
|
|
1397
1605
|
}
|
|
1398
1606
|
|
|
1399
|
-
SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const*
|
|
1400
|
-
simsimd_distance_t*
|
|
1607
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1608
|
+
simsimd_distance_t *result) {
|
|
1401
1609
|
simsimd_l2sq_f64_skylake(a, b, n, result);
|
|
1402
1610
|
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1403
1611
|
}
|
|
1404
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const*
|
|
1405
|
-
simsimd_distance_t*
|
|
1612
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1613
|
+
simsimd_distance_t *result) {
|
|
1406
1614
|
__m512d d2_vec = _mm512_setzero_pd();
|
|
1407
1615
|
__m512d a_vec, b_vec;
|
|
1408
1616
|
|
|
@@ -1412,21 +1620,21 @@ simsimd_l2sq_f64_skylake_cycle:
|
|
|
1412
1620
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1413
1621
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1414
1622
|
n = 0;
|
|
1415
|
-
}
|
|
1623
|
+
}
|
|
1624
|
+
else {
|
|
1416
1625
|
a_vec = _mm512_loadu_pd(a);
|
|
1417
1626
|
b_vec = _mm512_loadu_pd(b);
|
|
1418
1627
|
a += 8, b += 8, n -= 8;
|
|
1419
1628
|
}
|
|
1420
1629
|
__m512d d_vec = _mm512_sub_pd(a_vec, b_vec);
|
|
1421
1630
|
d2_vec = _mm512_fmadd_pd(d_vec, d_vec, d2_vec);
|
|
1422
|
-
if (n)
|
|
1423
|
-
goto simsimd_l2sq_f64_skylake_cycle;
|
|
1631
|
+
if (n) goto simsimd_l2sq_f64_skylake_cycle;
|
|
1424
1632
|
|
|
1425
1633
|
*result = _mm512_reduce_add_pd(d2_vec);
|
|
1426
1634
|
}
|
|
1427
1635
|
|
|
1428
|
-
SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const*
|
|
1429
|
-
simsimd_distance_t*
|
|
1636
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1637
|
+
simsimd_distance_t *result) {
|
|
1430
1638
|
__m512d ab_vec = _mm512_setzero_pd();
|
|
1431
1639
|
__m512d a2_vec = _mm512_setzero_pd();
|
|
1432
1640
|
__m512d b2_vec = _mm512_setzero_pd();
|
|
@@ -1438,7 +1646,8 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1438
1646
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1439
1647
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1440
1648
|
n = 0;
|
|
1441
|
-
}
|
|
1649
|
+
}
|
|
1650
|
+
else {
|
|
1442
1651
|
a_vec = _mm512_loadu_pd(a);
|
|
1443
1652
|
b_vec = _mm512_loadu_pd(b);
|
|
1444
1653
|
a += 8, b += 8, n -= 8;
|
|
@@ -1446,8 +1655,7 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1446
1655
|
ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
|
|
1447
1656
|
a2_vec = _mm512_fmadd_pd(a_vec, a_vec, a2_vec);
|
|
1448
1657
|
b2_vec = _mm512_fmadd_pd(b_vec, b_vec, b2_vec);
|
|
1449
|
-
if (n)
|
|
1450
|
-
goto simsimd_cos_f64_skylake_cycle;
|
|
1658
|
+
if (n) goto simsimd_cos_f64_skylake_cycle;
|
|
1451
1659
|
|
|
1452
1660
|
simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
|
|
1453
1661
|
simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
|
|
@@ -1462,7 +1670,7 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1462
1670
|
#if SIMSIMD_TARGET_GENOA
|
|
1463
1671
|
#pragma GCC push_options
|
|
1464
1672
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1465
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))),
|
|
1673
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1466
1674
|
apply_to = function)
|
|
1467
1675
|
|
|
1468
1676
|
SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
|
|
@@ -1517,13 +1725,13 @@ SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i
|
|
|
1517
1725
|
return d.ivec;
|
|
1518
1726
|
}
|
|
1519
1727
|
|
|
1520
|
-
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const*
|
|
1521
|
-
simsimd_distance_t*
|
|
1728
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1729
|
+
simsimd_distance_t *result) {
|
|
1522
1730
|
simsimd_l2sq_bf16_genoa(a, b, n, result);
|
|
1523
1731
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1524
1732
|
}
|
|
1525
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const*
|
|
1526
|
-
simsimd_distance_t*
|
|
1733
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1734
|
+
simsimd_distance_t *result) {
|
|
1527
1735
|
__m512 d2_vec = _mm512_setzero_ps();
|
|
1528
1736
|
__m512i a_i16_vec, b_i16_vec, d_i16_vec;
|
|
1529
1737
|
|
|
@@ -1533,21 +1741,21 @@ simsimd_l2sq_bf16_genoa_cycle:
|
|
|
1533
1741
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1534
1742
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1535
1743
|
n = 0;
|
|
1536
|
-
}
|
|
1744
|
+
}
|
|
1745
|
+
else {
|
|
1537
1746
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1538
1747
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1539
1748
|
a += 32, b += 32, n -= 32;
|
|
1540
1749
|
}
|
|
1541
1750
|
d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
|
|
1542
1751
|
d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
|
|
1543
|
-
if (n)
|
|
1544
|
-
goto simsimd_l2sq_bf16_genoa_cycle;
|
|
1752
|
+
if (n) goto simsimd_l2sq_bf16_genoa_cycle;
|
|
1545
1753
|
|
|
1546
1754
|
*result = _simsimd_reduce_f32x16_skylake(d2_vec);
|
|
1547
1755
|
}
|
|
1548
1756
|
|
|
1549
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const*
|
|
1550
|
-
simsimd_distance_t*
|
|
1757
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1758
|
+
simsimd_distance_t *result) {
|
|
1551
1759
|
__m512 ab_vec = _mm512_setzero_ps();
|
|
1552
1760
|
__m512 a2_vec = _mm512_setzero_ps();
|
|
1553
1761
|
__m512 b2_vec = _mm512_setzero_ps();
|
|
@@ -1559,7 +1767,8 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1559
1767
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1560
1768
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1561
1769
|
n = 0;
|
|
1562
|
-
}
|
|
1770
|
+
}
|
|
1771
|
+
else {
|
|
1563
1772
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1564
1773
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1565
1774
|
a += 32, b += 32, n -= 32;
|
|
@@ -1567,8 +1776,7 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1567
1776
|
ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
|
|
1568
1777
|
a2_vec = _mm512_dpbf16_ps(a2_vec, (__m512bh)(a_i16_vec), (__m512bh)(a_i16_vec));
|
|
1569
1778
|
b2_vec = _mm512_dpbf16_ps(b2_vec, (__m512bh)(b_i16_vec), (__m512bh)(b_i16_vec));
|
|
1570
|
-
if (n)
|
|
1571
|
-
goto simsimd_cos_bf16_genoa_cycle;
|
|
1779
|
+
if (n) goto simsimd_cos_bf16_genoa_cycle;
|
|
1572
1780
|
|
|
1573
1781
|
simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1574
1782
|
simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
|
|
@@ -1585,13 +1793,13 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1585
1793
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
1586
1794
|
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
1587
1795
|
|
|
1588
|
-
SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const*
|
|
1589
|
-
simsimd_distance_t*
|
|
1796
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1797
|
+
simsimd_distance_t *result) {
|
|
1590
1798
|
simsimd_l2sq_f16_sapphire(a, b, n, result);
|
|
1591
1799
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1592
1800
|
}
|
|
1593
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const*
|
|
1594
|
-
simsimd_distance_t*
|
|
1801
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1802
|
+
simsimd_distance_t *result) {
|
|
1595
1803
|
__m512h d2_vec = _mm512_setzero_ph();
|
|
1596
1804
|
__m512i a_i16_vec, b_i16_vec;
|
|
1597
1805
|
|
|
@@ -1601,21 +1809,21 @@ simsimd_l2sq_f16_sapphire_cycle:
|
|
|
1601
1809
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1602
1810
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1603
1811
|
n = 0;
|
|
1604
|
-
}
|
|
1812
|
+
}
|
|
1813
|
+
else {
|
|
1605
1814
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1606
1815
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1607
1816
|
a += 32, b += 32, n -= 32;
|
|
1608
1817
|
}
|
|
1609
1818
|
__m512h d_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec));
|
|
1610
1819
|
d2_vec = _mm512_fmadd_ph(d_vec, d_vec, d2_vec);
|
|
1611
|
-
if (n)
|
|
1612
|
-
goto simsimd_l2sq_f16_sapphire_cycle;
|
|
1820
|
+
if (n) goto simsimd_l2sq_f16_sapphire_cycle;
|
|
1613
1821
|
|
|
1614
1822
|
*result = _mm512_reduce_add_ph(d2_vec);
|
|
1615
1823
|
}
|
|
1616
1824
|
|
|
1617
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const*
|
|
1618
|
-
simsimd_distance_t*
|
|
1825
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1826
|
+
simsimd_distance_t *result) {
|
|
1619
1827
|
__m512h ab_vec = _mm512_setzero_ph();
|
|
1620
1828
|
__m512h a2_vec = _mm512_setzero_ph();
|
|
1621
1829
|
__m512h b2_vec = _mm512_setzero_ph();
|
|
@@ -1627,7 +1835,8 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1627
1835
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1628
1836
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1629
1837
|
n = 0;
|
|
1630
|
-
}
|
|
1838
|
+
}
|
|
1839
|
+
else {
|
|
1631
1840
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1632
1841
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1633
1842
|
a += 32, b += 32, n -= 32;
|
|
@@ -1635,8 +1844,7 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1635
1844
|
ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
|
|
1636
1845
|
a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(a_i16_vec), a2_vec);
|
|
1637
1846
|
b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_i16_vec), _mm512_castsi512_ph(b_i16_vec), b2_vec);
|
|
1638
|
-
if (n)
|
|
1639
|
-
goto simsimd_cos_f16_sapphire_cycle;
|
|
1847
|
+
if (n) goto simsimd_cos_f16_sapphire_cycle;
|
|
1640
1848
|
|
|
1641
1849
|
simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
|
|
1642
1850
|
simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
|
|
@@ -1651,92 +1859,463 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1651
1859
|
#if SIMSIMD_TARGET_ICE
|
|
1652
1860
|
#pragma GCC push_options
|
|
1653
1861
|
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1654
|
-
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))),
|
|
1862
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
|
|
1655
1863
|
apply_to = function)
|
|
1656
1864
|
|
|
1657
|
-
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const*
|
|
1658
|
-
simsimd_distance_t*
|
|
1865
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1866
|
+
simsimd_distance_t *result) {
|
|
1659
1867
|
simsimd_l2sq_i8_ice(a, b, n, result);
|
|
1660
1868
|
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1661
1869
|
}
|
|
1662
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const*
|
|
1663
|
-
simsimd_distance_t*
|
|
1664
|
-
__m512i
|
|
1665
|
-
__m512i
|
|
1870
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1871
|
+
simsimd_distance_t *result) {
|
|
1872
|
+
__m512i d2_i32_vec = _mm512_setzero_si512();
|
|
1873
|
+
__m512i a_i16_vec, b_i16_vec, d_i16s_vec;
|
|
1666
1874
|
|
|
1667
1875
|
simsimd_l2sq_i8_ice_cycle:
|
|
1668
|
-
if (n < 32) { // TODO: Avoid
|
|
1876
|
+
if (n < 32) { // TODO: Avoid eaarly i16 upcast to step through 64 values at a time
|
|
1669
1877
|
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1670
|
-
|
|
1671
|
-
|
|
1878
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1879
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
1672
1880
|
n = 0;
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
|
|
1881
|
+
}
|
|
1882
|
+
else {
|
|
1883
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
|
|
1884
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
|
|
1676
1885
|
a += 32, b += 32, n -= 32;
|
|
1677
1886
|
}
|
|
1678
|
-
d_i16s_vec = _mm512_sub_epi16(
|
|
1679
|
-
|
|
1680
|
-
if (n)
|
|
1681
|
-
goto simsimd_l2sq_i8_ice_cycle;
|
|
1887
|
+
d_i16s_vec = _mm512_sub_epi16(a_i16_vec, b_i16_vec);
|
|
1888
|
+
d2_i32_vec = _mm512_dpwssd_epi32(d2_i32_vec, d_i16s_vec, d_i16s_vec);
|
|
1889
|
+
if (n) goto simsimd_l2sq_i8_ice_cycle;
|
|
1682
1890
|
|
|
1683
|
-
*result = _mm512_reduce_add_epi32(
|
|
1891
|
+
*result = _mm512_reduce_add_epi32(d2_i32_vec);
|
|
1684
1892
|
}
|
|
1685
1893
|
|
|
1686
|
-
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const*
|
|
1687
|
-
simsimd_distance_t*
|
|
1688
|
-
__m512i ab_low_i32s_vec = _mm512_setzero_si512();
|
|
1689
|
-
__m512i ab_high_i32s_vec = _mm512_setzero_si512();
|
|
1690
|
-
__m512i a2_i32s_vec = _mm512_setzero_si512();
|
|
1691
|
-
__m512i b2_i32s_vec = _mm512_setzero_si512();
|
|
1692
|
-
__m512i a_vec, b_vec;
|
|
1693
|
-
__m512i a_abs_vec, b_abs_vec;
|
|
1894
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1895
|
+
simsimd_distance_t *result) {
|
|
1694
1896
|
|
|
1897
|
+
__m512i ab_i32_vec = _mm512_setzero_si512();
|
|
1898
|
+
__m512i a2_i32_vec = _mm512_setzero_si512();
|
|
1899
|
+
__m512i b2_i32_vec = _mm512_setzero_si512();
|
|
1900
|
+
__m512i a_i16_vec, b_i16_vec;
|
|
1695
1901
|
simsimd_cos_i8_ice_cycle:
|
|
1696
|
-
if (n <
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1902
|
+
if (n < 32) {
|
|
1903
|
+
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1904
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1905
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
1700
1906
|
n = 0;
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1907
|
+
}
|
|
1908
|
+
else {
|
|
1909
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
|
|
1910
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
|
|
1911
|
+
a += 32, b += 32, n -= 32;
|
|
1705
1912
|
}
|
|
1706
1913
|
|
|
1707
1914
|
// We can't directly use the `_mm512_dpbusd_epi32` intrinsic everywhere,
|
|
1708
1915
|
// as it's asymmetric with respect to the sign of the input arguments:
|
|
1916
|
+
//
|
|
1709
1917
|
// Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
|
|
1710
|
-
//
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1918
|
+
//
|
|
1919
|
+
// To compute the squares, we could just drop the sign bit of the second argument.
|
|
1920
|
+
// But this would lead to big-big problems on values like `-128`!
|
|
1921
|
+
// For dot-products we don't have the luxury of optimizing the sign bit away.
|
|
1922
|
+
// Assuming this is an approximate kernel (with reciprocal square root approximations)
|
|
1923
|
+
// in the end, we can allow clamping the value to [-127, 127] range.
|
|
1924
|
+
//
|
|
1925
|
+
// On Ice Lake:
|
|
1926
|
+
//
|
|
1927
|
+
// 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can only execute on port 0, with 5 cycle latency.
|
|
1928
|
+
// 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also only execute on port 0, with 5 cycle latency.
|
|
1929
|
+
// 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 5, with 5 cycle latency.
|
|
1930
|
+
//
|
|
1931
|
+
// On Zen4 Genoa:
|
|
1932
|
+
//
|
|
1933
|
+
// 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 4 cycle latency.
|
|
1934
|
+
// 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also execute on ports 0 and 1, with 4 cycle latency.
|
|
1935
|
+
// 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 3 cycle latency.
|
|
1936
|
+
//
|
|
1937
|
+
// The old solution was complex replied on 1. and 2.:
|
|
1938
|
+
//
|
|
1939
|
+
// a_i8_abs_vec = _mm512_abs_epi8(a_i8_vec);
|
|
1940
|
+
// b_i8_abs_vec = _mm512_abs_epi8(b_i8_vec);
|
|
1941
|
+
// a2_i32_vec = _mm512_dpbusds_epi32(a2_i32_vec, a_i8_abs_vec, a_i8_abs_vec);
|
|
1942
|
+
// b2_i32_vec = _mm512_dpbusds_epi32(b2_i32_vec, b_i8_abs_vec, b_i8_abs_vec);
|
|
1943
|
+
// ab_i32_low_vec = _mm512_dpwssds_epi32( //
|
|
1944
|
+
// ab_i32_low_vec, //
|
|
1945
|
+
// _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)), //
|
|
1946
|
+
// _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
|
|
1947
|
+
// ab_i32_high_vec = _mm512_dpwssds_epi32( //
|
|
1948
|
+
// ab_i32_high_vec, //
|
|
1949
|
+
// _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)), //
|
|
1950
|
+
// _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
|
|
1951
|
+
//
|
|
1952
|
+
// The new solution is simpler and relies on 3.:
|
|
1953
|
+
ab_i32_vec = _mm512_add_epi32(ab_i32_vec, _mm512_madd_epi16(a_i16_vec, b_i16_vec));
|
|
1954
|
+
a2_i32_vec = _mm512_add_epi32(a2_i32_vec, _mm512_madd_epi16(a_i16_vec, a_i16_vec));
|
|
1955
|
+
b2_i32_vec = _mm512_add_epi32(b2_i32_vec, _mm512_madd_epi16(b_i16_vec, b_i16_vec));
|
|
1956
|
+
if (n) goto simsimd_cos_i8_ice_cycle;
|
|
1957
|
+
|
|
1958
|
+
int ab = _mm512_reduce_add_epi32(ab_i32_vec);
|
|
1959
|
+
int a2 = _mm512_reduce_add_epi32(a2_i32_vec);
|
|
1960
|
+
int b2 = _mm512_reduce_add_epi32(b2_i32_vec);
|
|
1961
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1962
|
+
}
|
|
1963
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1964
|
+
simsimd_distance_t *result) {
|
|
1965
|
+
simsimd_l2sq_u8_ice(a, b, n, result);
|
|
1966
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1967
|
+
}
|
|
1968
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1969
|
+
simsimd_distance_t *result) {
|
|
1970
|
+
__m512i d2_i32_low_vec = _mm512_setzero_si512();
|
|
1971
|
+
__m512i d2_i32_high_vec = _mm512_setzero_si512();
|
|
1972
|
+
__m512i const zeros_vec = _mm512_setzero_si512();
|
|
1973
|
+
__m512i d_i16_low_vec, d_i16_high_vec;
|
|
1974
|
+
__m512i a_u8_vec, b_u8_vec, d_u8_vec;
|
|
1975
|
+
|
|
1976
|
+
simsimd_l2sq_u8_ice_cycle:
|
|
1977
|
+
if (n < 64) {
|
|
1978
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
1979
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1980
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1981
|
+
n = 0;
|
|
1982
|
+
}
|
|
1983
|
+
else {
|
|
1984
|
+
a_u8_vec = _mm512_loadu_si512(a);
|
|
1985
|
+
b_u8_vec = _mm512_loadu_si512(b);
|
|
1986
|
+
a += 64, b += 64, n -= 64;
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
// Substracting unsigned vectors in AVX-512 is done by saturating subtraction:
|
|
1990
|
+
d_u8_vec = _mm512_or_si512(_mm512_subs_epu8(a_u8_vec, b_u8_vec), _mm512_subs_epu8(b_u8_vec, a_u8_vec));
|
|
1991
|
+
d_i16_low_vec = _mm512_unpacklo_epi8(d_u8_vec, zeros_vec);
|
|
1992
|
+
d_i16_high_vec = _mm512_unpackhi_epi8(d_u8_vec, zeros_vec);
|
|
1993
|
+
|
|
1994
|
+
// Multiply and accumulate at `int16` level, accumulate at `int32` level:
|
|
1995
|
+
d2_i32_low_vec = _mm512_dpwssd_epi32(d2_i32_low_vec, d_i16_low_vec, d_i16_low_vec);
|
|
1996
|
+
d2_i32_high_vec = _mm512_dpwssd_epi32(d2_i32_high_vec, d_i16_high_vec, d_i16_high_vec);
|
|
1997
|
+
if (n) goto simsimd_l2sq_u8_ice_cycle;
|
|
1998
|
+
|
|
1999
|
+
*result = _mm512_reduce_add_epi32(_mm512_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
|
|
2000
|
+
}
|
|
2001
|
+
|
|
2002
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
2003
|
+
simsimd_distance_t *result) {
|
|
2004
|
+
|
|
2005
|
+
__m512i ab_i32_low_vec = _mm512_setzero_si512();
|
|
2006
|
+
__m512i ab_i32_high_vec = _mm512_setzero_si512();
|
|
2007
|
+
__m512i a2_i32_low_vec = _mm512_setzero_si512();
|
|
2008
|
+
__m512i a2_i32_high_vec = _mm512_setzero_si512();
|
|
2009
|
+
__m512i b2_i32_low_vec = _mm512_setzero_si512();
|
|
2010
|
+
__m512i b2_i32_high_vec = _mm512_setzero_si512();
|
|
2011
|
+
__m512i const zeros_vec = _mm512_setzero_si512();
|
|
2012
|
+
__m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
|
|
2013
|
+
__m512i a_u8_vec, b_u8_vec;
|
|
2014
|
+
|
|
2015
|
+
simsimd_cos_u8_ice_cycle:
|
|
2016
|
+
if (n < 64) {
|
|
2017
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
2018
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
2019
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
2020
|
+
n = 0;
|
|
2021
|
+
}
|
|
2022
|
+
else {
|
|
2023
|
+
a_u8_vec = _mm512_loadu_si512(a);
|
|
2024
|
+
b_u8_vec = _mm512_loadu_si512(b);
|
|
2025
|
+
a += 64, b += 64, n -= 64;
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
2029
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
2030
|
+
a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
2031
|
+
a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
2032
|
+
b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
2033
|
+
b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
2034
|
+
|
|
2035
|
+
// Multiply and accumulate as `int16`, accumulate products as `int32`:
|
|
2036
|
+
ab_i32_low_vec = _mm512_dpwssds_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
|
|
2037
|
+
ab_i32_high_vec = _mm512_dpwssds_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
|
|
2038
|
+
a2_i32_low_vec = _mm512_dpwssds_epi32(a2_i32_low_vec, a_i16_low_vec, a_i16_low_vec);
|
|
2039
|
+
a2_i32_high_vec = _mm512_dpwssds_epi32(a2_i32_high_vec, a_i16_high_vec, a_i16_high_vec);
|
|
2040
|
+
b2_i32_low_vec = _mm512_dpwssds_epi32(b2_i32_low_vec, b_i16_low_vec, b_i16_low_vec);
|
|
2041
|
+
b2_i32_high_vec = _mm512_dpwssds_epi32(b2_i32_high_vec, b_i16_high_vec, b_i16_high_vec);
|
|
2042
|
+
if (n) goto simsimd_cos_u8_ice_cycle;
|
|
2043
|
+
|
|
2044
|
+
int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
2045
|
+
int a2 = _mm512_reduce_add_epi32(_mm512_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
|
|
2046
|
+
int b2 = _mm512_reduce_add_epi32(_mm512_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
|
|
2047
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
|
|
2051
|
+
simsimd_distance_t *result) {
|
|
2052
|
+
simsimd_l2sq_i4x2_ice(a, b, n_words, result);
|
|
2053
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
2054
|
+
}
|
|
2055
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
|
|
2056
|
+
simsimd_distance_t *result) {
|
|
2057
|
+
|
|
2058
|
+
// While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
|
|
2059
|
+
// The absolute difference between two 4-bit integers is at most 15 and it is always a `uint4_t` value!
|
|
2060
|
+
// Moreover, it's square is at most 225, which fits into `uint8_t` and can be computed with a single
|
|
2061
|
+
// lookup table. Accumulating those values is similar to checksumming, a piece of cake for SIMD!
|
|
2062
|
+
__m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
|
|
2063
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2064
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2065
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2066
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
|
|
2067
|
+
__m512i const u4_squares_lookup_vec = _mm512_set_epi8( //
|
|
2068
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2069
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2070
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2071
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0);
|
|
2072
|
+
|
|
2073
|
+
/// The mask used to take the low nibble of each byte.
|
|
2074
|
+
__m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
|
|
2075
|
+
|
|
2076
|
+
// Temporaries:
|
|
2077
|
+
__m512i a_i4x2_vec, b_i4x2_vec;
|
|
2078
|
+
__m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
|
|
2079
|
+
__m512i d_u8_low_vec, d_u8_high_vec; //! Only the low 4 bits are actually used
|
|
2080
|
+
__m512i d2_u8_low_vec, d2_u8_high_vec;
|
|
2081
|
+
__m512i d2_u16_low_vec, d2_u16_high_vec;
|
|
2082
|
+
|
|
2083
|
+
// Accumulators:
|
|
2084
|
+
__m512i d2_u32_vec = _mm512_setzero_si512();
|
|
2085
|
+
|
|
2086
|
+
simsimd_l2sq_i4x2_ice_cycle:
|
|
2087
|
+
if (n_words < 64) {
|
|
2088
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
|
|
2089
|
+
a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
2090
|
+
b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
2091
|
+
n_words = 0;
|
|
2092
|
+
}
|
|
2093
|
+
else {
|
|
2094
|
+
a_i4x2_vec = _mm512_loadu_epi8(a);
|
|
2095
|
+
b_i4x2_vec = _mm512_loadu_epi8(b);
|
|
2096
|
+
a += 64, b += 64, n_words -= 64;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
// Unpack the 4-bit values into 8-bit values with an empty top nibble.
|
|
2100
|
+
a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
|
|
2101
|
+
a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
|
|
2102
|
+
b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
|
|
2103
|
+
b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
|
|
2104
|
+
a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
|
|
2105
|
+
a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
|
|
2106
|
+
b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
|
|
2107
|
+
b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
|
|
2108
|
+
|
|
2109
|
+
// We can implement subtraction with a lookup table, or using `_mm512_sub_epi8`.
|
|
2110
|
+
d_u8_low_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_low_vec, b_i8_low_vec));
|
|
2111
|
+
d_u8_high_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_high_vec, b_i8_high_vec));
|
|
2112
|
+
|
|
2113
|
+
// Now we can use the lookup table to compute the squares of the 4-bit unsigned integers
|
|
2114
|
+
// in the low nibbles of the `d_u8_low_vec` and `d_u8_high_vec` vectors.
|
|
2115
|
+
d2_u8_low_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_low_vec);
|
|
2116
|
+
d2_u8_high_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_high_vec);
|
|
2117
|
+
|
|
2118
|
+
// Aggregating into 16-bit integers, we need to first upcast our 8-bit values to 16 bits.
|
|
2119
|
+
// After that, we will perform one more operation, upcasting further into 32-bit integers.
|
|
2120
|
+
d2_u16_low_vec = //
|
|
2121
|
+
_mm512_add_epi16( //
|
|
2122
|
+
_mm512_unpacklo_epi8(d2_u8_low_vec, _mm512_setzero_si512()),
|
|
2123
|
+
_mm512_unpackhi_epi8(d2_u8_low_vec, _mm512_setzero_si512()));
|
|
2124
|
+
d2_u16_high_vec = //
|
|
2125
|
+
_mm512_add_epi16( //
|
|
2126
|
+
_mm512_unpacklo_epi8(d2_u8_high_vec, _mm512_setzero_si512()),
|
|
2127
|
+
_mm512_unpackhi_epi8(d2_u8_high_vec, _mm512_setzero_si512()));
|
|
2128
|
+
d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_low_vec, _mm512_setzero_si512()));
|
|
2129
|
+
d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_high_vec, _mm512_setzero_si512()));
|
|
2130
|
+
if (n_words) goto simsimd_l2sq_i4x2_ice_cycle;
|
|
2131
|
+
|
|
2132
|
+
// Finally, we can reduce the 16-bit integers to 32-bit integers and sum them up.
|
|
2133
|
+
int d2 = _mm512_reduce_add_epi32(d2_u32_vec);
|
|
2134
|
+
*result = d2;
|
|
2135
|
+
}
|
|
2136
|
+
SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
|
|
2137
|
+
simsimd_distance_t *result) {
|
|
2138
|
+
|
|
2139
|
+
// We need to compose a lookup table for all the scalar products of 4-bit integers.
|
|
2140
|
+
// While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
|
|
2141
|
+
// Practically speaking, the product of two 4-bit signed integers is a 7-bit integer,
|
|
2142
|
+
// as the maximum absolute value of the product is `abs(-8 * -8) == 64`.
|
|
2143
|
+
//
|
|
2144
|
+
// To store 128 possible values of 2^7 bits we only need 128 single-byte scalars,
|
|
2145
|
+
// or just 2x ZMM registers. In that case our lookup will only take `vpermi2b` instruction,
|
|
2146
|
+
// easily inokable with `_mm512_permutex2var_epi8` intrinsic with latency of 6 on Sapphire Rapids.
|
|
2147
|
+
// The problem is converting 2d indices of our symmetric matrix into 1d offsets in the dense array.
|
|
2148
|
+
//
|
|
2149
|
+
// Alternatively, we can take the entire symmetric (16 x 16) matrix of products,
|
|
2150
|
+
// put into 4x ZMM registers, and use it with `_mm512_shuffle_epi8`, remembering
|
|
2151
|
+
// that it can only lookup with 128-bit lanes (16x 8-bit values).
|
|
2152
|
+
// That intrinsic has latency 1, but will need to be repeated and combined with
|
|
2153
|
+
// multiple iterations of `_mm512_shuffle_i64x2` that has latency 3.
|
|
2154
|
+
//
|
|
2155
|
+
// Altenatively, we can get down to 3 cycles per lookup with `vpermb` and `_mm512_permutexvar_epi8` intrinsics.
|
|
2156
|
+
// For that we can split our (16 x 16) matrix into 4x (8 x 8) submatrices, and use 4x ZMM registers.
|
|
2157
|
+
//
|
|
2158
|
+
// Still, all of those solutions are quite heavy compared to two parallel calls to `_mm512_dpbusds_epi32`
|
|
2159
|
+
// for the dot product. But we can still use the `_mm512_permutexvar_epi8` to compute the squares of the
|
|
2160
|
+
// 16 possible `int4_t` values faster.
|
|
2161
|
+
//
|
|
2162
|
+
// Here is how our `int4_t` range looks:
|
|
2163
|
+
//
|
|
2164
|
+
// dec: 0 1 2 3 4 5 6 7 -8 -7 -6 -5 -4 -3 -2 -1
|
|
2165
|
+
// hex: 0 1 2 3 4 5 6 7 8 9 A B C D E F
|
|
2166
|
+
//
|
|
2167
|
+
// Squared:
|
|
2168
|
+
//
|
|
2169
|
+
// dec2: 0 1 4 9 16 25 36 49 64 49 36 25 16 9 4 1
|
|
2170
|
+
// hex2: 0 1 4 9 10 19 24 31 40 31 24 19 10 9 4 1
|
|
2171
|
+
//
|
|
2172
|
+
// Broadcast it to every lane, so that: `square(x) == _mm512_shuffle_epi8(i4_squares_lookup_vec, x)`.
|
|
2173
|
+
__m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
|
|
2174
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2175
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2176
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2177
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
|
|
2178
|
+
__m512i const i4_squares_lookup_vec = _mm512_set_epi8( //
|
|
2179
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2180
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2181
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2182
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0);
|
|
2183
|
+
|
|
2184
|
+
/// The mask used to take the low nibble of each byte.
|
|
2185
|
+
__m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
|
|
2186
|
+
|
|
2187
|
+
// Temporaries:
|
|
2188
|
+
__m512i a_i4x2_vec, b_i4x2_vec;
|
|
2189
|
+
__m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
|
|
2190
|
+
__m512i a2_u8_vec, b2_u8_vec;
|
|
2191
|
+
|
|
2192
|
+
// Accumulators:
|
|
2193
|
+
__m512i a2_u16_low_vec = _mm512_setzero_si512();
|
|
2194
|
+
__m512i a2_u16_high_vec = _mm512_setzero_si512();
|
|
2195
|
+
__m512i b2_u16_low_vec = _mm512_setzero_si512();
|
|
2196
|
+
__m512i b2_u16_high_vec = _mm512_setzero_si512();
|
|
2197
|
+
__m512i ab_i32_low_vec = _mm512_setzero_si512();
|
|
2198
|
+
__m512i ab_i32_high_vec = _mm512_setzero_si512();
|
|
2199
|
+
|
|
2200
|
+
simsimd_cos_i4x2_ice_cycle:
|
|
2201
|
+
if (n_words < 64) {
|
|
2202
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
|
|
2203
|
+
a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
2204
|
+
b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
2205
|
+
n_words = 0;
|
|
2206
|
+
}
|
|
2207
|
+
else {
|
|
2208
|
+
a_i4x2_vec = _mm512_loadu_epi8(a);
|
|
2209
|
+
b_i4x2_vec = _mm512_loadu_epi8(b);
|
|
2210
|
+
a += 64, b += 64, n_words -= 64;
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
// Unpack the 4-bit values into 8-bit values with an empty top nibble.
|
|
2214
|
+
// For now, they are not really 8-bit integers, as they are not sign-extended.
|
|
2215
|
+
// That part will come later, using the `i4_to_i8_lookup_vec` lookup.
|
|
2216
|
+
a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
|
|
2217
|
+
a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
|
|
2218
|
+
b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
|
|
2219
|
+
b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
|
|
2220
|
+
|
|
2221
|
+
// Compute the squares of the 4-bit integers.
|
|
2222
|
+
// For symmetry we could have used 4 registers, aka "a2_i8_low_vec", "a2_i8_high_vec", "b2_i8_low_vec",
|
|
2223
|
+
// "b2_i8_high_vec". But the largest square value is just 64, so we can safely aggregate into 8-bit unsigned values.
|
|
2224
|
+
a2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_low_vec),
|
|
2225
|
+
_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_high_vec));
|
|
2226
|
+
b2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_low_vec),
|
|
2227
|
+
_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_high_vec));
|
|
2228
|
+
|
|
2229
|
+
// We can safely aggregate into just 16-bit sums without overflow, if the vectors have less than:
|
|
2230
|
+
// (2 scalars / byte) * (64 bytes / register) * (256 non-overflowing 8-bit additions in 16-bit intesgers)
|
|
2231
|
+
// = 32'768 dimensions.
|
|
2232
|
+
//
|
|
2233
|
+
// We use saturated addition here to clearly inform in case of overflow.
|
|
2234
|
+
a2_u16_low_vec = _mm512_adds_epu16(a2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
|
|
2235
|
+
a2_u16_high_vec = _mm512_adds_epu16(a2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
|
|
2236
|
+
b2_u16_low_vec = _mm512_adds_epu16(b2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
|
|
2237
|
+
b2_u16_high_vec = _mm512_adds_epu16(b2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
|
|
2238
|
+
|
|
2239
|
+
// Time to perform the proper sign extension of the 4-bit integers to 8-bit integers.
|
|
2240
|
+
a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
|
|
2241
|
+
a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
|
|
2242
|
+
b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
|
|
2243
|
+
b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
|
|
1715
2244
|
|
|
1716
2245
|
// The same trick won't work for the primary dot-product, as the signs vector
|
|
1717
|
-
// components may differ significantly. So we have to use two `
|
|
2246
|
+
// components may differ significantly. So we have to use two `_mm512_dpwssds_epi32`
|
|
1718
2247
|
// intrinsics instead, upcasting four chunks to 16-bit integers beforehand!
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
2248
|
+
// Alternatively, we can flip the signs of the second argument and use `_mm512_dpbusds_epi32`,
|
|
2249
|
+
// but it ends up taking more instructions.
|
|
2250
|
+
ab_i32_low_vec = _mm512_dpwssds_epi32( //
|
|
2251
|
+
ab_i32_low_vec, //
|
|
2252
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_low_vec)), //
|
|
2253
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_low_vec)));
|
|
2254
|
+
ab_i32_low_vec = _mm512_dpwssds_epi32( //
|
|
2255
|
+
ab_i32_low_vec, //
|
|
2256
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_low_vec, 1)), //
|
|
2257
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_low_vec, 1)));
|
|
2258
|
+
ab_i32_high_vec = _mm512_dpwssds_epi32( //
|
|
2259
|
+
ab_i32_high_vec, //
|
|
2260
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_high_vec)), //
|
|
2261
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_high_vec)));
|
|
2262
|
+
ab_i32_high_vec = _mm512_dpwssds_epi32( //
|
|
2263
|
+
ab_i32_high_vec, //
|
|
2264
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_high_vec, 1)), //
|
|
2265
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_high_vec, 1)));
|
|
2266
|
+
if (n_words) goto simsimd_cos_i4x2_ice_cycle;
|
|
2267
|
+
|
|
2268
|
+
int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
2269
|
+
unsigned short a2_u16[32], b2_u16[32];
|
|
2270
|
+
_mm512_storeu_si512(a2_u16, _mm512_add_epi16(a2_u16_low_vec, a2_u16_high_vec));
|
|
2271
|
+
unsigned int a2 = 0, b2 = 0;
|
|
2272
|
+
for (int i = 0; i < 32; ++i) a2 += a2_u16[i], b2 += b2_u16[i];
|
|
1733
2273
|
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1734
2274
|
}
|
|
1735
2275
|
|
|
1736
2276
|
#pragma clang attribute pop
|
|
1737
2277
|
#pragma GCC pop_options
|
|
1738
2278
|
#endif // SIMSIMD_TARGET_ICE
|
|
1739
|
-
|
|
2279
|
+
|
|
2280
|
+
#if SIMSIMD_TARGET_SIERRA
|
|
2281
|
+
#pragma GCC push_options
|
|
2282
|
+
#pragma GCC target("avx2", "bmi2", "avx2vnni")
|
|
2283
|
+
#pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
|
|
2284
|
+
|
|
2285
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
2286
|
+
simsimd_distance_t *result) {
|
|
2287
|
+
|
|
2288
|
+
__m256i ab_i32_vec = _mm256_setzero_si256();
|
|
2289
|
+
__m256i a2_i32_vec = _mm256_setzero_si256();
|
|
2290
|
+
__m256i b2_i32_vec = _mm256_setzero_si256();
|
|
2291
|
+
|
|
2292
|
+
simsimd_size_t i = 0;
|
|
2293
|
+
for (; i + 32 <= n; i += 32) {
|
|
2294
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
2295
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
2296
|
+
ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
|
|
2297
|
+
a2_i32_vec = _mm256_dpbssds_epi32(a2_i32_vec, a_i8_vec, a_i8_vec);
|
|
2298
|
+
b2_i32_vec = _mm256_dpbssds_epi32(b2_i32_vec, b_i8_vec, b_i8_vec);
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
// Further reduce to a single sum for each vector
|
|
2302
|
+
int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
|
|
2303
|
+
int a2 = _simsimd_reduce_i32x8_haswell(a2_i32_vec);
|
|
2304
|
+
int b2 = _simsimd_reduce_i32x8_haswell(b2_i32_vec);
|
|
2305
|
+
|
|
2306
|
+
// Take care of the tail:
|
|
2307
|
+
for (; i < n; ++i) {
|
|
2308
|
+
int ai = a[i], bi = b[i];
|
|
2309
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
#pragma clang attribute pop
|
|
2316
|
+
#pragma GCC pop_options
|
|
2317
|
+
#endif // SIMSIMD_TARGET_SIERRA
|
|
2318
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
1740
2319
|
|
|
1741
2320
|
#ifdef __cplusplus
|
|
1742
2321
|
}
|