usearch 2.7.6 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +93 -0
- package/.github/workflows/Dockerfile.libs +6 -2
- package/.github/workflows/prerelease.yml +242 -245
- package/.github/workflows/release.yml +390 -356
- package/.gitignore +1 -0
- package/.vscode/settings.json +3 -1
- package/CITATION.cff +1 -1
- package/CMakeLists.txt +35 -5
- package/Cargo.toml +1 -1
- package/README.md +1 -1
- package/build.gradle +6 -0
- package/c/README.md +3 -3
- package/c/lib.cpp +75 -11
- package/c/usearch.h +22 -1
- package/conanfile.py +1 -1
- package/cpp/README.md +2 -1
- package/cpp/bench.cpp +4 -1
- package/cpp/test.cpp +12 -12
- package/csharp/nuget/nuget-package.props +1 -1
- package/docs/compilation.md +5 -13
- package/golang/lib.go +4 -1
- package/include/usearch/index.hpp +31 -30
- package/include/usearch/index_dense.hpp +49 -48
- package/include/usearch/index_plugins.hpp +91 -24
- package/java/cloud/unum/usearch/Index.java +118 -26
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +8 -8
- package/javascript/usearch.js +1 -0
- package/objc/USearchObjective.mm +4 -1
- package/objc/include/USearchObjective.h +1 -0
- package/package.json +1 -1
- package/python/lib.cpp +6 -6
- package/python/scripts/bench.ipynb +24 -2
- package/python/usearch/index.py +8 -3
- package/rust/lib.cpp +14 -12
- package/rust/lib.hpp +13 -13
- package/rust/lib.rs +46 -0
- package/setup.py +15 -3
- package/simsimd/.github/workflows/prerelease.yml +7 -8
- package/simsimd/.github/workflows/release.yml +18 -20
- package/simsimd/README.md +13 -8
- package/simsimd/binding.gyp +1 -1
- package/simsimd/cpp/bench.cxx +17 -3
- package/simsimd/include/simsimd/binary.h +1 -3
- package/simsimd/include/simsimd/probability.h +234 -361
- package/simsimd/include/simsimd/simsimd.h +95 -65
- package/simsimd/include/simsimd/spatial.h +0 -3
- package/simsimd/include/simsimd/types.h +12 -7
- package/simsimd/javascript/test.js +23 -5
- package/simsimd/package.json +2 -2
- package/wasmer.toml +1 -1
- package/wolfram/lib.cpp +1 -1
- package/csharp/eng/build_lib.sh +0 -65
- package/csharp/eng/clean.sh +0 -12
|
@@ -78,15 +78,30 @@ SIMSIMD_MAKE_JS(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16, 1e-6) // simsimd_acc
|
|
|
78
78
|
*/
|
|
79
79
|
|
|
80
80
|
__attribute__((target("+simd"))) //
|
|
81
|
-
inline
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
float32x4_t
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
81
|
+
inline float32x4_t
|
|
82
|
+
simsimd_neon_f32_log2(float32x4_t x) {
|
|
83
|
+
// Extracting the exponent
|
|
84
|
+
int32x4_t i = vreinterpretq_s32_f32(x);
|
|
85
|
+
int32x4_t e = vsubq_s32(vshrq_n_s32(vandq_s32(i, vdupq_n_s32(0x7F800000)), 23), vdupq_n_s32(127));
|
|
86
|
+
float32x4_t e_float = vcvtq_f32_s32(e);
|
|
87
|
+
|
|
88
|
+
// Extracting the mantissa
|
|
89
|
+
float32x4_t m = vreinterpretq_f32_s32(vorrq_s32(vandq_s32(i, vdupq_n_s32(0x007FFFFF)), vdupq_n_s32(0x3F800000)));
|
|
90
|
+
|
|
91
|
+
// Constants for polynomial
|
|
92
|
+
float32x4_t one = vdupq_n_f32(1.0f);
|
|
93
|
+
float32x4_t p = vdupq_n_f32(-3.4436006e-2f);
|
|
94
|
+
|
|
95
|
+
// Compute polynomial using Horner's method
|
|
96
|
+
p = vmlaq_f32(vdupq_n_f32(3.1821337e-1f), m, p);
|
|
97
|
+
p = vmlaq_f32(vdupq_n_f32(-1.2315303f), m, p);
|
|
98
|
+
p = vmlaq_f32(vdupq_n_f32(2.5988452f), m, p);
|
|
99
|
+
p = vmlaq_f32(vdupq_n_f32(-3.3241990f), m, p);
|
|
100
|
+
p = vmlaq_f32(vdupq_n_f32(3.1157899f), m, p);
|
|
101
|
+
|
|
102
|
+
// Final computation
|
|
103
|
+
float32x4_t result = vaddq_f32(vmulq_f32(p, vsubq_f32(m, one)), e_float);
|
|
104
|
+
return result;
|
|
90
105
|
}
|
|
91
106
|
|
|
92
107
|
__attribute__((target("+simd"))) //
|
|
@@ -100,11 +115,12 @@ simsimd_neon_f32_kl(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size
|
|
|
100
115
|
float32x4_t a_vec = vld1q_f32(a + i);
|
|
101
116
|
float32x4_t b_vec = vld1q_f32(b + i);
|
|
102
117
|
float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
|
|
103
|
-
float32x4_t log_ratio_vec =
|
|
118
|
+
float32x4_t log_ratio_vec = simsimd_neon_f32_log2(ratio_vec);
|
|
104
119
|
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
105
120
|
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
106
121
|
}
|
|
107
|
-
simsimd_f32_t
|
|
122
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
123
|
+
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
108
124
|
for (; i < n; ++i)
|
|
109
125
|
sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (b[i] + epsilon));
|
|
110
126
|
return sum;
|
|
@@ -119,22 +135,23 @@ simsimd_neon_f32_js(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size
|
|
|
119
135
|
for (; i + 4 <= n; i += 4) {
|
|
120
136
|
float32x4_t a_vec = vld1q_f32(a + i);
|
|
121
137
|
float32x4_t b_vec = vld1q_f32(b + i);
|
|
122
|
-
float32x4_t m_vec = vaddq_f32(a_vec, b_vec);
|
|
138
|
+
float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
|
|
123
139
|
float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
124
140
|
float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
125
|
-
float32x4_t log_ratio_a_vec =
|
|
126
|
-
float32x4_t log_ratio_b_vec =
|
|
141
|
+
float32x4_t log_ratio_a_vec = simsimd_neon_f32_log2(ratio_a_vec);
|
|
142
|
+
float32x4_t log_ratio_b_vec = simsimd_neon_f32_log2(ratio_b_vec);
|
|
127
143
|
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
128
144
|
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
129
145
|
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
130
146
|
}
|
|
131
|
-
simsimd_f32_t
|
|
147
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
148
|
+
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
132
149
|
for (; i < n; ++i) {
|
|
133
150
|
simsimd_f32_t mi = a[i] + b[i];
|
|
134
151
|
sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (mi + epsilon));
|
|
135
152
|
sum += b[i] * SIMSIMD_LOG((b[i] + epsilon) / (mi + epsilon));
|
|
136
153
|
}
|
|
137
|
-
return sum
|
|
154
|
+
return sum * 0.5f;
|
|
138
155
|
}
|
|
139
156
|
|
|
140
157
|
/*
|
|
@@ -158,11 +175,12 @@ simsimd_neon_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
|
|
|
158
175
|
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((float16_t const*)a + i));
|
|
159
176
|
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
|
|
160
177
|
float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
|
|
161
|
-
float32x4_t log_ratio_vec =
|
|
178
|
+
float32x4_t log_ratio_vec = simsimd_neon_f32_log2(ratio_vec);
|
|
162
179
|
float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
|
|
163
180
|
sum_vec = vaddq_f32(sum_vec, prod_vec);
|
|
164
181
|
}
|
|
165
|
-
simsimd_f32_t
|
|
182
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
183
|
+
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
166
184
|
for (; i < n; ++i)
|
|
167
185
|
sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
|
|
168
186
|
SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
|
|
@@ -179,22 +197,23 @@ simsimd_neon_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
|
|
|
179
197
|
for (; i + 4 <= n; i += 4) {
|
|
180
198
|
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((float16_t const*)a + i));
|
|
181
199
|
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
|
|
182
|
-
float32x4_t m_vec = vaddq_f32(a_vec, b_vec);
|
|
200
|
+
float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
|
|
183
201
|
float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
184
202
|
float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
|
|
185
|
-
float32x4_t log_ratio_a_vec =
|
|
186
|
-
float32x4_t log_ratio_b_vec =
|
|
203
|
+
float32x4_t log_ratio_a_vec = simsimd_neon_f32_log2(ratio_a_vec);
|
|
204
|
+
float32x4_t log_ratio_b_vec = simsimd_neon_f32_log2(ratio_b_vec);
|
|
187
205
|
float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
|
|
188
206
|
float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
|
|
189
207
|
sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
|
|
190
208
|
}
|
|
191
|
-
simsimd_f32_t
|
|
209
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
210
|
+
simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
|
|
192
211
|
for (; i < n; ++i) {
|
|
193
212
|
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
|
|
194
213
|
simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
|
|
195
214
|
simsimd_f32_t mi = ai + bi;
|
|
196
|
-
sum +=
|
|
197
|
-
sum +=
|
|
215
|
+
sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
216
|
+
sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
198
217
|
}
|
|
199
218
|
return sum / 2;
|
|
200
219
|
}
|
|
@@ -202,7 +221,7 @@ simsimd_neon_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
|
|
|
202
221
|
#endif // SIMSIMD_TARGET_ARM_NEON
|
|
203
222
|
#endif // SIMSIMD_TARGET_ARM
|
|
204
223
|
|
|
205
|
-
#if SIMSIMD_TARGET_X86
|
|
224
|
+
#if SIMSIMD_TARGET_X86
|
|
206
225
|
#if SIMSIMD_TARGET_X86_AVX2
|
|
207
226
|
|
|
208
227
|
/*
|
|
@@ -210,234 +229,112 @@ simsimd_neon_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
|
|
|
210
229
|
* @brief x86 AVX2 implementation of the most common similarity metrics for 16-bit floating point numbers.
|
|
211
230
|
* @author Ash Vardanian
|
|
212
231
|
*
|
|
213
|
-
* - Implements:
|
|
232
|
+
* - Implements: Kullback-Leibler and Jensen–Shannon divergence.
|
|
214
233
|
* - As AVX2 doesn't support masked loads of 16-bit words, implementations have a separate `for`-loop for tails.
|
|
215
234
|
* - Uses `f16` for both storage and `f32` for accumulation.
|
|
216
235
|
* - Requires compiler capabilities: avx2, f16c, fma.
|
|
217
236
|
*/
|
|
218
237
|
|
|
219
238
|
__attribute__((target("avx2,f16c,fma"))) //
|
|
220
|
-
inline
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
239
|
+
inline __m256
|
|
240
|
+
simsimd_avx2_f32_log2(__m256 x) {
|
|
241
|
+
// Extracting the exponent
|
|
242
|
+
__m256i i = _mm256_castps_si256(x);
|
|
243
|
+
__m256i e = _mm256_srli_epi32(_mm256_and_si256(i, _mm256_set1_epi32(0x7F800000)), 23);
|
|
244
|
+
e = _mm256_sub_epi32(e, _mm256_set1_epi32(127)); // removing the bias
|
|
245
|
+
__m256 e_float = _mm256_cvtepi32_ps(e);
|
|
246
|
+
|
|
247
|
+
// Extracting the mantissa
|
|
248
|
+
__m256 m = _mm256_castsi256_ps(
|
|
249
|
+
_mm256_or_si256(_mm256_and_si256(i, _mm256_set1_epi32(0x007FFFFF)), _mm256_set1_epi32(0x3F800000)));
|
|
250
|
+
|
|
251
|
+
// Constants for polynomial
|
|
252
|
+
__m256 one = _mm256_set1_ps(1.0f);
|
|
253
|
+
__m256 p = _mm256_set1_ps(-3.4436006e-2f);
|
|
254
|
+
|
|
255
|
+
// Compute the polynomial using Horner's method
|
|
256
|
+
p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(3.1821337e-1f));
|
|
257
|
+
p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(-1.2315303f));
|
|
258
|
+
p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(2.5988452f));
|
|
259
|
+
p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(-3.3241990f));
|
|
260
|
+
p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(3.1157899f));
|
|
261
|
+
|
|
262
|
+
// Final computation
|
|
263
|
+
__m256 result = _mm256_add_ps(_mm256_mul_ps(p, _mm256_sub_ps(m, one)), e_float);
|
|
243
264
|
return result;
|
|
244
265
|
}
|
|
245
266
|
|
|
246
267
|
__attribute__((target("avx2,f16c,fma"))) //
|
|
247
268
|
inline static simsimd_f32_t
|
|
248
|
-
|
|
249
|
-
__m256
|
|
269
|
+
simsimd_avx2_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
|
|
270
|
+
__m256 sum_vec = _mm256_set1_ps(0);
|
|
271
|
+
simsimd_f32_t epsilon = 1e-5;
|
|
272
|
+
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
250
273
|
simsimd_size_t i = 0;
|
|
251
274
|
for (; i + 8 <= n; i += 8) {
|
|
252
275
|
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
253
276
|
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
254
|
-
|
|
277
|
+
__m256 ratio_vec = _mm256_div_ps(_mm256_add_ps(a_vec, epsilon_vec), _mm256_add_ps(b_vec, epsilon_vec));
|
|
278
|
+
__m256 log_ratio_vec = simsimd_avx2_f32_log2(ratio_vec);
|
|
279
|
+
__m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
|
|
280
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_vec);
|
|
255
281
|
}
|
|
256
282
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
283
|
+
sum_vec = _mm256_add_ps(_mm256_permute2f128_ps(sum_vec, sum_vec, 1), sum_vec);
|
|
284
|
+
sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
|
|
285
|
+
sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
|
|
260
286
|
|
|
261
|
-
simsimd_f32_t
|
|
262
|
-
|
|
287
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
288
|
+
simsimd_f32_t sum;
|
|
289
|
+
_mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
|
|
290
|
+
sum *= log2_normalizer;
|
|
263
291
|
|
|
264
292
|
// Accumulate the tail:
|
|
265
293
|
for (; i < n; ++i)
|
|
266
|
-
|
|
267
|
-
|
|
294
|
+
sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
|
|
295
|
+
SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
|
|
296
|
+
return sum;
|
|
268
297
|
}
|
|
298
|
+
|
|
269
299
|
__attribute__((target("avx2,f16c,fma"))) //
|
|
270
300
|
inline static simsimd_f32_t
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
301
|
+
simsimd_avx2_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
|
|
302
|
+
__m256 sum_vec = _mm256_set1_ps(0);
|
|
303
|
+
simsimd_f32_t epsilon = 1e-5;
|
|
304
|
+
__m256 epsilon_vec = _mm256_set1_ps(epsilon);
|
|
274
305
|
simsimd_size_t i = 0;
|
|
275
306
|
for (; i + 8 <= n; i += 8) {
|
|
276
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
277
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
307
|
+
__m256 a_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i))), epsilon_vec);
|
|
308
|
+
__m256 b_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i))), epsilon_vec);
|
|
309
|
+
__m256 m_vec = _mm256_mul_ps(_mm256_add_ps(a_vec, b_vec), _mm256_set1_ps(0.5f)); // M = (P + Q) / 2
|
|
310
|
+
__m256 ratio_a_vec = _mm256_div_ps(a_vec, m_vec);
|
|
311
|
+
__m256 ratio_b_vec = _mm256_div_ps(b_vec, m_vec);
|
|
312
|
+
__m256 log_ratio_a_vec = simsimd_avx2_f32_log2(ratio_a_vec);
|
|
313
|
+
__m256 log_ratio_b_vec = simsimd_avx2_f32_log2(ratio_b_vec);
|
|
314
|
+
__m256 prod_a_vec = _mm256_mul_ps(a_vec, log_ratio_a_vec);
|
|
315
|
+
__m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
|
|
316
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
|
|
317
|
+
sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
|
|
281
318
|
}
|
|
282
319
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
|
|
288
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
289
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
320
|
+
sum_vec = _mm256_add_ps(_mm256_permute2f128_ps(sum_vec, sum_vec, 1), sum_vec);
|
|
321
|
+
sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
|
|
322
|
+
sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
|
|
290
323
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
simsimd_f32_t ab, a2, b2;
|
|
296
|
-
_mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
|
|
297
|
-
_mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
|
|
298
|
-
_mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
|
|
324
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
325
|
+
simsimd_f32_t sum;
|
|
326
|
+
_mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
|
|
327
|
+
sum *= log2_normalizer;
|
|
299
328
|
|
|
300
329
|
// Accumulate the tail:
|
|
301
330
|
for (; i < n; ++i) {
|
|
302
|
-
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i])
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
__m128 a2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
308
|
-
__m128 b2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
309
|
-
__m128 result = _mm_mul_ss(a2_sqrt_recjs, b2_sqrt_recjs); // Multjsly the recjsrocal square roots
|
|
310
|
-
result = _mm_mul_ss(result, _mm_set_ss((float)ab)); // Multjsly by ab
|
|
311
|
-
result = _mm_sub_ss(_mm_set_ss(1.0f), result); // Subtract from 1
|
|
312
|
-
return ab != 0 ? _mm_cvtss_f32(result) : 1; // Extract the final result
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
/*
|
|
316
|
-
* @file x86_avx2_i8.h
|
|
317
|
-
* @brief x86 AVX2 implementation of the most common similarity metrics for 8-bit signed integral numbers.
|
|
318
|
-
* @author Ash Vardanian
|
|
319
|
-
*
|
|
320
|
-
* - Implements: L2 squared, cosine similarity, inner product (same as cosine).
|
|
321
|
-
* - As AVX2 doesn't support masked loads of 16-bit words, implementations have a separate `for`-loop for tails.
|
|
322
|
-
* - Uses `i8` for storage, `i16` for multjslication, and `i32` for accumulation, if no better option is available.
|
|
323
|
-
* - Requires compiler capabilities: avx2.
|
|
324
|
-
*/
|
|
325
|
-
|
|
326
|
-
__attribute__((target("avx2"))) //
|
|
327
|
-
inline static simsimd_f32_t
|
|
328
|
-
simsimd_avx2_i8_kl(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
|
|
329
|
-
|
|
330
|
-
__m256i d2_high_vec = _mm256_setzero_si256();
|
|
331
|
-
__m256i d2_low_vec = _mm256_setzero_si256();
|
|
332
|
-
|
|
333
|
-
simsimd_size_t i = 0;
|
|
334
|
-
for (; i + 32 <= n; i += 32) {
|
|
335
|
-
__m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
|
|
336
|
-
__m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
|
|
337
|
-
|
|
338
|
-
// Sign extend int8 to int16
|
|
339
|
-
__m256i a_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_vec));
|
|
340
|
-
__m256i a_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
|
|
341
|
-
__m256i b_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_vec));
|
|
342
|
-
__m256i b_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
|
|
343
|
-
|
|
344
|
-
// Subtract and multjsly
|
|
345
|
-
__m256i d_low = _mm256_sub_epi16(a_low, b_low);
|
|
346
|
-
__m256i d_high = _mm256_sub_epi16(a_high, b_high);
|
|
347
|
-
__m256i d2_low_part = _mm256_madd_epi16(d_low, d_low);
|
|
348
|
-
__m256i d2_high_part = _mm256_madd_epi16(d_high, d_high);
|
|
349
|
-
|
|
350
|
-
// Accumulate into int32 vectors
|
|
351
|
-
d2_low_vec = _mm256_add_epi32(d2_low_vec, d2_low_part);
|
|
352
|
-
d2_high_vec = _mm256_add_epi32(d2_high_vec, d2_high_part);
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
// Accumulate the 32-bit integers from `d2_high_vec` and `d2_low_vec`
|
|
356
|
-
__m256i d2_vec = _mm256_add_epi32(d2_low_vec, d2_high_vec);
|
|
357
|
-
__m128i d2_sum = _mm_add_epi32(_mm256_extracti128_si256(d2_vec, 0), _mm256_extracti128_si256(d2_vec, 1));
|
|
358
|
-
d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
|
|
359
|
-
d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
|
|
360
|
-
int d2 = _mm_extract_epi32(d2_sum, 0);
|
|
361
|
-
|
|
362
|
-
// Take care of the tail:
|
|
363
|
-
for (; i < n; ++i) {
|
|
364
|
-
int n = a[i] - b[i];
|
|
365
|
-
d2 += n * n;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
return (simsimd_f32_t)d2;
|
|
369
|
-
}
|
|
370
|
-
__attribute__((target("avx2"))) //
|
|
371
|
-
inline static simsimd_f32_t
|
|
372
|
-
simsimd_avx2_i8_cos(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
|
|
373
|
-
|
|
374
|
-
__m256i ab_high_vec = _mm256_setzero_si256();
|
|
375
|
-
__m256i ab_low_vec = _mm256_setzero_si256();
|
|
376
|
-
__m256i a2_high_vec = _mm256_setzero_si256();
|
|
377
|
-
__m256i a2_low_vec = _mm256_setzero_si256();
|
|
378
|
-
__m256i b2_high_vec = _mm256_setzero_si256();
|
|
379
|
-
__m256i b2_low_vec = _mm256_setzero_si256();
|
|
380
|
-
|
|
381
|
-
simsimd_size_t i = 0;
|
|
382
|
-
for (; i + 32 <= n; i += 32) {
|
|
383
|
-
__m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
|
|
384
|
-
__m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
|
|
385
|
-
|
|
386
|
-
// Unpack int8 to int32
|
|
387
|
-
__m256i a_low = _mm256_cvtepi8_epi32(_mm256_castsi256_si128(a_vec));
|
|
388
|
-
__m256i a_high = _mm256_cvtepi8_epi32(_mm256_extracti128_si256(a_vec, 1));
|
|
389
|
-
__m256i b_low = _mm256_cvtepi8_epi32(_mm256_castsi256_si128(b_vec));
|
|
390
|
-
__m256i b_high = _mm256_cvtepi8_epi32(_mm256_extracti128_si256(b_vec, 1));
|
|
391
|
-
|
|
392
|
-
// Multjsly and accumulate
|
|
393
|
-
ab_low_vec = _mm256_add_epi32(ab_low_vec, _mm256_mullo_epi32(a_low, b_low));
|
|
394
|
-
ab_high_vec = _mm256_add_epi32(ab_high_vec, _mm256_mullo_epi32(a_high, b_high));
|
|
395
|
-
a2_low_vec = _mm256_add_epi32(a2_low_vec, _mm256_mullo_epi32(a_low, a_low));
|
|
396
|
-
a2_high_vec = _mm256_add_epi32(a2_high_vec, _mm256_mullo_epi32(a_high, a_high));
|
|
397
|
-
b2_low_vec = _mm256_add_epi32(b2_low_vec, _mm256_mullo_epi32(b_low, b_low));
|
|
398
|
-
b2_high_vec = _mm256_add_epi32(b2_high_vec, _mm256_mullo_epi32(b_high, b_high));
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
// Horizontal sum across the 256-bit register
|
|
402
|
-
__m256i ab_vec = _mm256_add_epi32(ab_low_vec, ab_high_vec);
|
|
403
|
-
__m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
|
|
404
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
405
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
406
|
-
|
|
407
|
-
__m256i a2_vec = _mm256_add_epi32(a2_low_vec, a2_high_vec);
|
|
408
|
-
__m128i a2_sum = _mm_add_epi32(_mm256_extracti128_si256(a2_vec, 0), _mm256_extracti128_si256(a2_vec, 1));
|
|
409
|
-
a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
|
|
410
|
-
a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
|
|
411
|
-
|
|
412
|
-
__m256i b2_vec = _mm256_add_epi32(b2_low_vec, b2_high_vec);
|
|
413
|
-
__m128i b2_sum = _mm_add_epi32(_mm256_extracti128_si256(b2_vec, 0), _mm256_extracti128_si256(b2_vec, 1));
|
|
414
|
-
b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
|
|
415
|
-
b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
|
|
416
|
-
|
|
417
|
-
// Further reduce to a single sum for each vector
|
|
418
|
-
int ab = _mm_extract_epi32(ab_sum, 0);
|
|
419
|
-
int a2 = _mm_extract_epi32(a2_sum, 0);
|
|
420
|
-
int b2 = _mm_extract_epi32(b2_sum, 0);
|
|
421
|
-
|
|
422
|
-
// Take care of the tail:
|
|
423
|
-
for (; i < n; ++i) {
|
|
424
|
-
int ai = a[i], bi = b[i];
|
|
425
|
-
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
331
|
+
simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
|
|
332
|
+
simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
|
|
333
|
+
simsimd_f32_t mi = ai + bi;
|
|
334
|
+
sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
|
|
335
|
+
sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
|
|
426
336
|
}
|
|
427
|
-
|
|
428
|
-
// Replace simsimd_approximate_inverse_square_root with `rsqrtss`
|
|
429
|
-
__m128 a2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
430
|
-
__m128 b2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
431
|
-
__m128 result = _mm_mul_ss(a2_sqrt_recjs, b2_sqrt_recjs); // Multjsly the recjsrocal square roots
|
|
432
|
-
result = _mm_mul_ss(result, _mm_set_ss((float)ab)); // Multjsly by ab
|
|
433
|
-
result = _mm_sub_ss(_mm_set_ss(1.0f), result); // Subtract from 1
|
|
434
|
-
return ab != 0 ? _mm_cvtss_f32(result) : 1; // Extract the final result
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
__attribute__((target("avx2"))) //
|
|
438
|
-
inline static simsimd_f32_t
|
|
439
|
-
simsimd_avx2_i8_js(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
|
|
440
|
-
return simsimd_avx2_i8_cos(a, b, n);
|
|
337
|
+
return sum / 2;
|
|
441
338
|
}
|
|
442
339
|
|
|
443
340
|
#endif // SIMSIMD_TARGET_X86_AVX2
|
|
@@ -449,63 +346,84 @@ simsimd_avx2_i8_js(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t
|
|
|
449
346
|
* @brief x86 AVX-512 implementation of the most common similarity metrics for 32-bit floating point numbers.
|
|
450
347
|
* @author Ash Vardanian
|
|
451
348
|
*
|
|
452
|
-
* - Implements:
|
|
349
|
+
* - Implements: Kullback-Leibler and Jensen–Shannon divergence.
|
|
453
350
|
* - Uses `f32` for storage and `f32` for accumulation.
|
|
454
351
|
* - Requires compiler capabilities: avx512f, avx512vl.
|
|
455
352
|
*/
|
|
456
353
|
|
|
457
354
|
__attribute__((target("avx512f,avx512vl"))) //
|
|
458
|
-
inline
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
355
|
+
inline __m512
|
|
356
|
+
simsimd_avx512_f32_log2(__m512 x) {
|
|
357
|
+
// Extract the exponent and mantissa
|
|
358
|
+
__m512 one = _mm512_set1_ps(1.0f);
|
|
359
|
+
__m512 e = _mm512_getexp_ps(x);
|
|
360
|
+
__m512 m = _mm512_getmant_ps(x, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_src);
|
|
361
|
+
|
|
362
|
+
// Compute the polynomial using Horner's method
|
|
363
|
+
__m512 p = _mm512_set1_ps(-3.4436006e-2f);
|
|
364
|
+
p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(3.1821337e-1f));
|
|
365
|
+
p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(-1.2315303f));
|
|
366
|
+
p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(2.5988452f));
|
|
367
|
+
p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(-3.3241990f));
|
|
368
|
+
p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(3.1157899f));
|
|
369
|
+
|
|
370
|
+
return _mm512_add_ps(_mm512_mul_ps(p, _mm512_sub_ps(m, one)), e);
|
|
469
371
|
}
|
|
470
372
|
|
|
471
373
|
__attribute__((target("avx512f,avx512vl"))) //
|
|
472
374
|
inline static simsimd_f32_t
|
|
473
|
-
|
|
474
|
-
__m512
|
|
375
|
+
simsimd_avx512_f32_kl(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
|
|
376
|
+
__m512 sum_vec = _mm512_set1_ps(0);
|
|
377
|
+
simsimd_f32_t epsilon = 1e-6;
|
|
378
|
+
__m512 epsilon_vec = _mm512_set1_ps(epsilon);
|
|
475
379
|
for (simsimd_size_t i = 0; i < n; i += 16) {
|
|
476
380
|
__mmask16 mask = n - i >= 16 ? 0xFFFF : ((1u << (n - i)) - 1u);
|
|
477
|
-
__m512 a_vec = _mm512_maskz_loadu_ps(mask, a + i);
|
|
478
|
-
__m512 b_vec = _mm512_maskz_loadu_ps(mask, b + i);
|
|
479
|
-
|
|
381
|
+
__m512 a_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, a + i), epsilon_vec);
|
|
382
|
+
__m512 b_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, b + i), epsilon_vec);
|
|
383
|
+
__m512 ratio_vec = _mm512_div_ps(a_vec, b_vec);
|
|
384
|
+
__m512 log_ratio_vec = simsimd_avx512_f32_log2(ratio_vec);
|
|
385
|
+
__m512 prod_vec = _mm512_mul_ps(a_vec, log_ratio_vec);
|
|
386
|
+
sum_vec = _mm512_maskz_add_ps(mask, sum_vec, prod_vec);
|
|
480
387
|
}
|
|
481
|
-
|
|
388
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
389
|
+
return _mm512_reduce_add_ps(sum_vec) * log2_normalizer;
|
|
482
390
|
}
|
|
483
391
|
|
|
484
392
|
__attribute__((target("avx512f,avx512vl"))) //
|
|
485
393
|
inline static simsimd_f32_t
|
|
486
|
-
|
|
487
|
-
__m512
|
|
488
|
-
__m512
|
|
489
|
-
|
|
490
|
-
|
|
394
|
+
simsimd_avx512_f32_js(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
|
|
395
|
+
__m512 sum_a_vec = _mm512_set1_ps(0);
|
|
396
|
+
__m512 sum_b_vec = _mm512_set1_ps(0);
|
|
397
|
+
simsimd_f32_t epsilon = 1e-6;
|
|
398
|
+
__m512 epsilon_vec = _mm512_set1_ps(epsilon);
|
|
491
399
|
for (simsimd_size_t i = 0; i < n; i += 16) {
|
|
492
400
|
__mmask16 mask = n - i >= 16 ? 0xFFFF : ((1u << (n - i)) - 1u);
|
|
493
401
|
__m512 a_vec = _mm512_maskz_loadu_ps(mask, a + i);
|
|
494
402
|
__m512 b_vec = _mm512_maskz_loadu_ps(mask, b + i);
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
403
|
+
__m512 m_vec = _mm512_mul_ps(_mm512_add_ps(a_vec, b_vec), _mm512_set1_ps(0.5f)); // M = (P + Q) / 2
|
|
404
|
+
|
|
405
|
+
// Avoid division by zero problems from probabilities under zero down the road.
|
|
406
|
+
// Masking is a nicer way to do this, than adding the `epsilon` to every component.
|
|
407
|
+
__mmask16 nonzero_mask_a = _mm512_cmp_ps_mask(a_vec, epsilon_vec, _CMP_GE_OQ);
|
|
408
|
+
__mmask16 nonzero_mask_b = _mm512_cmp_ps_mask(b_vec, epsilon_vec, _CMP_GE_OQ);
|
|
409
|
+
__mmask16 nonzero_mask = nonzero_mask_a & nonzero_mask_b & mask;
|
|
410
|
+
|
|
411
|
+
// Division is an expensive operation. Instead of doing it twice,
|
|
412
|
+
// we can approximate the reciprocal of `m` and multiply instead.
|
|
413
|
+
__m512 m_recip_approx = _mm512_rcp14_ps(m_vec); // or `_mm512_rcp28_ps` for slightly better precision
|
|
414
|
+
__m512 ratio_a_vec = _mm512_mul_ps(a_vec, m_recip_approx);
|
|
415
|
+
__m512 ratio_b_vec = _mm512_mul_ps(b_vec, m_recip_approx);
|
|
416
|
+
|
|
417
|
+
// The natural logarithm is equivalent to `log2`, multiplied by the `loge(2)`
|
|
418
|
+
__m512 log_ratio_a_vec = simsimd_avx512_f32_log2(ratio_a_vec);
|
|
419
|
+
__m512 log_ratio_b_vec = simsimd_avx512_f32_log2(ratio_b_vec);
|
|
420
|
+
|
|
421
|
+
// Instead of separate multiplication and addition, invoke the FMA
|
|
422
|
+
sum_a_vec = _mm512_maskz_fmadd_ps(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
423
|
+
sum_b_vec = _mm512_maskz_fmadd_ps(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
498
424
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
|
|
502
|
-
simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
|
|
503
|
-
|
|
504
|
-
__m128d a2_b2 = _mm_set_pd((double)a2, (double)b2);
|
|
505
|
-
__m128d rsqrts = _mm_mask_rsqrt14_pd(_mm_setzero_pd(), 0xFF, a2_b2);
|
|
506
|
-
double rsqrts_array[2];
|
|
507
|
-
_mm_storeu_pd(rsqrts_array, rsqrts);
|
|
508
|
-
return ab != 0 ? 1 - ab * rsqrts_array[0] * rsqrts_array[1] : 1;
|
|
425
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
426
|
+
return _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) * 0.5f * log2_normalizer;
|
|
509
427
|
}
|
|
510
428
|
|
|
511
429
|
/*
|
|
@@ -513,128 +431,83 @@ simsimd_avx512_f32_cos(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_s
|
|
|
513
431
|
* @brief x86 AVX-512 implementation of the most common similarity metrics for 16-bit floating point numbers.
|
|
514
432
|
* @author Ash Vardanian
|
|
515
433
|
*
|
|
516
|
-
* - Implements:
|
|
434
|
+
* - Implements: Kullback-Leibler and Jensen–Shannon divergence.
|
|
517
435
|
* - Uses `_mm512_maskz_loadu_epi16` intrinsics to perform masked unaligned loads.
|
|
518
436
|
* - Uses `f16` for both storage and accumulation, assuming it's resolution is enough for average case.
|
|
519
437
|
* - Requires compiler capabilities: avx512fp16, avx512f, avx512vl.
|
|
520
438
|
*/
|
|
521
439
|
|
|
522
|
-
__attribute__((target("
|
|
523
|
-
inline
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
440
|
+
__attribute__((target("avx512f,avx512vl,avx512fp16"))) //
|
|
441
|
+
inline __m512h
|
|
442
|
+
simsimd_avx512_f16_log2(__m512h x) {
|
|
443
|
+
// Extract the exponent and mantissa
|
|
444
|
+
__m512h one = _mm512_set1_ph((_Float16)1);
|
|
445
|
+
__m512h e = _mm512_getexp_ph(x);
|
|
446
|
+
__m512h m = _mm512_getmant_ph(x, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_src);
|
|
447
|
+
|
|
448
|
+
// Compute the polynomial using Horner's method
|
|
449
|
+
__m512h p = _mm512_set1_ph((_Float16)-3.4436006e-2f);
|
|
450
|
+
p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)3.1821337e-1f));
|
|
451
|
+
p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)-1.2315303f));
|
|
452
|
+
p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)2.5988452f));
|
|
453
|
+
p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)-3.3241990f));
|
|
454
|
+
p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)3.1157899f));
|
|
455
|
+
|
|
456
|
+
return _mm512_add_ph(_mm512_mul_ph(p, _mm512_sub_ph(m, one)), e);
|
|
534
457
|
}
|
|
535
458
|
|
|
536
|
-
__attribute__((target("
|
|
459
|
+
__attribute__((target("avx512f,avx512vl,avx512fp16"))) //
|
|
537
460
|
inline static simsimd_f32_t
|
|
538
|
-
|
|
539
|
-
__m512h
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
for (simsimd_size_t i = 0; i < n; i += 32) {
|
|
543
|
-
__mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
|
|
544
|
-
__m512i a_vec = _mm512_maskz_loadu_epi16(mask, a + i);
|
|
545
|
-
__m512i b_vec = _mm512_maskz_loadu_epi16(mask, b + i);
|
|
546
|
-
ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_vec);
|
|
547
|
-
}
|
|
548
|
-
return 1 - _mm512_reduce_add_ph(ab_vec);
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
__attribute__((target("avx512fp16,avx512vl,avx512f"))) //
|
|
552
|
-
inline static simsimd_f32_t
|
|
553
|
-
simsimd_avx512_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
|
|
554
|
-
__m512h ab_vec = _mm512_set1_ph(0);
|
|
555
|
-
__m512h a2_vec = _mm512_set1_ph(0);
|
|
556
|
-
__m512h b2_vec = _mm512_set1_ph(0);
|
|
557
|
-
|
|
558
|
-
for (simsimd_size_t i = 0; i < n; i += 32) {
|
|
559
|
-
__mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
|
|
560
|
-
__m512i a_vec = _mm512_maskz_loadu_epi16(mask, a + i);
|
|
561
|
-
__m512i b_vec = _mm512_maskz_loadu_epi16(mask, b + i);
|
|
562
|
-
ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_vec);
|
|
563
|
-
a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(a_vec), a2_vec);
|
|
564
|
-
b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(b_vec), b2_vec);
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
|
|
568
|
-
simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
|
|
569
|
-
simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
|
|
570
|
-
|
|
571
|
-
__m128d a2_b2 = _mm_set_pd((double)a2, (double)b2);
|
|
572
|
-
__m128d rsqrts = _mm_mask_rsqrt14_pd(_mm_setzero_pd(), 0xFF, a2_b2);
|
|
573
|
-
double rsqrts_array[2];
|
|
574
|
-
_mm_storeu_pd(rsqrts_array, rsqrts);
|
|
575
|
-
return ab != 0 ? 1 - ab * rsqrts_array[0] * rsqrts_array[1] : 1;
|
|
576
|
-
}
|
|
577
|
-
|
|
578
|
-
/*
|
|
579
|
-
* @file x86_avx512_i8.h
|
|
580
|
-
* @brief x86 AVX-512 implementation of the most common similarity metrics for 8-bit integers.
|
|
581
|
-
* @author Ash Vardanian
|
|
582
|
-
*
|
|
583
|
-
* - Implements: L2 squared, cosine similarity, inner product (same as cosine).
|
|
584
|
-
* - Uses `_mm512_maskz_loadu_epi16` intrinsics to perform masked unaligned loads.
|
|
585
|
-
* - Uses `i8` for storage, `i16` for multjslication, and `i32` for accumulation, if no better option is available.
|
|
586
|
-
* - Requires compiler capabilities: avx512f, avx512vl, avx512bw.
|
|
587
|
-
*/
|
|
588
|
-
|
|
589
|
-
__attribute__((target("avx512vl,avx512f,avx512bw"))) //
|
|
590
|
-
inline static simsimd_f32_t
|
|
591
|
-
simsimd_avx512_i8_kl(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
|
|
592
|
-
__m512i d2_i32s_vec = _mm512_setzero_si512();
|
|
593
|
-
|
|
461
|
+
simsimd_avx512_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
|
|
462
|
+
__m512h sum_vec = _mm512_set1_ph((_Float16)0);
|
|
463
|
+
__m512h epsilon_vec = _mm512_set1_ph((_Float16)1e-6f);
|
|
594
464
|
for (simsimd_size_t i = 0; i < n; i += 32) {
|
|
595
465
|
__mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
466
|
+
__m512h a_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a + i)), epsilon_vec);
|
|
467
|
+
__m512h b_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b + i)), epsilon_vec);
|
|
468
|
+
__m512h ratio_vec = _mm512_div_ph(a_vec, b_vec);
|
|
469
|
+
__m512h log_ratio_vec = simsimd_avx512_f16_log2(ratio_vec);
|
|
470
|
+
__m512h prod_vec = _mm512_mul_ph(a_vec, log_ratio_vec);
|
|
471
|
+
sum_vec = _mm512_maskz_add_ph(mask, sum_vec, prod_vec);
|
|
600
472
|
}
|
|
601
|
-
|
|
602
|
-
return
|
|
473
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
474
|
+
return _mm512_reduce_add_ph(sum_vec) * log2_normalizer;
|
|
603
475
|
}
|
|
604
476
|
|
|
605
|
-
__attribute__((target("avx512vl,
|
|
477
|
+
__attribute__((target("avx512f,avx512vl,avx512fp16"))) //
|
|
606
478
|
inline static simsimd_f32_t
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
simsimd_size_t i = 0;
|
|
612
|
-
|
|
479
|
+
simsimd_avx512_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
|
|
480
|
+
__m512h sum_a_vec = _mm512_set1_ph((_Float16)0);
|
|
481
|
+
__m512h sum_b_vec = _mm512_set1_ph((_Float16)0);
|
|
482
|
+
__m512h epsilon_vec = _mm512_set1_ph((_Float16)1e-6f);
|
|
613
483
|
for (simsimd_size_t i = 0; i < n; i += 32) {
|
|
614
484
|
__mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
485
|
+
__m512h a_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a + i));
|
|
486
|
+
__m512h b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b + i));
|
|
487
|
+
__m512h m_vec = _mm512_mul_ph(_mm512_add_ph(a_vec, b_vec), _mm512_set1_ph((_Float16)0.5f)); // M = (P + Q) / 2
|
|
488
|
+
|
|
489
|
+
// Avoid division by zero problems from probabilities under zero down the road.
|
|
490
|
+
// Masking is a nicer way to do this, than adding the `epsilon` to every component.
|
|
491
|
+
__mmask32 nonzero_mask_a = _mm512_cmp_ph_mask(a_vec, epsilon_vec, _CMP_GE_OQ);
|
|
492
|
+
__mmask32 nonzero_mask_b = _mm512_cmp_ph_mask(b_vec, epsilon_vec, _CMP_GE_OQ);
|
|
493
|
+
__mmask32 nonzero_mask = nonzero_mask_a & nonzero_mask_b & mask;
|
|
494
|
+
|
|
495
|
+
// Division is an expensive operation. Instead of doing it twice,
|
|
496
|
+
// we can approximate the reciprocal of `m` and multiply instead.
|
|
497
|
+
__m512h m_recip_approx = _mm512_rcp_ph(m_vec);
|
|
498
|
+
__m512h ratio_a_vec = _mm512_mul_ph(a_vec, m_recip_approx);
|
|
499
|
+
__m512h ratio_b_vec = _mm512_mul_ph(b_vec, m_recip_approx);
|
|
500
|
+
|
|
501
|
+
// The natural logarithm is equivalent to `log2`, multiplied by the `loge(2)`
|
|
502
|
+
__m512h log_ratio_a_vec = simsimd_avx512_f16_log2(ratio_a_vec);
|
|
503
|
+
__m512h log_ratio_b_vec = simsimd_avx512_f16_log2(ratio_b_vec);
|
|
504
|
+
|
|
505
|
+
// Instead of separate multiplication and addition, invoke the FMA
|
|
506
|
+
sum_a_vec = _mm512_maskz_fmadd_ph(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
|
|
507
|
+
sum_b_vec = _mm512_maskz_fmadd_ph(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
|
|
621
508
|
}
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
int a2 = _mm512_reduce_add_epi32(a2_i32s_vec);
|
|
625
|
-
int b2 = _mm512_reduce_add_epi32(b2_i32s_vec);
|
|
626
|
-
|
|
627
|
-
__m128d a2_b2 = _mm_set_pd((double)a2, (double)b2);
|
|
628
|
-
__m128d rsqrts = _mm_mask_rsqrt14_pd(_mm_setzero_pd(), 0xFF, a2_b2);
|
|
629
|
-
double rsqrts_array[2];
|
|
630
|
-
_mm_storeu_pd(rsqrts_array, rsqrts);
|
|
631
|
-
return ab != 0 ? 1 - ab * rsqrts_array[0] * rsqrts_array[1] : 1;
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
__attribute__((target("avx512vl,avx512f,avx512bw"))) //
|
|
635
|
-
inline static simsimd_f32_t
|
|
636
|
-
simsimd_avx512_i8_js(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
|
|
637
|
-
return simsimd_avx512_i8_cos(a, b, n);
|
|
509
|
+
simsimd_f32_t log2_normalizer = 0.693147181f;
|
|
510
|
+
return _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) * 0.5f * log2_normalizer;
|
|
638
511
|
}
|
|
639
512
|
|
|
640
513
|
#endif // SIMSIMD_TARGET_X86_AVX512
|