usearch 2.7.7 → 2.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/.dockerignore +93 -0
  2. package/.github/workflows/Dockerfile.libs +6 -2
  3. package/.github/workflows/prerelease.yml +242 -245
  4. package/.github/workflows/release.yml +399 -357
  5. package/.gitignore +1 -0
  6. package/.vscode/settings.json +3 -1
  7. package/CITATION.cff +1 -1
  8. package/CMakeLists.txt +35 -5
  9. package/Cargo.toml +1 -1
  10. package/README.md +1 -1
  11. package/build.gradle +6 -0
  12. package/c/README.md +3 -3
  13. package/c/lib.cpp +75 -11
  14. package/c/usearch.h +22 -1
  15. package/conanfile.py +1 -1
  16. package/cpp/README.md +2 -1
  17. package/cpp/bench.cpp +3 -0
  18. package/cpp/test.cpp +12 -12
  19. package/csharp/nuget/nuget-package.props +1 -1
  20. package/docs/compilation.md +5 -13
  21. package/golang/lib.go +4 -1
  22. package/include/usearch/index.hpp +31 -30
  23. package/include/usearch/index_dense.hpp +49 -48
  24. package/include/usearch/index_plugins.hpp +70 -8
  25. package/java/cloud/unum/usearch/Index.java +118 -26
  26. package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +8 -8
  27. package/javascript/usearch.js +1 -0
  28. package/objc/USearchObjective.mm +4 -1
  29. package/objc/include/USearchObjective.h +1 -0
  30. package/package.json +1 -1
  31. package/pyproject.toml +10 -8
  32. package/python/lib.cpp +3 -2
  33. package/python/scripts/bench.ipynb +24 -2
  34. package/python/usearch/index.py +8 -3
  35. package/rust/lib.cpp +14 -12
  36. package/rust/lib.hpp +13 -13
  37. package/rust/lib.rs +47 -0
  38. package/setup.py +15 -3
  39. package/simsimd/.github/workflows/prerelease.yml +7 -8
  40. package/simsimd/.github/workflows/release.yml +18 -20
  41. package/simsimd/README.md +13 -8
  42. package/simsimd/binding.gyp +1 -1
  43. package/simsimd/cpp/bench.cxx +17 -3
  44. package/simsimd/include/simsimd/binary.h +1 -3
  45. package/simsimd/include/simsimd/probability.h +234 -361
  46. package/simsimd/include/simsimd/simsimd.h +95 -65
  47. package/simsimd/include/simsimd/spatial.h +0 -3
  48. package/simsimd/include/simsimd/types.h +12 -7
  49. package/simsimd/javascript/test.js +23 -5
  50. package/simsimd/package.json +2 -2
  51. package/wasmer.toml +1 -1
  52. package/wolfram/lib.cpp +1 -1
  53. package/csharp/eng/build_lib.sh +0 -65
  54. package/csharp/eng/clean.sh +0 -12
@@ -78,15 +78,30 @@ SIMSIMD_MAKE_JS(accurate, f16, f64, SIMSIMD_UNCOMPRESS_F16, 1e-6) // simsimd_acc
78
78
  */
79
79
 
80
80
  __attribute__((target("+simd"))) //
81
- inline static float32x4_t
82
- simsimd_neon_f32_log(float32x4_t x) {
83
- float32x4_t a = vmlaq_f32(vdupq_n_f32(-2.29561495781f), vdupq_n_f32(5.17591238022f), x);
84
- float32x4_t b = vmlaq_f32(vdupq_n_f32(-5.68692588806f), vdupq_n_f32(0.844007015228f), x);
85
- float32x4_t c = vmlaq_f32(vdupq_n_f32(-2.47071170807f), vdupq_n_f32(4.58445882797f), x);
86
- float32x4_t d = vmlaq_f32(vdupq_n_f32(-0.165253549814f), vdupq_n_f32(0.0141278216615f), x);
87
- float32x4_t x2 = vmulq_f32(x, x);
88
- float32x4_t x4 = vmulq_f32(x2, x2);
89
- return vmlaq_f32(vmlaq_f32(a, b, x2), vmlaq_f32(c, d, x2), x4);
81
+ inline float32x4_t
82
+ simsimd_neon_f32_log2(float32x4_t x) {
83
+ // Extracting the exponent
84
+ int32x4_t i = vreinterpretq_s32_f32(x);
85
+ int32x4_t e = vsubq_s32(vshrq_n_s32(vandq_s32(i, vdupq_n_s32(0x7F800000)), 23), vdupq_n_s32(127));
86
+ float32x4_t e_float = vcvtq_f32_s32(e);
87
+
88
+ // Extracting the mantissa
89
+ float32x4_t m = vreinterpretq_f32_s32(vorrq_s32(vandq_s32(i, vdupq_n_s32(0x007FFFFF)), vdupq_n_s32(0x3F800000)));
90
+
91
+ // Constants for polynomial
92
+ float32x4_t one = vdupq_n_f32(1.0f);
93
+ float32x4_t p = vdupq_n_f32(-3.4436006e-2f);
94
+
95
+ // Compute polynomial using Horner's method
96
+ p = vmlaq_f32(vdupq_n_f32(3.1821337e-1f), m, p);
97
+ p = vmlaq_f32(vdupq_n_f32(-1.2315303f), m, p);
98
+ p = vmlaq_f32(vdupq_n_f32(2.5988452f), m, p);
99
+ p = vmlaq_f32(vdupq_n_f32(-3.3241990f), m, p);
100
+ p = vmlaq_f32(vdupq_n_f32(3.1157899f), m, p);
101
+
102
+ // Final computation
103
+ float32x4_t result = vaddq_f32(vmulq_f32(p, vsubq_f32(m, one)), e_float);
104
+ return result;
90
105
  }
91
106
 
92
107
  __attribute__((target("+simd"))) //
@@ -100,11 +115,12 @@ simsimd_neon_f32_kl(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size
100
115
  float32x4_t a_vec = vld1q_f32(a + i);
101
116
  float32x4_t b_vec = vld1q_f32(b + i);
102
117
  float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
103
- float32x4_t log_ratio_vec = simsimd_neon_f32_log(ratio_vec);
118
+ float32x4_t log_ratio_vec = simsimd_neon_f32_log2(ratio_vec);
104
119
  float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
105
120
  sum_vec = vaddq_f32(sum_vec, prod_vec);
106
121
  }
107
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
122
+ simsimd_f32_t log2_normalizer = 0.693147181f;
123
+ simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
108
124
  for (; i < n; ++i)
109
125
  sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (b[i] + epsilon));
110
126
  return sum;
@@ -119,22 +135,23 @@ simsimd_neon_f32_js(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size
119
135
  for (; i + 4 <= n; i += 4) {
120
136
  float32x4_t a_vec = vld1q_f32(a + i);
121
137
  float32x4_t b_vec = vld1q_f32(b + i);
122
- float32x4_t m_vec = vaddq_f32(a_vec, b_vec); // M = P + Q
138
+ float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
123
139
  float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
124
140
  float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
125
- float32x4_t log_ratio_a_vec = simsimd_neon_f32_log(ratio_a_vec);
126
- float32x4_t log_ratio_b_vec = simsimd_neon_f32_log(ratio_b_vec);
141
+ float32x4_t log_ratio_a_vec = simsimd_neon_f32_log2(ratio_a_vec);
142
+ float32x4_t log_ratio_b_vec = simsimd_neon_f32_log2(ratio_b_vec);
127
143
  float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
128
144
  float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
129
145
  sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
130
146
  }
131
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
147
+ simsimd_f32_t log2_normalizer = 0.693147181f;
148
+ simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
132
149
  for (; i < n; ++i) {
133
150
  simsimd_f32_t mi = a[i] + b[i];
134
151
  sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (mi + epsilon));
135
152
  sum += b[i] * SIMSIMD_LOG((b[i] + epsilon) / (mi + epsilon));
136
153
  }
137
- return sum / 2;
154
+ return sum * 0.5f;
138
155
  }
139
156
 
140
157
  /*
@@ -158,11 +175,12 @@ simsimd_neon_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
158
175
  float32x4_t a_vec = vcvt_f32_f16(vld1_f16((float16_t const*)a + i));
159
176
  float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
160
177
  float32x4_t ratio_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(b_vec, epsilon_vec));
161
- float32x4_t log_ratio_vec = simsimd_neon_f32_log(ratio_vec);
178
+ float32x4_t log_ratio_vec = simsimd_neon_f32_log2(ratio_vec);
162
179
  float32x4_t prod_vec = vmulq_f32(a_vec, log_ratio_vec);
163
180
  sum_vec = vaddq_f32(sum_vec, prod_vec);
164
181
  }
165
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
182
+ simsimd_f32_t log2_normalizer = 0.693147181f;
183
+ simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
166
184
  for (; i < n; ++i)
167
185
  sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
168
186
  SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
@@ -179,22 +197,23 @@ simsimd_neon_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
179
197
  for (; i + 4 <= n; i += 4) {
180
198
  float32x4_t a_vec = vcvt_f32_f16(vld1_f16((float16_t const*)a + i));
181
199
  float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
182
- float32x4_t m_vec = vaddq_f32(a_vec, b_vec); // M = P + Q
200
+ float32x4_t m_vec = vmulq_f32(vaddq_f32(a_vec, b_vec), vdupq_n_f32(0.5));
183
201
  float32x4_t ratio_a_vec = vdivq_f32(vaddq_f32(a_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
184
202
  float32x4_t ratio_b_vec = vdivq_f32(vaddq_f32(b_vec, epsilon_vec), vaddq_f32(m_vec, epsilon_vec));
185
- float32x4_t log_ratio_a_vec = simsimd_neon_f32_log(ratio_a_vec);
186
- float32x4_t log_ratio_b_vec = simsimd_neon_f32_log(ratio_b_vec);
203
+ float32x4_t log_ratio_a_vec = simsimd_neon_f32_log2(ratio_a_vec);
204
+ float32x4_t log_ratio_b_vec = simsimd_neon_f32_log2(ratio_b_vec);
187
205
  float32x4_t prod_a_vec = vmulq_f32(a_vec, log_ratio_a_vec);
188
206
  float32x4_t prod_b_vec = vmulq_f32(b_vec, log_ratio_b_vec);
189
207
  sum_vec = vaddq_f32(sum_vec, vaddq_f32(prod_a_vec, prod_b_vec));
190
208
  }
191
- simsimd_f32_t sum = vaddvq_f32(sum_vec);
209
+ simsimd_f32_t log2_normalizer = 0.693147181f;
210
+ simsimd_f32_t sum = vaddvq_f32(sum_vec) * log2_normalizer;
192
211
  for (; i < n; ++i) {
193
212
  simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
194
213
  simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
195
214
  simsimd_f32_t mi = ai + bi;
196
- sum += a[i] * SIMSIMD_LOG((a[i] + epsilon) / (mi + epsilon));
197
- sum += b[i] * SIMSIMD_LOG((b[i] + epsilon) / (mi + epsilon));
215
+ sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
216
+ sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
198
217
  }
199
218
  return sum / 2;
200
219
  }
@@ -202,7 +221,7 @@ simsimd_neon_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
202
221
  #endif // SIMSIMD_TARGET_ARM_NEON
203
222
  #endif // SIMSIMD_TARGET_ARM
204
223
 
205
- #if SIMSIMD_TARGET_X86 && 0
224
+ #if SIMSIMD_TARGET_X86
206
225
  #if SIMSIMD_TARGET_X86_AVX2
207
226
 
208
227
  /*
@@ -210,234 +229,112 @@ simsimd_neon_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size
210
229
  * @brief x86 AVX2 implementation of the most common similarity metrics for 16-bit floating point numbers.
211
230
  * @author Ash Vardanian
212
231
  *
213
- * - Implements: L2 squared, inner product, cosine similarity.
232
+ * - Implements: Kullback-Leibler and Jensen–Shannon divergence.
214
233
  * - As AVX2 doesn't support masked loads of 16-bit words, implementations have a separate `for`-loop for tails.
215
234
  * - Uses `f16` for both storage and `f32` for accumulation.
216
235
  * - Requires compiler capabilities: avx2, f16c, fma.
217
236
  */
218
237
 
219
238
  __attribute__((target("avx2,f16c,fma"))) //
220
- inline static simsimd_f32_t
221
- simsimd_avx2_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
222
- __m256 d2_vec = _mm256_set1_ps(0);
223
- simsimd_size_t i = 0;
224
- for (; i + 8 <= n; i += 8) {
225
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
226
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
227
- __m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
228
- d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
229
- }
230
-
231
- d2_vec = _mm256_add_ps(_mm256_permute2f128_ps(d2_vec, d2_vec, 1), d2_vec);
232
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
233
- d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
234
-
235
- simsimd_f32_t result;
236
- _mm_store_ss(&result, _mm256_castps256_ps128(d2_vec));
237
-
238
- // Accumulate the tail:
239
- for (; i < n; ++i) {
240
- simsimd_f32_t n = SIMSIMD_UNCOMPRESS_F16(a[i]) - SIMSIMD_UNCOMPRESS_F16(b[i]);
241
- result += n * n;
242
- }
239
+ inline __m256
240
+ simsimd_avx2_f32_log2(__m256 x) {
241
+ // Extracting the exponent
242
+ __m256i i = _mm256_castps_si256(x);
243
+ __m256i e = _mm256_srli_epi32(_mm256_and_si256(i, _mm256_set1_epi32(0x7F800000)), 23);
244
+ e = _mm256_sub_epi32(e, _mm256_set1_epi32(127)); // removing the bias
245
+ __m256 e_float = _mm256_cvtepi32_ps(e);
246
+
247
+ // Extracting the mantissa
248
+ __m256 m = _mm256_castsi256_ps(
249
+ _mm256_or_si256(_mm256_and_si256(i, _mm256_set1_epi32(0x007FFFFF)), _mm256_set1_epi32(0x3F800000)));
250
+
251
+ // Constants for polynomial
252
+ __m256 one = _mm256_set1_ps(1.0f);
253
+ __m256 p = _mm256_set1_ps(-3.4436006e-2f);
254
+
255
+ // Compute the polynomial using Horner's method
256
+ p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(3.1821337e-1f));
257
+ p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(-1.2315303f));
258
+ p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(2.5988452f));
259
+ p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(-3.3241990f));
260
+ p = _mm256_fmadd_ps(m, p, _mm256_set1_ps(3.1157899f));
261
+
262
+ // Final computation
263
+ __m256 result = _mm256_add_ps(_mm256_mul_ps(p, _mm256_sub_ps(m, one)), e_float);
243
264
  return result;
244
265
  }
245
266
 
246
267
  __attribute__((target("avx2,f16c,fma"))) //
247
268
  inline static simsimd_f32_t
248
- simsimd_avx2_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
249
- __m256 ab_vec = _mm256_set1_ps(0);
269
+ simsimd_avx2_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
270
+ __m256 sum_vec = _mm256_set1_ps(0);
271
+ simsimd_f32_t epsilon = 1e-5;
272
+ __m256 epsilon_vec = _mm256_set1_ps(epsilon);
250
273
  simsimd_size_t i = 0;
251
274
  for (; i + 8 <= n; i += 8) {
252
275
  __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
253
276
  __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
254
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
277
+ __m256 ratio_vec = _mm256_div_ps(_mm256_add_ps(a_vec, epsilon_vec), _mm256_add_ps(b_vec, epsilon_vec));
278
+ __m256 log_ratio_vec = simsimd_avx2_f32_log2(ratio_vec);
279
+ __m256 prod_vec = _mm256_mul_ps(a_vec, log_ratio_vec);
280
+ sum_vec = _mm256_add_ps(sum_vec, prod_vec);
255
281
  }
256
282
 
257
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
258
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
259
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
283
+ sum_vec = _mm256_add_ps(_mm256_permute2f128_ps(sum_vec, sum_vec, 1), sum_vec);
284
+ sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
285
+ sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
260
286
 
261
- simsimd_f32_t result;
262
- _mm_store_ss(&result, _mm256_castps256_ps128(ab_vec));
287
+ simsimd_f32_t log2_normalizer = 0.693147181f;
288
+ simsimd_f32_t sum;
289
+ _mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
290
+ sum *= log2_normalizer;
263
291
 
264
292
  // Accumulate the tail:
265
293
  for (; i < n; ++i)
266
- result += SIMSIMD_UNCOMPRESS_F16(a[i]) * SIMSIMD_UNCOMPRESS_F16(b[i]);
267
- return 1 - result;
294
+ sum += SIMSIMD_UNCOMPRESS_F16(a[i]) *
295
+ SIMSIMD_LOG((SIMSIMD_UNCOMPRESS_F16(a[i]) + epsilon) / (SIMSIMD_UNCOMPRESS_F16(b[i]) + epsilon));
296
+ return sum;
268
297
  }
298
+
269
299
  __attribute__((target("avx2,f16c,fma"))) //
270
300
  inline static simsimd_f32_t
271
- simsimd_avx2_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
272
-
273
- __m256 ab_vec = _mm256_set1_ps(0), a2_vec = _mm256_set1_ps(0), b2_vec = _mm256_set1_ps(0);
301
+ simsimd_avx2_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
302
+ __m256 sum_vec = _mm256_set1_ps(0);
303
+ simsimd_f32_t epsilon = 1e-5;
304
+ __m256 epsilon_vec = _mm256_set1_ps(epsilon);
274
305
  simsimd_size_t i = 0;
275
306
  for (; i + 8 <= n; i += 8) {
276
- __m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
277
- __m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
278
- ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
279
- a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
280
- b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
307
+ __m256 a_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i))), epsilon_vec);
308
+ __m256 b_vec = _mm256_add_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i))), epsilon_vec);
309
+ __m256 m_vec = _mm256_mul_ps(_mm256_add_ps(a_vec, b_vec), _mm256_set1_ps(0.5f)); // M = (P + Q) / 2
310
+ __m256 ratio_a_vec = _mm256_div_ps(a_vec, m_vec);
311
+ __m256 ratio_b_vec = _mm256_div_ps(b_vec, m_vec);
312
+ __m256 log_ratio_a_vec = simsimd_avx2_f32_log2(ratio_a_vec);
313
+ __m256 log_ratio_b_vec = simsimd_avx2_f32_log2(ratio_b_vec);
314
+ __m256 prod_a_vec = _mm256_mul_ps(a_vec, log_ratio_a_vec);
315
+ __m256 prod_b_vec = _mm256_mul_ps(b_vec, log_ratio_b_vec);
316
+ sum_vec = _mm256_add_ps(sum_vec, prod_a_vec);
317
+ sum_vec = _mm256_add_ps(sum_vec, prod_b_vec);
281
318
  }
282
319
 
283
- ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 1), ab_vec);
284
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
285
- ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
286
-
287
- a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
288
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
289
- a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
320
+ sum_vec = _mm256_add_ps(_mm256_permute2f128_ps(sum_vec, sum_vec, 1), sum_vec);
321
+ sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
322
+ sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
290
323
 
291
- b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
292
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
293
- b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
294
-
295
- simsimd_f32_t ab, a2, b2;
296
- _mm_store_ss(&ab, _mm256_castps256_ps128(ab_vec));
297
- _mm_store_ss(&a2, _mm256_castps256_ps128(a2_vec));
298
- _mm_store_ss(&b2, _mm256_castps256_ps128(b2_vec));
324
+ simsimd_f32_t log2_normalizer = 0.693147181f;
325
+ simsimd_f32_t sum;
326
+ _mm_store_ss(&sum, _mm256_castps256_ps128(sum_vec));
327
+ sum *= log2_normalizer;
299
328
 
300
329
  // Accumulate the tail:
301
330
  for (; i < n; ++i) {
302
- simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]), bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
303
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
304
- }
305
-
306
- // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
307
- __m128 a2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)a2));
308
- __m128 b2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)b2));
309
- __m128 result = _mm_mul_ss(a2_sqrt_recjs, b2_sqrt_recjs); // Multjsly the recjsrocal square roots
310
- result = _mm_mul_ss(result, _mm_set_ss((float)ab)); // Multjsly by ab
311
- result = _mm_sub_ss(_mm_set_ss(1.0f), result); // Subtract from 1
312
- return ab != 0 ? _mm_cvtss_f32(result) : 1; // Extract the final result
313
- }
314
-
315
- /*
316
- * @file x86_avx2_i8.h
317
- * @brief x86 AVX2 implementation of the most common similarity metrics for 8-bit signed integral numbers.
318
- * @author Ash Vardanian
319
- *
320
- * - Implements: L2 squared, cosine similarity, inner product (same as cosine).
321
- * - As AVX2 doesn't support masked loads of 16-bit words, implementations have a separate `for`-loop for tails.
322
- * - Uses `i8` for storage, `i16` for multjslication, and `i32` for accumulation, if no better option is available.
323
- * - Requires compiler capabilities: avx2.
324
- */
325
-
326
- __attribute__((target("avx2"))) //
327
- inline static simsimd_f32_t
328
- simsimd_avx2_i8_kl(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
329
-
330
- __m256i d2_high_vec = _mm256_setzero_si256();
331
- __m256i d2_low_vec = _mm256_setzero_si256();
332
-
333
- simsimd_size_t i = 0;
334
- for (; i + 32 <= n; i += 32) {
335
- __m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
336
- __m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
337
-
338
- // Sign extend int8 to int16
339
- __m256i a_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_vec));
340
- __m256i a_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
341
- __m256i b_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_vec));
342
- __m256i b_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
343
-
344
- // Subtract and multjsly
345
- __m256i d_low = _mm256_sub_epi16(a_low, b_low);
346
- __m256i d_high = _mm256_sub_epi16(a_high, b_high);
347
- __m256i d2_low_part = _mm256_madd_epi16(d_low, d_low);
348
- __m256i d2_high_part = _mm256_madd_epi16(d_high, d_high);
349
-
350
- // Accumulate into int32 vectors
351
- d2_low_vec = _mm256_add_epi32(d2_low_vec, d2_low_part);
352
- d2_high_vec = _mm256_add_epi32(d2_high_vec, d2_high_part);
353
- }
354
-
355
- // Accumulate the 32-bit integers from `d2_high_vec` and `d2_low_vec`
356
- __m256i d2_vec = _mm256_add_epi32(d2_low_vec, d2_high_vec);
357
- __m128i d2_sum = _mm_add_epi32(_mm256_extracti128_si256(d2_vec, 0), _mm256_extracti128_si256(d2_vec, 1));
358
- d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
359
- d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
360
- int d2 = _mm_extract_epi32(d2_sum, 0);
361
-
362
- // Take care of the tail:
363
- for (; i < n; ++i) {
364
- int n = a[i] - b[i];
365
- d2 += n * n;
366
- }
367
-
368
- return (simsimd_f32_t)d2;
369
- }
370
- __attribute__((target("avx2"))) //
371
- inline static simsimd_f32_t
372
- simsimd_avx2_i8_cos(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
373
-
374
- __m256i ab_high_vec = _mm256_setzero_si256();
375
- __m256i ab_low_vec = _mm256_setzero_si256();
376
- __m256i a2_high_vec = _mm256_setzero_si256();
377
- __m256i a2_low_vec = _mm256_setzero_si256();
378
- __m256i b2_high_vec = _mm256_setzero_si256();
379
- __m256i b2_low_vec = _mm256_setzero_si256();
380
-
381
- simsimd_size_t i = 0;
382
- for (; i + 32 <= n; i += 32) {
383
- __m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
384
- __m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
385
-
386
- // Unpack int8 to int32
387
- __m256i a_low = _mm256_cvtepi8_epi32(_mm256_castsi256_si128(a_vec));
388
- __m256i a_high = _mm256_cvtepi8_epi32(_mm256_extracti128_si256(a_vec, 1));
389
- __m256i b_low = _mm256_cvtepi8_epi32(_mm256_castsi256_si128(b_vec));
390
- __m256i b_high = _mm256_cvtepi8_epi32(_mm256_extracti128_si256(b_vec, 1));
391
-
392
- // Multjsly and accumulate
393
- ab_low_vec = _mm256_add_epi32(ab_low_vec, _mm256_mullo_epi32(a_low, b_low));
394
- ab_high_vec = _mm256_add_epi32(ab_high_vec, _mm256_mullo_epi32(a_high, b_high));
395
- a2_low_vec = _mm256_add_epi32(a2_low_vec, _mm256_mullo_epi32(a_low, a_low));
396
- a2_high_vec = _mm256_add_epi32(a2_high_vec, _mm256_mullo_epi32(a_high, a_high));
397
- b2_low_vec = _mm256_add_epi32(b2_low_vec, _mm256_mullo_epi32(b_low, b_low));
398
- b2_high_vec = _mm256_add_epi32(b2_high_vec, _mm256_mullo_epi32(b_high, b_high));
399
- }
400
-
401
- // Horizontal sum across the 256-bit register
402
- __m256i ab_vec = _mm256_add_epi32(ab_low_vec, ab_high_vec);
403
- __m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
404
- ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
405
- ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
406
-
407
- __m256i a2_vec = _mm256_add_epi32(a2_low_vec, a2_high_vec);
408
- __m128i a2_sum = _mm_add_epi32(_mm256_extracti128_si256(a2_vec, 0), _mm256_extracti128_si256(a2_vec, 1));
409
- a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
410
- a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
411
-
412
- __m256i b2_vec = _mm256_add_epi32(b2_low_vec, b2_high_vec);
413
- __m128i b2_sum = _mm_add_epi32(_mm256_extracti128_si256(b2_vec, 0), _mm256_extracti128_si256(b2_vec, 1));
414
- b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
415
- b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
416
-
417
- // Further reduce to a single sum for each vector
418
- int ab = _mm_extract_epi32(ab_sum, 0);
419
- int a2 = _mm_extract_epi32(a2_sum, 0);
420
- int b2 = _mm_extract_epi32(b2_sum, 0);
421
-
422
- // Take care of the tail:
423
- for (; i < n; ++i) {
424
- int ai = a[i], bi = b[i];
425
- ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
331
+ simsimd_f32_t ai = SIMSIMD_UNCOMPRESS_F16(a[i]);
332
+ simsimd_f32_t bi = SIMSIMD_UNCOMPRESS_F16(b[i]);
333
+ simsimd_f32_t mi = ai + bi;
334
+ sum += ai * SIMSIMD_LOG((ai + epsilon) / (mi + epsilon));
335
+ sum += bi * SIMSIMD_LOG((bi + epsilon) / (mi + epsilon));
426
336
  }
427
-
428
- // Replace simsimd_approximate_inverse_square_root with `rsqrtss`
429
- __m128 a2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)a2));
430
- __m128 b2_sqrt_recjs = _mm_rsqrt_ss(_mm_set_ss((float)b2));
431
- __m128 result = _mm_mul_ss(a2_sqrt_recjs, b2_sqrt_recjs); // Multjsly the recjsrocal square roots
432
- result = _mm_mul_ss(result, _mm_set_ss((float)ab)); // Multjsly by ab
433
- result = _mm_sub_ss(_mm_set_ss(1.0f), result); // Subtract from 1
434
- return ab != 0 ? _mm_cvtss_f32(result) : 1; // Extract the final result
435
- }
436
-
437
- __attribute__((target("avx2"))) //
438
- inline static simsimd_f32_t
439
- simsimd_avx2_i8_js(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
440
- return simsimd_avx2_i8_cos(a, b, n);
337
+ return sum / 2;
441
338
  }
442
339
 
443
340
  #endif // SIMSIMD_TARGET_X86_AVX2
@@ -449,63 +346,84 @@ simsimd_avx2_i8_js(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t
449
346
  * @brief x86 AVX-512 implementation of the most common similarity metrics for 32-bit floating point numbers.
450
347
  * @author Ash Vardanian
451
348
  *
452
- * - Implements: L2 squared, inner product, cosine similarity.
349
+ * - Implements: Kullback-Leibler and Jensen–Shannon divergence.
453
350
  * - Uses `f32` for storage and `f32` for accumulation.
454
351
  * - Requires compiler capabilities: avx512f, avx512vl.
455
352
  */
456
353
 
457
354
  __attribute__((target("avx512f,avx512vl"))) //
458
- inline static simsimd_f32_t
459
- simsimd_avx512_f32_kl(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
460
- __m512 d2_vec = _mm512_set1_ps(0);
461
- for (simsimd_size_t i = 0; i < n; i += 16) {
462
- __mmask16 mask = n - i >= 16 ? 0xFFFF : ((1u << (n - i)) - 1u);
463
- __m512 a_vec = _mm512_maskz_loadu_ps(mask, a + i);
464
- __m512 b_vec = _mm512_maskz_loadu_ps(mask, b + i);
465
- __m512 d_vec = _mm512_sub_ps(a_vec, b_vec);
466
- d2_vec = _mm512_fmadd_ps(d_vec, d_vec, d2_vec);
467
- }
468
- return _mm512_reduce_add_ps(d2_vec);
355
+ inline __m512
356
+ simsimd_avx512_f32_log2(__m512 x) {
357
+ // Extract the exponent and mantissa
358
+ __m512 one = _mm512_set1_ps(1.0f);
359
+ __m512 e = _mm512_getexp_ps(x);
360
+ __m512 m = _mm512_getmant_ps(x, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_src);
361
+
362
+ // Compute the polynomial using Horner's method
363
+ __m512 p = _mm512_set1_ps(-3.4436006e-2f);
364
+ p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(3.1821337e-1f));
365
+ p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(-1.2315303f));
366
+ p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(2.5988452f));
367
+ p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(-3.3241990f));
368
+ p = _mm512_fmadd_ps(m, p, _mm512_set1_ps(3.1157899f));
369
+
370
+ return _mm512_add_ps(_mm512_mul_ps(p, _mm512_sub_ps(m, one)), e);
469
371
  }
470
372
 
471
373
  __attribute__((target("avx512f,avx512vl"))) //
472
374
  inline static simsimd_f32_t
473
- simsimd_avx512_f32_js(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
474
- __m512 ab_vec = _mm512_set1_ps(0);
375
+ simsimd_avx512_f32_kl(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
376
+ __m512 sum_vec = _mm512_set1_ps(0);
377
+ simsimd_f32_t epsilon = 1e-6;
378
+ __m512 epsilon_vec = _mm512_set1_ps(epsilon);
475
379
  for (simsimd_size_t i = 0; i < n; i += 16) {
476
380
  __mmask16 mask = n - i >= 16 ? 0xFFFF : ((1u << (n - i)) - 1u);
477
- __m512 a_vec = _mm512_maskz_loadu_ps(mask, a + i);
478
- __m512 b_vec = _mm512_maskz_loadu_ps(mask, b + i);
479
- ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
381
+ __m512 a_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, a + i), epsilon_vec);
382
+ __m512 b_vec = _mm512_add_ps(_mm512_maskz_loadu_ps(mask, b + i), epsilon_vec);
383
+ __m512 ratio_vec = _mm512_div_ps(a_vec, b_vec);
384
+ __m512 log_ratio_vec = simsimd_avx512_f32_log2(ratio_vec);
385
+ __m512 prod_vec = _mm512_mul_ps(a_vec, log_ratio_vec);
386
+ sum_vec = _mm512_maskz_add_ps(mask, sum_vec, prod_vec);
480
387
  }
481
- return 1 - _mm512_reduce_add_ps(ab_vec);
388
+ simsimd_f32_t log2_normalizer = 0.693147181f;
389
+ return _mm512_reduce_add_ps(sum_vec) * log2_normalizer;
482
390
  }
483
391
 
484
392
  __attribute__((target("avx512f,avx512vl"))) //
485
393
  inline static simsimd_f32_t
486
- simsimd_avx512_f32_cos(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
487
- __m512 ab_vec = _mm512_set1_ps(0);
488
- __m512 a2_vec = _mm512_set1_ps(0);
489
- __m512 b2_vec = _mm512_set1_ps(0);
490
-
394
+ simsimd_avx512_f32_js(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n) {
395
+ __m512 sum_a_vec = _mm512_set1_ps(0);
396
+ __m512 sum_b_vec = _mm512_set1_ps(0);
397
+ simsimd_f32_t epsilon = 1e-6;
398
+ __m512 epsilon_vec = _mm512_set1_ps(epsilon);
491
399
  for (simsimd_size_t i = 0; i < n; i += 16) {
492
400
  __mmask16 mask = n - i >= 16 ? 0xFFFF : ((1u << (n - i)) - 1u);
493
401
  __m512 a_vec = _mm512_maskz_loadu_ps(mask, a + i);
494
402
  __m512 b_vec = _mm512_maskz_loadu_ps(mask, b + i);
495
- ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
496
- a2_vec = _mm512_fmadd_ps(a_vec, a_vec, a2_vec);
497
- b2_vec = _mm512_fmadd_ps(b_vec, b_vec, b2_vec);
403
+ __m512 m_vec = _mm512_mul_ps(_mm512_add_ps(a_vec, b_vec), _mm512_set1_ps(0.5f)); // M = (P + Q) / 2
404
+
405
+ // Avoid division by zero problems from probabilities under zero down the road.
406
+ // Masking is a nicer way to do this, than adding the `epsilon` to every component.
407
+ __mmask16 nonzero_mask_a = _mm512_cmp_ps_mask(a_vec, epsilon_vec, _CMP_GE_OQ);
408
+ __mmask16 nonzero_mask_b = _mm512_cmp_ps_mask(b_vec, epsilon_vec, _CMP_GE_OQ);
409
+ __mmask16 nonzero_mask = nonzero_mask_a & nonzero_mask_b & mask;
410
+
411
+ // Division is an expensive operation. Instead of doing it twice,
412
+ // we can approximate the reciprocal of `m` and multiply instead.
413
+ __m512 m_recip_approx = _mm512_rcp14_ps(m_vec); // or `_mm512_rcp28_ps` for slightly better precision
414
+ __m512 ratio_a_vec = _mm512_mul_ps(a_vec, m_recip_approx);
415
+ __m512 ratio_b_vec = _mm512_mul_ps(b_vec, m_recip_approx);
416
+
417
+ // The natural logarithm is equivalent to `log2`, multiplied by the `loge(2)`
418
+ __m512 log_ratio_a_vec = simsimd_avx512_f32_log2(ratio_a_vec);
419
+ __m512 log_ratio_b_vec = simsimd_avx512_f32_log2(ratio_b_vec);
420
+
421
+ // Instead of separate multiplication and addition, invoke the FMA
422
+ sum_a_vec = _mm512_maskz_fmadd_ps(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
423
+ sum_b_vec = _mm512_maskz_fmadd_ps(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
498
424
  }
499
-
500
- simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
501
- simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
502
- simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
503
-
504
- __m128d a2_b2 = _mm_set_pd((double)a2, (double)b2);
505
- __m128d rsqrts = _mm_mask_rsqrt14_pd(_mm_setzero_pd(), 0xFF, a2_b2);
506
- double rsqrts_array[2];
507
- _mm_storeu_pd(rsqrts_array, rsqrts);
508
- return ab != 0 ? 1 - ab * rsqrts_array[0] * rsqrts_array[1] : 1;
425
+ simsimd_f32_t log2_normalizer = 0.693147181f;
426
+ return _mm512_reduce_add_ps(_mm512_add_ps(sum_a_vec, sum_b_vec)) * 0.5f * log2_normalizer;
509
427
  }
510
428
 
511
429
  /*
@@ -513,128 +431,83 @@ simsimd_avx512_f32_cos(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_s
513
431
  * @brief x86 AVX-512 implementation of the most common similarity metrics for 16-bit floating point numbers.
514
432
  * @author Ash Vardanian
515
433
  *
516
- * - Implements: L2 squared, inner product, cosine similarity.
434
+ * - Implements: Kullback-Leibler and Jensen–Shannon divergence.
517
435
  * - Uses `_mm512_maskz_loadu_epi16` intrinsics to perform masked unaligned loads.
518
436
  * - Uses `f16` for both storage and accumulation, assuming it's resolution is enough for average case.
519
437
  * - Requires compiler capabilities: avx512fp16, avx512f, avx512vl.
520
438
  */
521
439
 
522
- __attribute__((target("avx512fp16,avx512vl,avx512f"))) //
523
- inline static simsimd_f32_t
524
- simsimd_avx512_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
525
- __m512h d2_vec = _mm512_set1_ph(0);
526
- for (simsimd_size_t i = 0; i < n; i += 32) {
527
- __mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
528
- __m512i a_vec = _mm512_maskz_loadu_epi16(mask, a + i);
529
- __m512i b_vec = _mm512_maskz_loadu_epi16(mask, b + i);
530
- __m512h d_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec));
531
- d2_vec = _mm512_fmadd_ph(d_vec, d_vec, d2_vec);
532
- }
533
- return _mm512_reduce_add_ph(d2_vec);
440
+ __attribute__((target("avx512f,avx512vl,avx512fp16"))) //
441
+ inline __m512h
442
+ simsimd_avx512_f16_log2(__m512h x) {
443
+ // Extract the exponent and mantissa
444
+ __m512h one = _mm512_set1_ph((_Float16)1);
445
+ __m512h e = _mm512_getexp_ph(x);
446
+ __m512h m = _mm512_getmant_ph(x, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_src);
447
+
448
+ // Compute the polynomial using Horner's method
449
+ __m512h p = _mm512_set1_ph((_Float16)-3.4436006e-2f);
450
+ p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)3.1821337e-1f));
451
+ p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)-1.2315303f));
452
+ p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)2.5988452f));
453
+ p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)-3.3241990f));
454
+ p = _mm512_fmadd_ph(m, p, _mm512_set1_ph((_Float16)3.1157899f));
455
+
456
+ return _mm512_add_ph(_mm512_mul_ph(p, _mm512_sub_ph(m, one)), e);
534
457
  }
535
458
 
536
- __attribute__((target("avx512fp16,avx512vl,avx512f"))) //
459
+ __attribute__((target("avx512f,avx512vl,avx512fp16"))) //
537
460
  inline static simsimd_f32_t
538
- simsimd_avx512_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
539
- __m512h ab_vec = _mm512_set1_ph(0);
540
- simsimd_size_t i = 0;
541
-
542
- for (simsimd_size_t i = 0; i < n; i += 32) {
543
- __mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
544
- __m512i a_vec = _mm512_maskz_loadu_epi16(mask, a + i);
545
- __m512i b_vec = _mm512_maskz_loadu_epi16(mask, b + i);
546
- ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_vec);
547
- }
548
- return 1 - _mm512_reduce_add_ph(ab_vec);
549
- }
550
-
551
- __attribute__((target("avx512fp16,avx512vl,avx512f"))) //
552
- inline static simsimd_f32_t
553
- simsimd_avx512_f16_cos(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
554
- __m512h ab_vec = _mm512_set1_ph(0);
555
- __m512h a2_vec = _mm512_set1_ph(0);
556
- __m512h b2_vec = _mm512_set1_ph(0);
557
-
558
- for (simsimd_size_t i = 0; i < n; i += 32) {
559
- __mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
560
- __m512i a_vec = _mm512_maskz_loadu_epi16(mask, a + i);
561
- __m512i b_vec = _mm512_maskz_loadu_epi16(mask, b + i);
562
- ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(b_vec), ab_vec);
563
- a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_vec), _mm512_castsi512_ph(a_vec), a2_vec);
564
- b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_vec), _mm512_castsi512_ph(b_vec), b2_vec);
565
- }
566
-
567
- simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
568
- simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
569
- simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
570
-
571
- __m128d a2_b2 = _mm_set_pd((double)a2, (double)b2);
572
- __m128d rsqrts = _mm_mask_rsqrt14_pd(_mm_setzero_pd(), 0xFF, a2_b2);
573
- double rsqrts_array[2];
574
- _mm_storeu_pd(rsqrts_array, rsqrts);
575
- return ab != 0 ? 1 - ab * rsqrts_array[0] * rsqrts_array[1] : 1;
576
- }
577
-
578
- /*
579
- * @file x86_avx512_i8.h
580
- * @brief x86 AVX-512 implementation of the most common similarity metrics for 8-bit integers.
581
- * @author Ash Vardanian
582
- *
583
- * - Implements: L2 squared, cosine similarity, inner product (same as cosine).
584
- * - Uses `_mm512_maskz_loadu_epi16` intrinsics to perform masked unaligned loads.
585
- * - Uses `i8` for storage, `i16` for multjslication, and `i32` for accumulation, if no better option is available.
586
- * - Requires compiler capabilities: avx512f, avx512vl, avx512bw.
587
- */
588
-
589
- __attribute__((target("avx512vl,avx512f,avx512bw"))) //
590
- inline static simsimd_f32_t
591
- simsimd_avx512_i8_kl(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
592
- __m512i d2_i32s_vec = _mm512_setzero_si512();
593
-
461
+ simsimd_avx512_f16_kl(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
462
+ __m512h sum_vec = _mm512_set1_ph((_Float16)0);
463
+ __m512h epsilon_vec = _mm512_set1_ph((_Float16)1e-6f);
594
464
  for (simsimd_size_t i = 0; i < n; i += 32) {
595
465
  __mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
596
- __m512i a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a + i)); // Load 8-bit integers
597
- __m512i b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b + i)); // Load 8-bit integers
598
- __m512i d_i16s_vec = _mm512_sub_epi16(a_vec, b_vec);
599
- d2_i32s_vec = _mm512_add_epi32(d2_i32s_vec, _mm512_madd_epi16(d_i16s_vec, d_i16s_vec));
466
+ __m512h a_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a + i)), epsilon_vec);
467
+ __m512h b_vec = _mm512_add_ph(_mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b + i)), epsilon_vec);
468
+ __m512h ratio_vec = _mm512_div_ph(a_vec, b_vec);
469
+ __m512h log_ratio_vec = simsimd_avx512_f16_log2(ratio_vec);
470
+ __m512h prod_vec = _mm512_mul_ph(a_vec, log_ratio_vec);
471
+ sum_vec = _mm512_maskz_add_ph(mask, sum_vec, prod_vec);
600
472
  }
601
-
602
- return _mm512_reduce_add_epi32(d2_i32s_vec);
473
+ simsimd_f32_t log2_normalizer = 0.693147181f;
474
+ return _mm512_reduce_add_ph(sum_vec) * log2_normalizer;
603
475
  }
604
476
 
605
- __attribute__((target("avx512vl,avx512f,avx512bw"))) //
477
+ __attribute__((target("avx512f,avx512vl,avx512fp16"))) //
606
478
  inline static simsimd_f32_t
607
- simsimd_avx512_i8_cos(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
608
- __m512i ab_i32s_vec = _mm512_setzero_si512();
609
- __m512i a2_i32s_vec = _mm512_setzero_si512();
610
- __m512i b2_i32s_vec = _mm512_setzero_si512();
611
- simsimd_size_t i = 0;
612
-
479
+ simsimd_avx512_f16_js(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n) {
480
+ __m512h sum_a_vec = _mm512_set1_ph((_Float16)0);
481
+ __m512h sum_b_vec = _mm512_set1_ph((_Float16)0);
482
+ __m512h epsilon_vec = _mm512_set1_ph((_Float16)1e-6f);
613
483
  for (simsimd_size_t i = 0; i < n; i += 32) {
614
484
  __mmask32 mask = n - i >= 32 ? 0xFFFFFFFF : ((1u << (n - i)) - 1u);
615
- __m512i a_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a + i)); // Load 8-bit integers
616
- __m512i b_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b + i)); // Load 8-bit integers
617
-
618
- ab_i32s_vec = _mm512_add_epi32(ab_i32s_vec, _mm512_madd_epi16(a_vec, b_vec));
619
- a2_i32s_vec = _mm512_add_epi32(a2_i32s_vec, _mm512_madd_epi16(a_vec, a_vec));
620
- b2_i32s_vec = _mm512_add_epi32(b2_i32s_vec, _mm512_madd_epi16(b_vec, b_vec));
485
+ __m512h a_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a + i));
486
+ __m512h b_vec = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b + i));
487
+ __m512h m_vec = _mm512_mul_ph(_mm512_add_ph(a_vec, b_vec), _mm512_set1_ph((_Float16)0.5f)); // M = (P + Q) / 2
488
+
489
+ // Avoid division by zero problems from probabilities under zero down the road.
490
+ // Masking is a nicer way to do this, than adding the `epsilon` to every component.
491
+ __mmask32 nonzero_mask_a = _mm512_cmp_ph_mask(a_vec, epsilon_vec, _CMP_GE_OQ);
492
+ __mmask32 nonzero_mask_b = _mm512_cmp_ph_mask(b_vec, epsilon_vec, _CMP_GE_OQ);
493
+ __mmask32 nonzero_mask = nonzero_mask_a & nonzero_mask_b & mask;
494
+
495
+ // Division is an expensive operation. Instead of doing it twice,
496
+ // we can approximate the reciprocal of `m` and multiply instead.
497
+ __m512h m_recip_approx = _mm512_rcp_ph(m_vec);
498
+ __m512h ratio_a_vec = _mm512_mul_ph(a_vec, m_recip_approx);
499
+ __m512h ratio_b_vec = _mm512_mul_ph(b_vec, m_recip_approx);
500
+
501
+ // The natural logarithm is equivalent to `log2`, multiplied by the `loge(2)`
502
+ __m512h log_ratio_a_vec = simsimd_avx512_f16_log2(ratio_a_vec);
503
+ __m512h log_ratio_b_vec = simsimd_avx512_f16_log2(ratio_b_vec);
504
+
505
+ // Instead of separate multiplication and addition, invoke the FMA
506
+ sum_a_vec = _mm512_maskz_fmadd_ph(nonzero_mask, a_vec, log_ratio_a_vec, sum_a_vec);
507
+ sum_b_vec = _mm512_maskz_fmadd_ph(nonzero_mask, b_vec, log_ratio_b_vec, sum_b_vec);
621
508
  }
622
-
623
- int ab = _mm512_reduce_add_epi32(ab_i32s_vec);
624
- int a2 = _mm512_reduce_add_epi32(a2_i32s_vec);
625
- int b2 = _mm512_reduce_add_epi32(b2_i32s_vec);
626
-
627
- __m128d a2_b2 = _mm_set_pd((double)a2, (double)b2);
628
- __m128d rsqrts = _mm_mask_rsqrt14_pd(_mm_setzero_pd(), 0xFF, a2_b2);
629
- double rsqrts_array[2];
630
- _mm_storeu_pd(rsqrts_array, rsqrts);
631
- return ab != 0 ? 1 - ab * rsqrts_array[0] * rsqrts_array[1] : 1;
632
- }
633
-
634
- __attribute__((target("avx512vl,avx512f,avx512bw"))) //
635
- inline static simsimd_f32_t
636
- simsimd_avx512_i8_js(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n) {
637
- return simsimd_avx512_i8_cos(a, b, n);
509
+ simsimd_f32_t log2_normalizer = 0.693147181f;
510
+ return _mm512_reduce_add_ph(_mm512_add_ph(sum_a_vec, sum_b_vec)) * 0.5f * log2_normalizer;
638
511
  }
639
512
 
640
513
  #endif // SIMSIMD_TARGET_X86_AVX512