usearch 2.2.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +12 -3
- package/Cargo.toml +1 -1
- package/conanfile.py +1 -1
- package/include/usearch/index_plugins.hpp +59 -13
- package/package.json +1 -1
- package/simsimd/include/simsimd/simsimd.h +106 -40
- package/wasmer.toml +1 -1
|
@@ -148,7 +148,10 @@ jobs:
|
|
|
148
148
|
- uses: actions/setup-node@v3
|
|
149
149
|
with:
|
|
150
150
|
node-version: 18
|
|
151
|
-
- run:
|
|
151
|
+
- run: |
|
|
152
|
+
export CC=/usr/bin/gcc-12
|
|
153
|
+
export CXX=/usr/bin/g++-12
|
|
154
|
+
npm install
|
|
152
155
|
- run: npm ci
|
|
153
156
|
- run: npm test
|
|
154
157
|
|
|
@@ -176,7 +179,10 @@ jobs:
|
|
|
176
179
|
- name: Setup Gradle
|
|
177
180
|
uses: gradle/gradle-build-action@v2.4.2
|
|
178
181
|
- name: Execute Gradle build
|
|
179
|
-
run:
|
|
182
|
+
run:
|
|
183
|
+
export CC=/usr/bin/gcc-12
|
|
184
|
+
export CXX=/usr/bin/g++-12
|
|
185
|
+
gradle clean build
|
|
180
186
|
|
|
181
187
|
test_swift:
|
|
182
188
|
name: Test ObjC & Swift
|
|
@@ -251,7 +257,10 @@ jobs:
|
|
|
251
257
|
|
|
252
258
|
- name: Build library
|
|
253
259
|
run: |
|
|
254
|
-
cd golang
|
|
260
|
+
cd golang/
|
|
261
|
+
export CC=/usr/bin/gcc-12
|
|
262
|
+
export CXX=/usr/bin/g++-12
|
|
263
|
+
make -C ../c libusearch_c.so
|
|
255
264
|
sudo mv ../c/libusearch_c.so /usr/local/lib/libusearch.a
|
|
256
265
|
sudo mv ../c/usearch.h /usr/local/include/usearch.h
|
|
257
266
|
|
package/Cargo.toml
CHANGED
package/conanfile.py
CHANGED
|
@@ -4,7 +4,7 @@ from conans import ConanFile
|
|
|
4
4
|
class USearchConan(ConanFile):
|
|
5
5
|
|
|
6
6
|
name = 'USearch'
|
|
7
|
-
version = '2.2.
|
|
7
|
+
version = '2.2.1'
|
|
8
8
|
license = 'Apache 2.0'
|
|
9
9
|
url = 'https://github.com/unum-cloud/usearch'
|
|
10
10
|
description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
|
|
@@ -217,19 +217,47 @@ inline bool hardware_supports(isa_kind_t isa_kind) noexcept {
|
|
|
217
217
|
#endif
|
|
218
218
|
|
|
219
219
|
// When compiling with GCC, one may use the "built-ins", including ones
|
|
220
|
-
// designed for CPU capability detection.
|
|
221
|
-
|
|
222
|
-
|
|
220
|
+
// designed for CPU capability detection, but they are not fully compatible with Clang.
|
|
221
|
+
// So instead I've decided to add a pinch of inline assembly.
|
|
222
|
+
#if defined(USEARCH_DEFINED_X86)
|
|
223
|
+
auto check_cpu_feature = [](std::uint64_t feature_mask, int function_id, int register_id) -> bool {
|
|
224
|
+
std::uint32_t eax, ebx, ecx, edx;
|
|
225
|
+
// Execute CPUID instruction and store results in eax, ebx, ecx, edx
|
|
226
|
+
// Setting %ecx to 0 as well
|
|
227
|
+
__asm__ volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(function_id), "c"(0));
|
|
228
|
+
std::uint32_t register_value;
|
|
229
|
+
switch (register_id) {
|
|
230
|
+
case 0: register_value = eax; break;
|
|
231
|
+
case 1: register_value = ebx; break;
|
|
232
|
+
case 2: register_value = ecx; break;
|
|
233
|
+
case 3: register_value = edx; break;
|
|
234
|
+
default: return false;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return (register_value & feature_mask) != 0;
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
// Check for AVX2 (Function ID 7, EBX register)
|
|
241
|
+
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L148
|
|
242
|
+
bool supports_avx2 = check_cpu_feature(1 << 5, 7, 1);
|
|
243
|
+
|
|
244
|
+
// Check for F16C (Function ID 1, ECX register)
|
|
245
|
+
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L107
|
|
246
|
+
bool supports_f16c = check_cpu_feature(1 << 29, 1, 2);
|
|
247
|
+
|
|
248
|
+
// Check for AVX512F (Function ID 7, EBX register)
|
|
249
|
+
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L155
|
|
250
|
+
bool supports_avx512f = check_cpu_feature(1 << 16, 7, 1);
|
|
251
|
+
|
|
252
|
+
// Check for AVX512FP16 (Function ID 7, EDX register)
|
|
253
|
+
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L198C9-L198C23
|
|
254
|
+
bool supports_avx512fp16 = check_cpu_feature(1 << 23, 7, 3);
|
|
255
|
+
|
|
223
256
|
switch (isa_kind) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
case isa_kind_t::
|
|
227
|
-
|
|
228
|
-
#if __GNUC__ > 12 || (__GNUC__ == 12 && __GNUC_MINOR__ >= 0)
|
|
229
|
-
case isa_kind_t::avx512f16_k: return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("avx512fp16");
|
|
230
|
-
#endif
|
|
231
|
-
case isa_kind_t::avx2_k: return __builtin_cpu_supports("avx2");
|
|
232
|
-
case isa_kind_t::avx512_k: return __builtin_cpu_supports("avx512f");
|
|
257
|
+
case isa_kind_t::avx2f16_k: return supports_avx2 && supports_f16c;
|
|
258
|
+
case isa_kind_t::avx512f16_k: return supports_avx512fp16 && supports_avx512f;
|
|
259
|
+
case isa_kind_t::avx2_k: return supports_avx2;
|
|
260
|
+
case isa_kind_t::avx512_k: return supports_avx512f;
|
|
233
261
|
default: return false;
|
|
234
262
|
}
|
|
235
263
|
#endif
|
|
@@ -1426,6 +1454,21 @@ class metric_punned_t {
|
|
|
1426
1454
|
return {to_stl_<metric_cos_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1427
1455
|
}
|
|
1428
1456
|
|
|
1457
|
+
static metric_punned_t ip_metric_f16_(std::size_t bytes_per_vector) {
|
|
1458
|
+
#if USEARCH_USE_SIMSIMD
|
|
1459
|
+
#if SIMSIMD_TARGET_X86_AVX512
|
|
1460
|
+
if (hardware_supports(isa_kind_t::avx512f16_k) && bytes_per_vector % 32 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_dot_f16x16_avx512(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f16_k, isa_kind_t::avx512f16_k};
|
|
1461
|
+
#endif
|
|
1462
|
+
#if SIMSIMD_TARGET_X86_AVX2
|
|
1463
|
+
if (hardware_supports(isa_kind_t::avx2f16_k) && bytes_per_vector % 16 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_dot_f16x8_avx2(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f16_k, isa_kind_t::avx2f16_k};
|
|
1464
|
+
#endif
|
|
1465
|
+
#if SIMSIMD_TARGET_ARM_NEON
|
|
1466
|
+
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 8 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_dot_f16x4_neon(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f16_k, isa_kind_t::neon_k};
|
|
1467
|
+
#endif
|
|
1468
|
+
#endif
|
|
1469
|
+
return {to_stl_<metric_cos_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1429
1472
|
static metric_punned_t l2sq_metric_f16_(std::size_t bytes_per_vector) {
|
|
1430
1473
|
#if USEARCH_USE_SIMSIMD
|
|
1431
1474
|
#if SIMSIMD_TARGET_X86_AVX512
|
|
@@ -1446,6 +1489,9 @@ class metric_punned_t {
|
|
|
1446
1489
|
#if SIMSIMD_TARGET_ARM_NEON
|
|
1447
1490
|
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 16 == 0) return {pun_stl_<int8_t>([=](int8_t const* a, int8_t const* b) { return simsimd_cos_i8x16_neon(a, b, bytes_per_vector); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::neon_k};
|
|
1448
1491
|
#endif
|
|
1492
|
+
#if SIMSIMD_TARGET_X86_AVX2
|
|
1493
|
+
if (hardware_supports(isa_kind_t::avx2_k) && bytes_per_vector % 32 == 0) return {pun_stl_<int8_t>([=](int8_t const* a, int8_t const* b) { return simsimd_cos_i8x32_avx2(a, b, bytes_per_vector); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::avx2_k};
|
|
1494
|
+
#endif
|
|
1449
1495
|
#endif
|
|
1450
1496
|
return {to_stl_<cos_i8_t>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1451
1497
|
}
|
|
@@ -1453,7 +1499,7 @@ class metric_punned_t {
|
|
|
1453
1499
|
static metric_punned_t ip_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
|
|
1454
1500
|
switch (scalar_kind) { // The two most common numeric types for the most common metric have optimized versions
|
|
1455
1501
|
case scalar_kind_t::f32_k: return ip_metric_f32_(bytes_per_vector);
|
|
1456
|
-
case scalar_kind_t::f16_k: return
|
|
1502
|
+
case scalar_kind_t::f16_k: return ip_metric_f16_(bytes_per_vector);
|
|
1457
1503
|
case scalar_kind_t::i8_k: return cos_metric_i8_(bytes_per_vector);
|
|
1458
1504
|
case scalar_kind_t::f64_k: return {to_stl_<metric_ip_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1459
1505
|
default: return {};
|
package/package.json
CHANGED
|
@@ -45,6 +45,9 @@
|
|
|
45
45
|
#endif
|
|
46
46
|
#endif
|
|
47
47
|
|
|
48
|
+
#undef SIMSIMD_TARGET_X86
|
|
49
|
+
#undef SIMSIMD_TARGET_X86_AVX2
|
|
50
|
+
#undef SIMSIMD_TARGET_X86_AVX512
|
|
48
51
|
#define SIMSIMD_TARGET_X86 0
|
|
49
52
|
#define SIMSIMD_TARGET_X86_AVX2 0
|
|
50
53
|
#define SIMSIMD_TARGET_X86_AVX512 0
|
|
@@ -71,6 +74,9 @@
|
|
|
71
74
|
#endif
|
|
72
75
|
#endif
|
|
73
76
|
|
|
77
|
+
#undef SIMSIMD_TARGET_ARM
|
|
78
|
+
#undef SIMSIMD_TARGET_ARM_NEON
|
|
79
|
+
#undef SIMSIMD_TARGET_ARM_SVE
|
|
74
80
|
#define SIMSIMD_TARGET_ARM 0
|
|
75
81
|
#define SIMSIMD_TARGET_ARM_NEON 0
|
|
76
82
|
#define SIMSIMD_TARGET_ARM_SVE 0
|
|
@@ -143,7 +149,7 @@ simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d) {
|
|
|
143
149
|
simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
|
|
144
150
|
simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
|
|
145
151
|
simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
|
|
146
|
-
return 1 - ab / (
|
|
152
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
147
153
|
}
|
|
148
154
|
|
|
149
155
|
__attribute__((target("+sve"))) inline static simsimd_f32_t //
|
|
@@ -184,7 +190,7 @@ simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16_t const* b_enum, si
|
|
|
184
190
|
simsimd_f16_t ab = svaddv_f16(svptrue_b16(), ab_vec);
|
|
185
191
|
simsimd_f16_t a2 = svaddv_f16(svptrue_b16(), a2_vec);
|
|
186
192
|
simsimd_f16_t b2 = svaddv_f16(svptrue_b16(), b2_vec);
|
|
187
|
-
return 1 - ab / (
|
|
193
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
188
194
|
}
|
|
189
195
|
|
|
190
196
|
__attribute__((target("+sve+fp16"))) inline static simsimd_f32_t //
|
|
@@ -235,6 +241,18 @@ simsimd_dot_f32x4_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d)
|
|
|
235
241
|
return 1 - vaddvq_f32(ab_vec);
|
|
236
242
|
}
|
|
237
243
|
|
|
244
|
+
__attribute__((target("+simd+fp16"))) inline static simsimd_f32_t //
|
|
245
|
+
simsimd_dot_f16x4_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
246
|
+
|
|
247
|
+
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
248
|
+
for (size_t i = 0; i != d; i += 4) {
|
|
249
|
+
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((float16_t const*)a + i));
|
|
250
|
+
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((float16_t const*)b + i));
|
|
251
|
+
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
252
|
+
}
|
|
253
|
+
return 1 - vaddvq_f32(ab_vec);
|
|
254
|
+
}
|
|
255
|
+
|
|
238
256
|
__attribute__((target("+simd+fp16"))) inline static simsimd_f32_t //
|
|
239
257
|
simsimd_cos_f16x4_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
240
258
|
|
|
@@ -252,7 +270,7 @@ simsimd_cos_f16x4_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d)
|
|
|
252
270
|
simsimd_f32_t ab = vaddvq_f32(ab_vec);
|
|
253
271
|
simsimd_f32_t a2 = vaddvq_f32(a2_vec);
|
|
254
272
|
simsimd_f32_t b2 = vaddvq_f32(b2_vec);
|
|
255
|
-
return 1 - ab / (
|
|
273
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
256
274
|
}
|
|
257
275
|
|
|
258
276
|
__attribute__((target("+simd"))) inline static simsimd_f32_t //
|
|
@@ -290,7 +308,7 @@ simsimd_cos_i8x16_neon(int8_t const* a, int8_t const* b, size_t d) {
|
|
|
290
308
|
int32_t a2 = vget_lane_s32(vpadd_s32(a2_part, a2_part), 0);
|
|
291
309
|
int32x2_t b2_part = vadd_s32(vget_high_s32(b2_vec), vget_low_s32(b2_vec));
|
|
292
310
|
int32_t b2 = vget_lane_s32(vpadd_s32(b2_part, b2_part), 0);
|
|
293
|
-
return 1 - ab / (
|
|
311
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
294
312
|
}
|
|
295
313
|
|
|
296
314
|
__attribute__((target("+simd"))) inline static simsimd_f32_t //
|
|
@@ -327,7 +345,7 @@ simsimd_cos_f32x4_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d)
|
|
|
327
345
|
simsimd_f32_t ab = vaddvq_f32(ab_vec);
|
|
328
346
|
simsimd_f32_t a2 = vaddvq_f32(a2_vec);
|
|
329
347
|
simsimd_f32_t b2 = vaddvq_f32(b2_vec);
|
|
330
|
-
return 1 - ab / (
|
|
348
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
331
349
|
}
|
|
332
350
|
|
|
333
351
|
__attribute__((target("+sve"))) inline static simsimd_f32_t //
|
|
@@ -385,7 +403,7 @@ simsimd_cos_f32x4_avx2(simsimd_f32_t const* a, simsimd_f32_t const* b, size_t d)
|
|
|
385
403
|
union simsimd_f32i32_t ab_union = {_mm_cvtsi128_si32(_mm_castps_si128(ab_vec))};
|
|
386
404
|
union simsimd_f32i32_t a2_union = {_mm_cvtsi128_si32(_mm_castps_si128(a2_vec))};
|
|
387
405
|
union simsimd_f32i32_t b2_union = {_mm_cvtsi128_si32(_mm_castps_si128(b2_vec))};
|
|
388
|
-
return 1 - ab_union.f / (
|
|
406
|
+
return 1 - ab_union.f / (sqrtf(a2_union.f) * sqrtf(b2_union.f));
|
|
389
407
|
}
|
|
390
408
|
|
|
391
409
|
__attribute__((target("avx2"))) //
|
|
@@ -407,16 +425,17 @@ __attribute__((target("avx2"))) //
|
|
|
407
425
|
__attribute__((target("f16c"))) //
|
|
408
426
|
__attribute__((target("fma"))) inline static simsimd_f32_t //
|
|
409
427
|
simsimd_dot_f16x8_avx2(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
410
|
-
|
|
428
|
+
__m256 ab_vec = _mm256_set1_ps(0);
|
|
411
429
|
for (size_t i = 0; i != d; i += 8) {
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
ab_vec =
|
|
430
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
431
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
432
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
415
433
|
}
|
|
416
|
-
ab_vec =
|
|
417
|
-
ab_vec =
|
|
434
|
+
ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 0x81), ab_vec);
|
|
435
|
+
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
436
|
+
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
418
437
|
float result[1];
|
|
419
|
-
_mm_store_ss(result, ab_vec);
|
|
438
|
+
_mm_store_ss(result, _mm256_castps256_ps128(ab_vec));
|
|
420
439
|
return 1 - result[0];
|
|
421
440
|
}
|
|
422
441
|
|
|
@@ -424,47 +443,94 @@ __attribute__((target("avx2"))) //
|
|
|
424
443
|
__attribute__((target("f16c"))) //
|
|
425
444
|
__attribute__((target("fma"))) inline static simsimd_f32_t //
|
|
426
445
|
simsimd_cos_f16x8_avx2(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
446
|
+
__m256 ab_vec = _mm256_set1_ps(0);
|
|
447
|
+
__m256 a2_vec = _mm256_set1_ps(0);
|
|
448
|
+
__m256 b2_vec = _mm256_set1_ps(0);
|
|
430
449
|
for (size_t i = 0; i != d; i += 8) {
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
ab_vec =
|
|
434
|
-
a2_vec =
|
|
435
|
-
b2_vec =
|
|
450
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
451
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
452
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
453
|
+
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
454
|
+
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
436
455
|
}
|
|
437
|
-
ab_vec =
|
|
438
|
-
ab_vec =
|
|
439
|
-
|
|
440
|
-
a2_vec =
|
|
441
|
-
|
|
442
|
-
|
|
456
|
+
ab_vec = _mm256_add_ps(_mm256_permute2f128_ps(ab_vec, ab_vec, 0x81), ab_vec);
|
|
457
|
+
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
458
|
+
ab_vec = _mm256_hadd_ps(ab_vec, ab_vec);
|
|
459
|
+
a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 0x81), a2_vec);
|
|
460
|
+
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
461
|
+
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
462
|
+
b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 0x81), b2_vec);
|
|
463
|
+
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
464
|
+
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
443
465
|
float ab_result[1], a2_result[1], b2_result[1];
|
|
444
|
-
_mm_store_ss(ab_result, ab_vec);
|
|
445
|
-
_mm_store_ss(a2_result, a2_vec);
|
|
446
|
-
_mm_store_ss(b2_result, b2_vec);
|
|
447
|
-
return 1 - ab_result[0] / (
|
|
466
|
+
_mm_store_ss(ab_result, _mm256_castps256_ps128(ab_vec));
|
|
467
|
+
_mm_store_ss(a2_result, _mm256_castps256_ps128(a2_vec));
|
|
468
|
+
_mm_store_ss(b2_result, _mm256_castps256_ps128(b2_vec));
|
|
469
|
+
return 1 - ab_result[0] / (sqrtf(a2_result[0]) * sqrtf(b2_result[0]));
|
|
448
470
|
}
|
|
449
471
|
|
|
450
472
|
__attribute__((target("avx2"))) //
|
|
451
473
|
__attribute__((target("f16c"))) //
|
|
452
474
|
__attribute__((target("fma"))) inline static simsimd_f32_t //
|
|
453
475
|
simsimd_l2sq_f16x8_avx2(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t d) {
|
|
454
|
-
|
|
476
|
+
__m256 sum_vec = _mm256_set1_ps(0);
|
|
455
477
|
for (size_t i = 0; i != d; i += 8) {
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
sum_vec =
|
|
478
|
+
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
479
|
+
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
480
|
+
__m256 diff_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
481
|
+
sum_vec = _mm256_fmadd_ps(diff_vec, diff_vec, sum_vec);
|
|
460
482
|
}
|
|
461
|
-
sum_vec =
|
|
462
|
-
sum_vec =
|
|
483
|
+
sum_vec = _mm256_add_ps(_mm256_permute2f128_ps(sum_vec, sum_vec, 0x81), sum_vec);
|
|
484
|
+
sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
|
|
485
|
+
sum_vec = _mm256_hadd_ps(sum_vec, sum_vec);
|
|
463
486
|
float result[1];
|
|
464
|
-
_mm_store_ss(result, sum_vec);
|
|
487
|
+
_mm_store_ss(result, _mm256_castps256_ps128(sum_vec));
|
|
465
488
|
return result[0];
|
|
466
489
|
}
|
|
467
490
|
|
|
491
|
+
__attribute__((target("avx2"))) //
|
|
492
|
+
inline static simsimd_f32_t
|
|
493
|
+
simsimd_cos_i8x32_avx2(int8_t const* a, int8_t const* b, size_t d) {
|
|
494
|
+
|
|
495
|
+
__m256i ab_vec = _mm256_setzero_si256();
|
|
496
|
+
__m256i a2_vec = _mm256_setzero_si256();
|
|
497
|
+
__m256i b2_vec = _mm256_setzero_si256();
|
|
498
|
+
|
|
499
|
+
for (size_t i = 0; i != d; i += 32) {
|
|
500
|
+
__m256i a_vec = _mm256_loadu_si256((__m256i const*)(a + i));
|
|
501
|
+
__m256i b_vec = _mm256_loadu_si256((__m256i const*)(b + i));
|
|
502
|
+
|
|
503
|
+
// Multiply and add packed 8-bit integers
|
|
504
|
+
__m256i ab_part_vec = _mm256_maddubs_epi16(a_vec, b_vec);
|
|
505
|
+
__m256i a2_part_vec = _mm256_maddubs_epi16(a_vec, a_vec);
|
|
506
|
+
__m256i b2_part_vec = _mm256_maddubs_epi16(b_vec, b_vec);
|
|
507
|
+
|
|
508
|
+
ab_vec = _mm256_add_epi32(ab_vec, ab_part_vec);
|
|
509
|
+
a2_vec = _mm256_add_epi32(a2_vec, a2_part_vec);
|
|
510
|
+
b2_vec = _mm256_add_epi32(b2_vec, b2_part_vec);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Horizontal sum across the 256-bit register
|
|
514
|
+
__m128i ab_low = _mm256_extracti128_si256(ab_vec, 0);
|
|
515
|
+
__m128i ab_high = _mm256_extracti128_si256(ab_vec, 1);
|
|
516
|
+
__m128i ab_sum = _mm_add_epi32(ab_low, ab_high);
|
|
517
|
+
|
|
518
|
+
__m128i a2_low = _mm256_extracti128_si256(a2_vec, 0);
|
|
519
|
+
__m128i a2_high = _mm256_extracti128_si256(a2_vec, 1);
|
|
520
|
+
__m128i a2_sum = _mm_add_epi32(a2_low, a2_high);
|
|
521
|
+
|
|
522
|
+
__m128i b2_low = _mm256_extracti128_si256(b2_vec, 0);
|
|
523
|
+
__m128i b2_high = _mm256_extracti128_si256(b2_vec, 1);
|
|
524
|
+
__m128i b2_sum = _mm_add_epi32(b2_low, b2_high);
|
|
525
|
+
|
|
526
|
+
// Further reduce to a single sum for each vector
|
|
527
|
+
int ab = _mm_extract_epi32(_mm_hadd_epi32(_mm_hadd_epi32(ab_sum, ab_sum), ab_sum), 0);
|
|
528
|
+
int a2 = _mm_extract_epi32(_mm_hadd_epi32(_mm_hadd_epi32(a2_sum, a2_sum), a2_sum), 0);
|
|
529
|
+
int b2 = _mm_extract_epi32(_mm_hadd_epi32(_mm_hadd_epi32(b2_sum, b2_sum), b2_sum), 0);
|
|
530
|
+
|
|
531
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
532
|
+
}
|
|
533
|
+
|
|
468
534
|
#endif // SIMSIMD_TARGET_X86_AVX2
|
|
469
535
|
|
|
470
536
|
#if SIMSIMD_TARGET_X86_AVX512
|
|
@@ -498,7 +564,7 @@ simsimd_cos_f16x16_avx512(simsimd_f16_t const* a, simsimd_f16_t const* b, size_t
|
|
|
498
564
|
simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
|
|
499
565
|
simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
|
|
500
566
|
simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
|
|
501
|
-
return 1 - ab / (
|
|
567
|
+
return 1 - ab / (sqrtf(a2) * sqrtf(b2));
|
|
502
568
|
}
|
|
503
569
|
|
|
504
570
|
__attribute__((target("avx512fp16"))) //
|