usearch 2.15.2 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.gyp +1 -1
- package/c/CMakeLists.txt +10 -3
- package/cpp/README.md +1 -1
- package/cpp/bench.cpp +2 -2
- package/cpp/test.cpp +16 -0
- package/include/usearch/index.hpp +44 -17
- package/include/usearch/index_dense.hpp +51 -89
- package/include/usearch/index_plugins.hpp +468 -43
- package/javascript/lib.cpp +17 -5
- package/javascript/usearch.test.js +205 -34
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/simsimd/.git-blame-ignore-revs +2 -0
- package/simsimd/CODE_OF_CONDUCT.md +128 -0
- package/simsimd/CONTRIBUTING.md +84 -26
- package/simsimd/Cargo.lock +1 -1
- package/simsimd/Cargo.toml +4 -4
- package/simsimd/MANIFEST.in +3 -0
- package/simsimd/Package.swift +15 -3
- package/simsimd/README.md +534 -177
- package/simsimd/binding.gyp +6 -2
- package/simsimd/build.rs +13 -1
- package/simsimd/c/lib.c +266 -53
- package/simsimd/include/simsimd/binary.h +321 -112
- package/simsimd/include/simsimd/curved.h +803 -0
- package/simsimd/include/simsimd/dot.h +694 -521
- package/simsimd/include/simsimd/elementwise.h +1437 -0
- package/simsimd/include/simsimd/geospatial.h +9 -3
- package/simsimd/include/simsimd/mesh.h +69 -0
- package/simsimd/include/simsimd/probability.h +234 -206
- package/simsimd/include/simsimd/simsimd.h +1476 -617
- package/simsimd/include/simsimd/sparse.h +1428 -0
- package/simsimd/include/simsimd/spatial.h +1556 -742
- package/simsimd/include/simsimd/types.h +276 -132
- package/simsimd/javascript/lib.c +1 -2
- package/simsimd/javascript/simsimd.ts +4 -4
- package/simsimd/package.json +4 -4
- package/simsimd/rust/lib.rs +179 -1
- package/simsimd/scripts/bench.cxx +1070 -0
- package/simsimd/scripts/bench.ipynb +187 -0
- package/simsimd/{javascript → scripts}/bench.js +1 -1
- package/simsimd/{rust/benches/sqeuclidean.rs → scripts/bench_sqeuclidean.rs} +6 -6
- package/simsimd/scripts/bench_vectors.py +700 -0
- package/simsimd/scripts/bench_vectors_live.py +213 -0
- package/simsimd/scripts/test.c +155 -0
- package/simsimd/{javascript → scripts}/test.mjs +2 -2
- package/simsimd/scripts/test.py +1410 -0
- package/simsimd/swift/{source/SimSIMD.swift → SimSIMD.swift} +6 -2
- package/simsimd/swift/{tests/SimSIMDTests.swift → Test.swift} +1 -1
- /package/simsimd/{rust/benches/cosine.rs → scripts/bench_cosine.rs} +0 -0
- /package/simsimd/{rust/benches/native.rs → scripts/bench_native.rs} +0 -0
|
@@ -5,19 +5,21 @@
|
|
|
5
5
|
* @date March 14, 2023
|
|
6
6
|
*
|
|
7
7
|
* Contains:
|
|
8
|
-
* - L2 (Euclidean) squared distance
|
|
9
|
-
* - Cosine (Angular) similarity
|
|
8
|
+
* - L2 (Euclidean) regular and squared distance
|
|
9
|
+
* - Cosine (Angular) distance - @b not similarity!
|
|
10
10
|
*
|
|
11
11
|
* For datatypes:
|
|
12
12
|
* - 64-bit IEEE floating point numbers
|
|
13
13
|
* - 32-bit IEEE floating point numbers
|
|
14
14
|
* - 16-bit IEEE floating point numbers
|
|
15
15
|
* - 16-bit brain floating point numbers
|
|
16
|
+
* - 8-bit unsigned integral numbers
|
|
16
17
|
* - 8-bit signed integral numbers
|
|
18
|
+
* - 4-bit signed integral numbers
|
|
17
19
|
*
|
|
18
20
|
* For hardware architectures:
|
|
19
|
-
* - Arm
|
|
20
|
-
* - x86
|
|
21
|
+
* - Arm: NEON, SVE
|
|
22
|
+
* - x86: Haswell, Skylake, Ice Lake, Genoa, Sapphire
|
|
21
23
|
*
|
|
22
24
|
* x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
|
|
23
25
|
* Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
|
|
@@ -27,7 +29,7 @@
|
|
|
27
29
|
|
|
28
30
|
#include "types.h"
|
|
29
31
|
|
|
30
|
-
#include "dot.h" // `
|
|
32
|
+
#include "dot.h" // `_simsimd_reduce_f32x8_haswell`
|
|
31
33
|
|
|
32
34
|
#ifdef __cplusplus
|
|
33
35
|
extern "C" {
|
|
@@ -39,49 +41,75 @@ extern "C" {
|
|
|
39
41
|
* By default they use 32-bit arithmetic, unless the arguments themselves contain 64-bit floats.
|
|
40
42
|
* For double-precision computation check out the "*_accurate" variants of those "*_serial" functions.
|
|
41
43
|
*/
|
|
44
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
42
45
|
SIMSIMD_PUBLIC void simsimd_l2sq_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
43
46
|
SIMSIMD_PUBLIC void simsimd_cos_f64_serial(simsimd_f64_t const* a, simsimd_f64_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
47
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
44
48
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
45
49
|
SIMSIMD_PUBLIC void simsimd_cos_f32_serial(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
50
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
46
51
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
47
52
|
SIMSIMD_PUBLIC void simsimd_cos_f16_serial(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
53
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
48
54
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
49
55
|
SIMSIMD_PUBLIC void simsimd_cos_bf16_serial(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
56
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
50
57
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
51
58
|
SIMSIMD_PUBLIC void simsimd_cos_i8_serial(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
59
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
60
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
61
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_serial(simsimd_u8_t const* a, simsimd_u8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
52
62
|
|
|
53
63
|
/* Double-precision serial backends for all numeric types.
|
|
54
64
|
* For single-precision computation check out the "*_serial" counterparts of those "*_accurate" functions.
|
|
55
65
|
*/
|
|
66
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
56
67
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
57
68
|
SIMSIMD_PUBLIC void simsimd_cos_f32_accurate(simsimd_f32_t const* a, simsimd_f32_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
69
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
58
70
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
59
71
|
SIMSIMD_PUBLIC void simsimd_cos_f16_accurate(simsimd_f16_t const* a, simsimd_f16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
72
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
60
73
|
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
61
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_accurate(simsimd_bf16_t const* a, simsimd_bf16_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
62
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
63
|
-
SIMSIMD_PUBLIC void simsimd_cos_i8_accurate(simsimd_i8_t const* a, simsimd_i8_t const*, simsimd_size_t n, simsimd_distance_t* d);
|
|
64
74
|
|
|
65
75
|
/* SIMD-powered backends for Arm NEON, mostly using 32-bit arithmetic over 128-bit words.
|
|
66
76
|
* By far the most portable backend, covering most Arm v8 devices, over a billion phones, and almost all
|
|
67
77
|
* server CPUs produced before 2023.
|
|
68
78
|
*/
|
|
79
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
80
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
81
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
82
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
69
83
|
SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
70
84
|
SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
85
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
71
86
|
SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
72
87
|
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
88
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
89
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
90
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
91
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
73
92
|
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
74
93
|
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
94
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
95
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
96
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
75
97
|
|
|
76
98
|
/* SIMD-powered backends for Arm SVE, mostly using 32-bit arithmetic over variable-length platform-defined word sizes.
|
|
77
99
|
* Designed for Arm Graviton 3, Microsoft Cobalt, as well as Nvidia Grace and newer Ampere Altra CPUs.
|
|
78
100
|
*/
|
|
79
|
-
SIMSIMD_PUBLIC void
|
|
80
|
-
SIMSIMD_PUBLIC void
|
|
81
|
-
SIMSIMD_PUBLIC void
|
|
82
|
-
SIMSIMD_PUBLIC void
|
|
83
|
-
SIMSIMD_PUBLIC void
|
|
84
|
-
SIMSIMD_PUBLIC void
|
|
101
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
102
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
103
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
104
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
105
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
106
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
107
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
108
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
109
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
110
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
111
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
112
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
85
113
|
|
|
86
114
|
/* SIMD-powered backends for AVX2 CPUs of Haswell generation and newer, using 32-bit arithmetic over 256-bit words.
|
|
87
115
|
* First demonstrated in 2011, at least one Haswell-based processor was still being sold in 2022 — the Pentium G3420.
|
|
@@ -89,100 +117,201 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
89
117
|
* On other hand, there is no need to implement AVX2 versions of `f32` and `f64` functions, as those are
|
|
90
118
|
* properly vectorized by recent compilers.
|
|
91
119
|
*/
|
|
92
|
-
SIMSIMD_PUBLIC void
|
|
93
|
-
SIMSIMD_PUBLIC void
|
|
94
|
-
SIMSIMD_PUBLIC void
|
|
95
|
-
SIMSIMD_PUBLIC void
|
|
96
|
-
SIMSIMD_PUBLIC void
|
|
97
|
-
SIMSIMD_PUBLIC void
|
|
98
|
-
SIMSIMD_PUBLIC void
|
|
99
|
-
SIMSIMD_PUBLIC void
|
|
120
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
121
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
122
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
123
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
124
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
125
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
126
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
127
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
128
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
129
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
130
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
131
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
132
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
133
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
134
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
135
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
136
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
137
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
100
138
|
|
|
101
139
|
/* SIMD-powered backends for AVX512 CPUs of Skylake generation and newer, using 32-bit arithmetic over 512-bit words.
|
|
102
140
|
* Skylake was launched in 2015, and discontinued in 2019. Skylake had support for F, CD, VL, DQ, and BW extensions,
|
|
103
141
|
* as well as masked operations. This is enough to supersede auto-vectorization on `f32` and `f64` types.
|
|
104
142
|
*/
|
|
105
|
-
SIMSIMD_PUBLIC void
|
|
106
|
-
SIMSIMD_PUBLIC void
|
|
107
|
-
SIMSIMD_PUBLIC void
|
|
108
|
-
SIMSIMD_PUBLIC void
|
|
143
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
144
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
145
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const* a, simsimd_f32_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
146
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
147
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
148
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const* a, simsimd_f64_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
109
149
|
|
|
110
150
|
/* SIMD-powered backends for AVX512 CPUs of Ice Lake generation and newer, using mixed arithmetic over 512-bit words.
|
|
111
151
|
* Ice Lake added VNNI, VPOPCNTDQ, IFMA, VBMI, VAES, GFNI, VBMI2, BITALG, VPCLMULQDQ, and other extensions for integral operations.
|
|
112
152
|
* Sapphire Rapids added tiled matrix operations, but we are most interested in the new mixed-precision FMA instructions.
|
|
113
153
|
*/
|
|
114
|
-
SIMSIMD_PUBLIC void
|
|
115
|
-
SIMSIMD_PUBLIC void
|
|
116
|
-
SIMSIMD_PUBLIC void
|
|
117
|
-
SIMSIMD_PUBLIC void
|
|
118
|
-
SIMSIMD_PUBLIC void
|
|
119
|
-
SIMSIMD_PUBLIC void
|
|
120
|
-
|
|
154
|
+
SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
155
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
156
|
+
SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const* a, simsimd_i4x2_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
157
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
158
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
159
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
160
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
161
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
162
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const* a, simsimd_u8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
163
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
164
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
165
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
166
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
167
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
168
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const* a, simsimd_f16_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
169
|
+
|
|
170
|
+
/* SIMD-powered backends for AVX-INT8-VNNI extensions on Xeon 6 CPUs, including Sierra Forest and Granite Rapids.
|
|
171
|
+
* The packs many "efficiency" cores into a single socket, avoiding heavy 512-bit operations, and focusing on 256-bit ones.
|
|
172
|
+
*/
|
|
173
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const* a, simsimd_i8_t const* b, simsimd_size_t n, simsimd_distance_t* d);
|
|
121
174
|
// clang-format on
|
|
122
175
|
|
|
123
|
-
#define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type,
|
|
124
|
-
SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const*
|
|
125
|
-
simsimd_##input_type##_t const*
|
|
126
|
-
simsimd_distance_t*
|
|
127
|
-
simsimd_##accumulator_type##_t d2 = 0;
|
|
128
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
129
|
-
simsimd_##accumulator_type##_t ai =
|
|
130
|
-
simsimd_##accumulator_type##_t bi =
|
|
131
|
-
d2 += (ai - bi) * (ai - bi);
|
|
132
|
-
}
|
|
133
|
-
*result = d2;
|
|
176
|
+
#define SIMSIMD_MAKE_L2SQ(name, input_type, accumulator_type, load_and_convert) \
|
|
177
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
178
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
179
|
+
simsimd_distance_t *result) { \
|
|
180
|
+
simsimd_##accumulator_type##_t d2 = 0; \
|
|
181
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
182
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
183
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
184
|
+
d2 += (ai - bi) * (ai - bi); \
|
|
185
|
+
} \
|
|
186
|
+
*result = d2; \
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
#define SIMSIMD_MAKE_L2(name, input_type, accumulator_type, load_and_convert) \
|
|
190
|
+
SIMSIMD_PUBLIC void simsimd_l2_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
191
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
192
|
+
simsimd_distance_t *result) { \
|
|
193
|
+
simsimd_l2sq_##input_type##_##name(a, b, n, result); \
|
|
194
|
+
*result = SIMSIMD_SQRT(*result); \
|
|
134
195
|
}
|
|
135
196
|
|
|
136
|
-
#define SIMSIMD_MAKE_COS(name, input_type, accumulator_type,
|
|
137
|
-
SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const*
|
|
138
|
-
simsimd_##input_type##_t const*
|
|
139
|
-
simsimd_distance_t*
|
|
140
|
-
simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0;
|
|
141
|
-
for (simsimd_size_t i = 0; i != n; ++i) {
|
|
142
|
-
simsimd_##accumulator_type##_t ai =
|
|
143
|
-
simsimd_##accumulator_type##_t bi =
|
|
144
|
-
ab += ai * bi;
|
|
145
|
-
a2 += ai * ai;
|
|
146
|
-
b2 += bi * bi;
|
|
147
|
-
}
|
|
148
|
-
|
|
197
|
+
#define SIMSIMD_MAKE_COS(name, input_type, accumulator_type, load_and_convert) \
|
|
198
|
+
SIMSIMD_PUBLIC void simsimd_cos_##input_type##_##name(simsimd_##input_type##_t const *a, \
|
|
199
|
+
simsimd_##input_type##_t const *b, simsimd_size_t n, \
|
|
200
|
+
simsimd_distance_t *result) { \
|
|
201
|
+
simsimd_##accumulator_type##_t ab = 0, a2 = 0, b2 = 0; \
|
|
202
|
+
for (simsimd_size_t i = 0; i != n; ++i) { \
|
|
203
|
+
simsimd_##accumulator_type##_t ai = load_and_convert(a + i); \
|
|
204
|
+
simsimd_##accumulator_type##_t bi = load_and_convert(b + i); \
|
|
205
|
+
ab += ai * bi; \
|
|
206
|
+
a2 += ai * ai; \
|
|
207
|
+
b2 += bi * bi; \
|
|
208
|
+
} \
|
|
209
|
+
if (a2 == 0 && b2 == 0) { *result = 0; } \
|
|
210
|
+
else if (ab == 0) { *result = 1; } \
|
|
211
|
+
else { \
|
|
212
|
+
simsimd_distance_t unclipped_result = 1 - ab * SIMSIMD_RSQRT(a2) * SIMSIMD_RSQRT(b2); \
|
|
213
|
+
*result = unclipped_result > 0 ? unclipped_result : 0; \
|
|
214
|
+
} \
|
|
149
215
|
}
|
|
150
216
|
|
|
151
|
-
|
|
152
|
-
|
|
217
|
+
SIMSIMD_MAKE_COS(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f64_serial
|
|
218
|
+
SIMSIMD_MAKE_L2SQ(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f64_serial
|
|
219
|
+
SIMSIMD_MAKE_L2(serial, f64, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f64_serial
|
|
153
220
|
|
|
154
|
-
|
|
155
|
-
|
|
221
|
+
SIMSIMD_MAKE_COS(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_serial
|
|
222
|
+
SIMSIMD_MAKE_L2SQ(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_serial
|
|
223
|
+
SIMSIMD_MAKE_L2(serial, f32, f32, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_serial
|
|
156
224
|
|
|
157
|
-
|
|
158
|
-
|
|
225
|
+
SIMSIMD_MAKE_COS(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_serial
|
|
226
|
+
SIMSIMD_MAKE_L2SQ(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_serial
|
|
227
|
+
SIMSIMD_MAKE_L2(serial, f16, f32, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_serial
|
|
159
228
|
|
|
160
|
-
|
|
161
|
-
|
|
229
|
+
SIMSIMD_MAKE_COS(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_serial
|
|
230
|
+
SIMSIMD_MAKE_L2SQ(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_serial
|
|
231
|
+
SIMSIMD_MAKE_L2(serial, bf16, f32, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_serial
|
|
162
232
|
|
|
163
|
-
|
|
164
|
-
|
|
233
|
+
SIMSIMD_MAKE_COS(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_i8_serial
|
|
234
|
+
SIMSIMD_MAKE_L2SQ(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_i8_serial
|
|
235
|
+
SIMSIMD_MAKE_L2(serial, i8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_i8_serial
|
|
165
236
|
|
|
166
|
-
|
|
167
|
-
|
|
237
|
+
SIMSIMD_MAKE_COS(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_cos_u8_serial
|
|
238
|
+
SIMSIMD_MAKE_L2SQ(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2sq_u8_serial
|
|
239
|
+
SIMSIMD_MAKE_L2(serial, u8, i32, SIMSIMD_DEREFERENCE) // simsimd_l2_u8_serial
|
|
168
240
|
|
|
169
|
-
|
|
170
|
-
|
|
241
|
+
SIMSIMD_MAKE_COS(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_cos_f32_accurate
|
|
242
|
+
SIMSIMD_MAKE_L2SQ(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2sq_f32_accurate
|
|
243
|
+
SIMSIMD_MAKE_L2(accurate, f32, f64, SIMSIMD_DEREFERENCE) // simsimd_l2_f32_accurate
|
|
171
244
|
|
|
172
|
-
|
|
173
|
-
|
|
245
|
+
SIMSIMD_MAKE_COS(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_cos_f16_accurate
|
|
246
|
+
SIMSIMD_MAKE_L2SQ(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2sq_f16_accurate
|
|
247
|
+
SIMSIMD_MAKE_L2(accurate, f16, f64, SIMSIMD_F16_TO_F32) // simsimd_l2_f16_accurate
|
|
174
248
|
|
|
175
|
-
|
|
176
|
-
|
|
249
|
+
SIMSIMD_MAKE_COS(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_cos_bf16_accurate
|
|
250
|
+
SIMSIMD_MAKE_L2SQ(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2sq_bf16_accurate
|
|
251
|
+
SIMSIMD_MAKE_L2(accurate, bf16, f64, SIMSIMD_BF16_TO_F32) // simsimd_l2_bf16_accurate
|
|
177
252
|
|
|
178
|
-
#if
|
|
253
|
+
#if _SIMSIMD_TARGET_ARM
|
|
179
254
|
#if SIMSIMD_TARGET_NEON
|
|
180
255
|
#pragma GCC push_options
|
|
181
|
-
#pragma GCC target("+simd")
|
|
182
|
-
#pragma clang attribute push(__attribute__((target("+simd"))), apply_to = function)
|
|
256
|
+
#pragma GCC target("arch=armv8.2-a+simd")
|
|
257
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd"))), apply_to = function)
|
|
183
258
|
|
|
184
|
-
|
|
185
|
-
|
|
259
|
+
SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_neon(simsimd_f32_t x) {
|
|
260
|
+
return vget_lane_f32(vsqrt_f32(vdup_n_f32(x)), 0);
|
|
261
|
+
}
|
|
262
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_neon(simsimd_f64_t x) {
|
|
263
|
+
return vget_lane_f64(vsqrt_f64(vdup_n_f64(x)), 0);
|
|
264
|
+
}
|
|
265
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_neon(simsimd_f32_t ab, simsimd_f32_t a2,
|
|
266
|
+
simsimd_f32_t b2) {
|
|
267
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
268
|
+
if (ab == 0) return 1;
|
|
269
|
+
simsimd_f32_t squares_arr[2] = {a2, b2};
|
|
270
|
+
float32x2_t squares = vld1_f32(squares_arr);
|
|
271
|
+
// Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
|
|
272
|
+
// Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
|
|
273
|
+
// One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
|
|
274
|
+
// https://github.com/lighttransport/embree-aarch64/issues/24
|
|
275
|
+
// https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
|
|
276
|
+
float32x2_t rsqrts = vrsqrte_f32(squares);
|
|
277
|
+
// Perform two rounds of Newton-Raphson refinement:
|
|
278
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
279
|
+
rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
|
|
280
|
+
rsqrts = vmul_f32(rsqrts, vrsqrts_f32(vmul_f32(squares, rsqrts), rsqrts));
|
|
281
|
+
vst1_f32(squares_arr, rsqrts);
|
|
282
|
+
simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
|
|
283
|
+
return result > 0 ? result : 0;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_neon(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
287
|
+
simsimd_f64_t b2) {
|
|
288
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
289
|
+
if (ab == 0) return 1;
|
|
290
|
+
simsimd_f64_t squares_arr[2] = {a2, b2};
|
|
291
|
+
float64x2_t squares = vld1q_f64(squares_arr);
|
|
292
|
+
|
|
293
|
+
// Unlike x86, Arm NEON manuals don't explicitly mention the accuracy of their `rsqrt` approximation.
|
|
294
|
+
// Third party research suggests, that it's less accurate than SSE instructions, having an error of 1.5*2^-12.
|
|
295
|
+
// One or two rounds of Newton-Raphson refinement are recommended to improve the accuracy.
|
|
296
|
+
// https://github.com/lighttransport/embree-aarch64/issues/24
|
|
297
|
+
// https://github.com/lighttransport/embree-aarch64/blob/3f75f8cb4e553d13dced941b5fefd4c826835a6b/common/math/math.h#L137-L145
|
|
298
|
+
float64x2_t rsqrts = vrsqrteq_f64(squares);
|
|
299
|
+
// Perform two rounds of Newton-Raphson refinement:
|
|
300
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
301
|
+
rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
|
|
302
|
+
rsqrts = vmulq_f64(rsqrts, vrsqrtsq_f64(vmulq_f64(squares, rsqrts), rsqrts));
|
|
303
|
+
vst1q_f64(squares_arr, rsqrts);
|
|
304
|
+
simsimd_distance_t result = 1 - ab * squares_arr[0] * squares_arr[1];
|
|
305
|
+
return result > 0 ? result : 0;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
309
|
+
simsimd_distance_t *result) {
|
|
310
|
+
simsimd_l2sq_f32_neon(a, b, n, result);
|
|
311
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
312
|
+
}
|
|
313
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
314
|
+
simsimd_distance_t *result) {
|
|
186
315
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
187
316
|
simsimd_size_t i = 0;
|
|
188
317
|
for (; i + 4 <= n; i += 4) {
|
|
@@ -199,8 +328,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_neon(simsimd_f32_t const* a, simsimd_f32_t
|
|
|
199
328
|
*result = sum;
|
|
200
329
|
}
|
|
201
330
|
|
|
202
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const*
|
|
203
|
-
simsimd_distance_t*
|
|
331
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
332
|
+
simsimd_distance_t *result) {
|
|
204
333
|
float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
|
|
205
334
|
simsimd_size_t i = 0;
|
|
206
335
|
for (; i + 4 <= n; i += 4) {
|
|
@@ -216,103 +345,125 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_neon(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
216
345
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
217
346
|
}
|
|
218
347
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
348
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
352
|
+
simsimd_distance_t *result) {
|
|
353
|
+
simsimd_l2sq_f64_neon(a, b, n, result);
|
|
354
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
355
|
+
}
|
|
356
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
357
|
+
simsimd_distance_t *result) {
|
|
358
|
+
float64x2_t sum_vec = vdupq_n_f64(0);
|
|
359
|
+
simsimd_size_t i = 0;
|
|
360
|
+
for (; i + 2 <= n; i += 2) {
|
|
361
|
+
float64x2_t a_vec = vld1q_f64(a + i);
|
|
362
|
+
float64x2_t b_vec = vld1q_f64(b + i);
|
|
363
|
+
float64x2_t diff_vec = vsubq_f64(a_vec, b_vec);
|
|
364
|
+
sum_vec = vfmaq_f64(sum_vec, diff_vec, diff_vec);
|
|
365
|
+
}
|
|
366
|
+
simsimd_f64_t sum = vaddvq_f64(sum_vec);
|
|
367
|
+
for (; i < n; ++i) {
|
|
368
|
+
simsimd_f64_t diff = a[i] - b[i];
|
|
369
|
+
sum += diff * diff;
|
|
370
|
+
}
|
|
371
|
+
*result = sum;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_neon(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
375
|
+
simsimd_distance_t *result) {
|
|
376
|
+
float64x2_t ab_vec = vdupq_n_f64(0), a2_vec = vdupq_n_f64(0), b2_vec = vdupq_n_f64(0);
|
|
377
|
+
simsimd_size_t i = 0;
|
|
378
|
+
for (; i + 2 <= n; i += 2) {
|
|
379
|
+
float64x2_t a_vec = vld1q_f64(a + i);
|
|
380
|
+
float64x2_t b_vec = vld1q_f64(b + i);
|
|
381
|
+
ab_vec = vfmaq_f64(ab_vec, a_vec, b_vec);
|
|
382
|
+
a2_vec = vfmaq_f64(a2_vec, a_vec, a_vec);
|
|
383
|
+
b2_vec = vfmaq_f64(b2_vec, b_vec, b_vec);
|
|
384
|
+
}
|
|
385
|
+
simsimd_f64_t ab = vaddvq_f64(ab_vec), a2 = vaddvq_f64(a2_vec), b2 = vaddvq_f64(b2_vec);
|
|
386
|
+
for (; i < n; ++i) {
|
|
387
|
+
simsimd_f64_t ai = a[i], bi = b[i];
|
|
388
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
223
392
|
}
|
|
224
393
|
|
|
225
394
|
#pragma clang attribute pop
|
|
226
395
|
#pragma GCC pop_options
|
|
396
|
+
#endif // SIMSIMD_TARGET_NEON
|
|
227
397
|
|
|
398
|
+
#if SIMSIMD_TARGET_NEON_F16
|
|
228
399
|
#pragma GCC push_options
|
|
229
|
-
#pragma GCC target("+simd+fp16")
|
|
230
|
-
#pragma clang attribute push(__attribute__((target("+simd+fp16"))), apply_to = function)
|
|
400
|
+
#pragma GCC target("arch=armv8.2-a+simd+fp16")
|
|
401
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
|
|
231
402
|
|
|
232
|
-
SIMSIMD_PUBLIC void
|
|
233
|
-
|
|
403
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
404
|
+
simsimd_distance_t *result) {
|
|
405
|
+
simsimd_l2sq_f16_neon(a, b, n, result);
|
|
406
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
407
|
+
}
|
|
408
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
409
|
+
simsimd_distance_t *result) {
|
|
410
|
+
float32x4_t a_vec, b_vec;
|
|
234
411
|
float32x4_t sum_vec = vdupq_n_f32(0);
|
|
235
|
-
simsimd_size_t i = 0;
|
|
236
|
-
for (; i + 4 <= n; i += 4) {
|
|
237
|
-
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
|
|
238
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
|
|
239
|
-
float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
|
|
240
|
-
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
241
|
-
}
|
|
242
412
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
254
|
-
for (; j < 4; ++j)
|
|
255
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
256
|
-
float32x4_t diff_vec = vsubq_f32(vcvt_f32_f16(a_padded_tail.f16_vec), vcvt_f32_f16(b_padded_tail.f16_vec));
|
|
257
|
-
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
413
|
+
simsimd_l2sq_f16_neon_cycle:
|
|
414
|
+
if (n < 4) {
|
|
415
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
416
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
417
|
+
n = 0;
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
421
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
422
|
+
n -= 4, a += 4, b += 4;
|
|
258
423
|
}
|
|
424
|
+
float32x4_t diff_vec = vsubq_f32(a_vec, b_vec);
|
|
425
|
+
sum_vec = vfmaq_f32(sum_vec, diff_vec, diff_vec);
|
|
426
|
+
if (n) goto simsimd_l2sq_f16_neon_cycle;
|
|
259
427
|
|
|
260
|
-
|
|
261
|
-
*result = sum;
|
|
428
|
+
*result = vaddvq_f32(sum_vec);
|
|
262
429
|
}
|
|
263
430
|
|
|
264
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const*
|
|
265
|
-
simsimd_distance_t*
|
|
431
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_neon(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
432
|
+
simsimd_distance_t *result) {
|
|
266
433
|
float32x4_t ab_vec = vdupq_n_f32(0), a2_vec = vdupq_n_f32(0), b2_vec = vdupq_n_f32(0);
|
|
267
|
-
|
|
268
|
-
for (; i + 4 <= n; i += 4) {
|
|
269
|
-
float32x4_t a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)a + i));
|
|
270
|
-
float32x4_t b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const*)b + i));
|
|
271
|
-
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
272
|
-
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
273
|
-
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
274
|
-
}
|
|
434
|
+
float32x4_t a_vec, b_vec;
|
|
275
435
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
float16x4_t f16_vec;
|
|
282
|
-
simsimd_f16_t f16[4];
|
|
283
|
-
} a_padded_tail, b_padded_tail;
|
|
284
|
-
simsimd_size_t j = 0;
|
|
285
|
-
for (; i < n; ++i, ++j)
|
|
286
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
287
|
-
for (; j < 4; ++j)
|
|
288
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
289
|
-
float32x4_t a_vec = vcvt_f32_f16(a_padded_tail.f16_vec);
|
|
290
|
-
float32x4_t b_vec = vcvt_f32_f16(b_padded_tail.f16_vec);
|
|
291
|
-
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
292
|
-
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
293
|
-
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
436
|
+
simsimd_cos_f16_neon_cycle:
|
|
437
|
+
if (n < 4) {
|
|
438
|
+
a_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(a, n));
|
|
439
|
+
b_vec = vcvt_f32_f16(_simsimd_partial_load_f16x4_neon(b, n));
|
|
440
|
+
n = 0;
|
|
294
441
|
}
|
|
442
|
+
else {
|
|
443
|
+
a_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)a));
|
|
444
|
+
b_vec = vcvt_f32_f16(vld1_f16((simsimd_f16_for_arm_simd_t const *)b));
|
|
445
|
+
n -= 4, a += 4, b += 4;
|
|
446
|
+
}
|
|
447
|
+
ab_vec = vfmaq_f32(ab_vec, a_vec, b_vec);
|
|
448
|
+
a2_vec = vfmaq_f32(a2_vec, a_vec, a_vec);
|
|
449
|
+
b2_vec = vfmaq_f32(b2_vec, b_vec, b_vec);
|
|
450
|
+
if (n) goto simsimd_cos_f16_neon_cycle;
|
|
295
451
|
|
|
296
|
-
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
297
452
|
simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
|
|
298
|
-
|
|
299
|
-
float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
|
|
300
|
-
a2_b2 = vrsqrte_f32(a2_b2);
|
|
301
|
-
vst1_f32(a2_b2_arr, a2_b2);
|
|
302
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
453
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
303
454
|
}
|
|
304
455
|
|
|
305
456
|
#pragma clang attribute pop
|
|
306
457
|
#pragma GCC pop_options
|
|
307
|
-
#endif //
|
|
458
|
+
#endif // SIMSIMD_TARGET_NEON_F16
|
|
308
459
|
|
|
309
460
|
#if SIMSIMD_TARGET_NEON_BF16
|
|
310
461
|
#pragma GCC push_options
|
|
311
462
|
#pragma GCC target("arch=armv8.6-a+simd+bf16")
|
|
312
463
|
#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
|
|
313
464
|
|
|
314
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const*
|
|
315
|
-
simsimd_distance_t*
|
|
465
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
466
|
+
simsimd_distance_t *result) {
|
|
316
467
|
|
|
317
468
|
// Similar to `simsimd_cos_i8_neon`, we can use the `BFMMLA` instruction through
|
|
318
469
|
// the `vbfmmlaq_f32` intrinsic to compute matrix products and later drop 1/4 of values.
|
|
@@ -341,95 +492,76 @@ SIMSIMD_PUBLIC void simsimd_cos_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16_
|
|
|
341
492
|
// }
|
|
342
493
|
// float32x4_t products_vec = vaddq_f32(products_high_vec, products_low_vec);
|
|
343
494
|
// simsimd_f32_t a2 = products_vec[0], ab = products_vec[1], b2 = products_vec[3];
|
|
495
|
+
//
|
|
496
|
+
// Another way of accomplishing the same thing is to process the odd and even elements separately,
|
|
497
|
+
// using special `vbfmlaltq_f32` and `vbfmlalbq_f32` intrinsics:
|
|
498
|
+
//
|
|
499
|
+
// ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_vec, b_vec);
|
|
500
|
+
// ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_vec, b_vec);
|
|
501
|
+
// a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_vec, a_vec);
|
|
502
|
+
// a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
|
|
503
|
+
// b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
|
|
504
|
+
// b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
|
|
505
|
+
//
|
|
344
506
|
|
|
345
|
-
float32x4_t
|
|
346
|
-
float32x4_t
|
|
347
|
-
float32x4_t
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_vec, a_vec);
|
|
356
|
-
b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_vec, b_vec);
|
|
357
|
-
b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_vec, b_vec);
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
// In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
|
|
361
|
-
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
362
|
-
// loads and stores with vectorized math.
|
|
363
|
-
if (i < n) {
|
|
364
|
-
union {
|
|
365
|
-
bfloat16x8_t bf16_vec;
|
|
366
|
-
simsimd_bf16_t bf16[8];
|
|
367
|
-
} a_padded_tail, b_padded_tail;
|
|
368
|
-
simsimd_size_t j = 0;
|
|
369
|
-
for (; i < n; ++i, ++j)
|
|
370
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
371
|
-
for (; j < 8; ++j)
|
|
372
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
373
|
-
ab_high_vec = vbfmlaltq_f32(ab_high_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
374
|
-
ab_low_vec = vbfmlalbq_f32(ab_low_vec, a_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
375
|
-
a2_high_vec = vbfmlaltq_f32(a2_high_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
|
|
376
|
-
a2_low_vec = vbfmlalbq_f32(a2_low_vec, a_padded_tail.bf16_vec, a_padded_tail.bf16_vec);
|
|
377
|
-
b2_high_vec = vbfmlaltq_f32(b2_high_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
378
|
-
b2_low_vec = vbfmlalbq_f32(b2_low_vec, b_padded_tail.bf16_vec, b_padded_tail.bf16_vec);
|
|
507
|
+
float32x4_t ab_vec = vdupq_n_f32(0);
|
|
508
|
+
float32x4_t a2_vec = vdupq_n_f32(0);
|
|
509
|
+
float32x4_t b2_vec = vdupq_n_f32(0);
|
|
510
|
+
bfloat16x8_t a_vec, b_vec;
|
|
511
|
+
|
|
512
|
+
simsimd_cos_bf16_neon_cycle:
|
|
513
|
+
if (n < 8) {
|
|
514
|
+
a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
515
|
+
b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
516
|
+
n = 0;
|
|
379
517
|
}
|
|
518
|
+
else {
|
|
519
|
+
a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
|
|
520
|
+
b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
|
|
521
|
+
n -= 8, a += 8, b += 8;
|
|
522
|
+
}
|
|
523
|
+
ab_vec = vbfdotq_f32(ab_vec, a_vec, b_vec);
|
|
524
|
+
a2_vec = vbfdotq_f32(a2_vec, a_vec, a_vec);
|
|
525
|
+
b2_vec = vbfdotq_f32(b2_vec, b_vec, b_vec);
|
|
526
|
+
if (n) goto simsimd_cos_bf16_neon_cycle;
|
|
380
527
|
|
|
381
528
|
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
382
|
-
simsimd_f32_t ab = vaddvq_f32(
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
*result =
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
float32x4_t diff_high_vec = vdupq_n_f32(0), diff_low_vec = vdupq_n_f32(0);
|
|
529
|
+
simsimd_f32_t ab = vaddvq_f32(ab_vec), a2 = vaddvq_f32(a2_vec), b2 = vaddvq_f32(b2_vec);
|
|
530
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
534
|
+
simsimd_distance_t *result) {
|
|
535
|
+
simsimd_l2sq_bf16_neon(a, b, n, result);
|
|
536
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
537
|
+
}
|
|
538
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
539
|
+
simsimd_distance_t *result) {
|
|
540
|
+
float32x4_t diff_high_vec, diff_low_vec;
|
|
395
541
|
float32x4_t sum_high_vec = vdupq_n_f32(0), sum_low_vec = vdupq_n_f32(0);
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
bfloat16x8_t
|
|
400
|
-
|
|
401
|
-
// and then subtract, converting back to `bf16` for computing the squared difference.
|
|
542
|
+
|
|
543
|
+
simsimd_l2sq_bf16_neon_cycle:
|
|
544
|
+
if (n < 8) {
|
|
545
|
+
bfloat16x8_t a_vec = _simsimd_partial_load_bf16x8_neon(a, n);
|
|
546
|
+
bfloat16x8_t b_vec = _simsimd_partial_load_bf16x8_neon(b, n);
|
|
402
547
|
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
|
|
403
548
|
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
|
|
404
|
-
|
|
405
|
-
sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
// In case the software emulation for `bf16` scalars is enabled, the `simsimd_uncompress_bf16`
|
|
409
|
-
// function will run. It is extremely slow, so even for the tail, let's combine serial
|
|
410
|
-
// loads and stores with vectorized math.
|
|
411
|
-
if (i < n) {
|
|
412
|
-
union {
|
|
413
|
-
bfloat16x8_t bf16_vec;
|
|
414
|
-
simsimd_bf16_t bf16[8];
|
|
415
|
-
} a_padded_tail, b_padded_tail;
|
|
416
|
-
simsimd_size_t j = 0;
|
|
417
|
-
for (; i < n; ++i, ++j)
|
|
418
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
419
|
-
for (; j < 8; ++j)
|
|
420
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
421
|
-
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_padded_tail.bf16_vec)),
|
|
422
|
-
vcvt_f32_bf16(vget_high_bf16(b_padded_tail.bf16_vec)));
|
|
423
|
-
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_padded_tail.bf16_vec)),
|
|
424
|
-
vcvt_f32_bf16(vget_low_bf16(b_padded_tail.bf16_vec)));
|
|
425
|
-
sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
|
|
426
|
-
sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
|
|
549
|
+
n = 0;
|
|
427
550
|
}
|
|
551
|
+
else {
|
|
552
|
+
bfloat16x8_t a_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)a);
|
|
553
|
+
bfloat16x8_t b_vec = vld1q_bf16((simsimd_bf16_for_arm_simd_t const *)b);
|
|
554
|
+
diff_high_vec = vsubq_f32(vcvt_f32_bf16(vget_high_bf16(a_vec)), vcvt_f32_bf16(vget_high_bf16(b_vec)));
|
|
555
|
+
diff_low_vec = vsubq_f32(vcvt_f32_bf16(vget_low_bf16(a_vec)), vcvt_f32_bf16(vget_low_bf16(b_vec)));
|
|
556
|
+
n -= 8, a += 8, b += 8;
|
|
557
|
+
}
|
|
558
|
+
sum_high_vec = vfmaq_f32(sum_high_vec, diff_high_vec, diff_high_vec);
|
|
559
|
+
sum_low_vec = vfmaq_f32(sum_low_vec, diff_low_vec, diff_low_vec);
|
|
560
|
+
if (n) goto simsimd_l2sq_bf16_neon_cycle;
|
|
428
561
|
|
|
429
|
-
|
|
430
|
-
simsimd_f32_t sum = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
|
|
431
|
-
*result = sum;
|
|
562
|
+
*result = vaddvq_f32(vaddq_f32(sum_high_vec, sum_low_vec));
|
|
432
563
|
}
|
|
564
|
+
|
|
433
565
|
#pragma clang attribute pop
|
|
434
566
|
#pragma GCC pop_options
|
|
435
567
|
#endif // SIMSIMD_TARGET_NEON_BF16
|
|
@@ -439,30 +571,40 @@ SIMSIMD_PUBLIC void simsimd_l2sq_bf16_neon(simsimd_bf16_t const* a, simsimd_bf16
|
|
|
439
571
|
#pragma GCC target("arch=armv8.2-a+dotprod+i8mm")
|
|
440
572
|
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod+i8mm"))), apply_to = function)
|
|
441
573
|
|
|
442
|
-
SIMSIMD_PUBLIC void
|
|
443
|
-
|
|
444
|
-
|
|
574
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
575
|
+
simsimd_distance_t *result) {
|
|
576
|
+
simsimd_l2sq_i8_neon(a, b, n, result);
|
|
577
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
578
|
+
}
|
|
579
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
580
|
+
simsimd_distance_t *result) {
|
|
581
|
+
|
|
582
|
+
// The naive approach is to upcast 8-bit signed integers into 16-bit signed integers
|
|
583
|
+
// for subtraction, then multiply within 16-bit integers and accumulate the results
|
|
584
|
+
// into 32-bit integers. This approach is slow on modern Arm CPUs. On Graviton 4,
|
|
585
|
+
// that approach results in 17 GB/s of throughput, compared to 39 GB/s for `i8`
|
|
586
|
+
// dot-products.
|
|
587
|
+
//
|
|
588
|
+
// Luckily we can use the `vabdq_s8` which technically returns `i8` values, but it's a
|
|
589
|
+
// matter of reinterpret-casting! That approach boosts us to 33 GB/s of throughput.
|
|
590
|
+
uint32x4_t d2_vec = vdupq_n_u32(0);
|
|
445
591
|
simsimd_size_t i = 0;
|
|
446
|
-
for (; i +
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
int32x4_t d_high = vmull_s16(vget_high_s16(d_vec), vget_high_s16(d_vec));
|
|
454
|
-
d2_vec = vaddq_s32(d2_vec, vaddq_s32(d_low, d_high));
|
|
455
|
-
}
|
|
456
|
-
int32_t d2 = vaddvq_s32(d2_vec);
|
|
592
|
+
for (; i + 16 <= n; i += 16) {
|
|
593
|
+
int8x16_t a_vec = vld1q_s8(a + i);
|
|
594
|
+
int8x16_t b_vec = vld1q_s8(b + i);
|
|
595
|
+
uint8x16_t d_vec = vreinterpretq_u8_s8(vabdq_s8(a_vec, b_vec));
|
|
596
|
+
d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
|
|
597
|
+
}
|
|
598
|
+
uint32_t d2 = vaddvq_u32(d2_vec);
|
|
457
599
|
for (; i < n; ++i) {
|
|
458
|
-
int32_t n = a[i] - b[i];
|
|
459
|
-
d2 += n * n;
|
|
600
|
+
int32_t n = (int32_t)a[i] - b[i];
|
|
601
|
+
d2 += (uint32_t)(n * n);
|
|
460
602
|
}
|
|
461
603
|
*result = d2;
|
|
462
604
|
}
|
|
463
605
|
|
|
464
|
-
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const*
|
|
465
|
-
simsimd_distance_t*
|
|
606
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
607
|
+
simsimd_distance_t *result) {
|
|
466
608
|
|
|
467
609
|
simsimd_size_t i = 0;
|
|
468
610
|
|
|
@@ -577,12 +719,57 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
577
719
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
578
720
|
}
|
|
579
721
|
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
722
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
726
|
+
simsimd_distance_t *result) {
|
|
727
|
+
simsimd_l2sq_u8_neon(a, b, n, result);
|
|
728
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
729
|
+
}
|
|
730
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
731
|
+
simsimd_distance_t *result) {
|
|
732
|
+
uint32x4_t d2_vec = vdupq_n_u32(0);
|
|
733
|
+
simsimd_size_t i = 0;
|
|
734
|
+
for (; i + 16 <= n; i += 16) {
|
|
735
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
736
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
737
|
+
uint8x16_t d_vec = vabdq_u8(a_vec, b_vec);
|
|
738
|
+
d2_vec = vdotq_u32(d2_vec, d_vec, d_vec);
|
|
739
|
+
}
|
|
740
|
+
uint32_t d2 = vaddvq_u32(d2_vec);
|
|
741
|
+
for (; i < n; ++i) {
|
|
742
|
+
int32_t n = (int32_t)a[i] - b[i];
|
|
743
|
+
d2 += (uint32_t)(n * n);
|
|
744
|
+
}
|
|
745
|
+
*result = d2;
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_neon(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
749
|
+
simsimd_distance_t *result) {
|
|
750
|
+
|
|
751
|
+
simsimd_size_t i = 0;
|
|
752
|
+
uint32x4_t ab_vec = vdupq_n_u32(0);
|
|
753
|
+
uint32x4_t a2_vec = vdupq_n_u32(0);
|
|
754
|
+
uint32x4_t b2_vec = vdupq_n_u32(0);
|
|
755
|
+
for (; i + 16 <= n; i += 16) {
|
|
756
|
+
uint8x16_t a_vec = vld1q_u8(a + i);
|
|
757
|
+
uint8x16_t b_vec = vld1q_u8(b + i);
|
|
758
|
+
ab_vec = vdotq_u32(ab_vec, a_vec, b_vec);
|
|
759
|
+
a2_vec = vdotq_u32(a2_vec, a_vec, a_vec);
|
|
760
|
+
b2_vec = vdotq_u32(b2_vec, b_vec, b_vec);
|
|
761
|
+
}
|
|
762
|
+
uint32_t ab = vaddvq_u32(ab_vec);
|
|
763
|
+
uint32_t a2 = vaddvq_u32(a2_vec);
|
|
764
|
+
uint32_t b2 = vaddvq_u32(b2_vec);
|
|
765
|
+
|
|
766
|
+
// Take care of the tail:
|
|
767
|
+
for (; i < n; ++i) {
|
|
768
|
+
uint32_t ai = a[i], bi = b[i];
|
|
769
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
586
773
|
}
|
|
587
774
|
|
|
588
775
|
#pragma clang attribute pop
|
|
@@ -591,11 +778,16 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_neon(simsimd_i8_t const* a, simsimd_i8_t cons
|
|
|
591
778
|
|
|
592
779
|
#if SIMSIMD_TARGET_SVE
|
|
593
780
|
#pragma GCC push_options
|
|
594
|
-
#pragma GCC target("+sve")
|
|
595
|
-
#pragma clang attribute push(__attribute__((target("+sve"))), apply_to = function)
|
|
781
|
+
#pragma GCC target("arch=armv8.2-a+sve")
|
|
782
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
|
|
596
783
|
|
|
597
|
-
SIMSIMD_PUBLIC void
|
|
598
|
-
|
|
784
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
785
|
+
simsimd_distance_t *result) {
|
|
786
|
+
simsimd_l2sq_f32_sve(a, b, n, result);
|
|
787
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
788
|
+
}
|
|
789
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
790
|
+
simsimd_distance_t *result) {
|
|
599
791
|
simsimd_size_t i = 0;
|
|
600
792
|
svfloat32_t d2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
601
793
|
do {
|
|
@@ -610,8 +802,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_sve(simsimd_f32_t const* a, simsimd_f32_t c
|
|
|
610
802
|
*result = d2;
|
|
611
803
|
}
|
|
612
804
|
|
|
613
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const*
|
|
614
|
-
simsimd_distance_t*
|
|
805
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
806
|
+
simsimd_distance_t *result) {
|
|
615
807
|
simsimd_size_t i = 0;
|
|
616
808
|
svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
617
809
|
svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
@@ -629,17 +821,16 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_sve(simsimd_f32_t const* a, simsimd_f32_t co
|
|
|
629
821
|
simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
|
|
630
822
|
simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
|
|
631
823
|
simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
|
|
632
|
-
|
|
633
|
-
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
634
|
-
simsimd_f32_t a2_b2_arr[2] = {a2, b2};
|
|
635
|
-
float32x2_t a2_b2 = vld1_f32(a2_b2_arr);
|
|
636
|
-
a2_b2 = vrsqrte_f32(a2_b2);
|
|
637
|
-
vst1_f32(a2_b2_arr, a2_b2);
|
|
638
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
824
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
639
825
|
}
|
|
640
826
|
|
|
641
|
-
SIMSIMD_PUBLIC void
|
|
642
|
-
|
|
827
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
828
|
+
simsimd_distance_t *result) {
|
|
829
|
+
simsimd_l2sq_f64_sve(a, b, n, result);
|
|
830
|
+
*result = _simsimd_sqrt_f64_neon(*result);
|
|
831
|
+
}
|
|
832
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
833
|
+
simsimd_distance_t *result) {
|
|
643
834
|
simsimd_size_t i = 0;
|
|
644
835
|
svfloat64_t d2_vec = svdupq_n_f64(0.0, 0.0);
|
|
645
836
|
do {
|
|
@@ -654,8 +845,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f64_sve(simsimd_f64_t const* a, simsimd_f64_t c
|
|
|
654
845
|
*result = d2;
|
|
655
846
|
}
|
|
656
847
|
|
|
657
|
-
SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const*
|
|
658
|
-
simsimd_distance_t*
|
|
848
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
849
|
+
simsimd_distance_t *result) {
|
|
659
850
|
simsimd_size_t i = 0;
|
|
660
851
|
svfloat64_t ab_vec = svdupq_n_f64(0.0, 0.0);
|
|
661
852
|
svfloat64_t a2_vec = svdupq_n_f64(0.0, 0.0);
|
|
@@ -673,28 +864,29 @@ SIMSIMD_PUBLIC void simsimd_cos_f64_sve(simsimd_f64_t const* a, simsimd_f64_t co
|
|
|
673
864
|
simsimd_f64_t ab = svaddv_f64(svptrue_b32(), ab_vec);
|
|
674
865
|
simsimd_f64_t a2 = svaddv_f64(svptrue_b32(), a2_vec);
|
|
675
866
|
simsimd_f64_t b2 = svaddv_f64(svptrue_b32(), b2_vec);
|
|
676
|
-
|
|
677
|
-
// Avoid `simsimd_approximate_inverse_square_root` on Arm NEON
|
|
678
|
-
simsimd_f64_t a2_b2_arr[2] = {a2, b2};
|
|
679
|
-
float64x2_t a2_b2 = vld1q_f64(a2_b2_arr);
|
|
680
|
-
a2_b2 = vrsqrteq_f64(a2_b2);
|
|
681
|
-
vst1q_f64(a2_b2_arr, a2_b2);
|
|
682
|
-
*result = ab != 0 ? 1 - ab * a2_b2_arr[0] * a2_b2_arr[1] : 1;
|
|
867
|
+
*result = _simsimd_cos_normalize_f64_neon(ab, a2, b2);
|
|
683
868
|
}
|
|
684
869
|
|
|
685
870
|
#pragma clang attribute pop
|
|
686
871
|
#pragma GCC pop_options
|
|
872
|
+
#endif // SIMSIMD_TARGET_SVE
|
|
687
873
|
|
|
874
|
+
#if SIMSIMD_TARGET_SVE_F16
|
|
688
875
|
#pragma GCC push_options
|
|
689
|
-
#pragma GCC target("+sve+fp16")
|
|
690
|
-
#pragma clang attribute push(__attribute__((target("+sve+fp16"))), apply_to = function)
|
|
876
|
+
#pragma GCC target("arch=armv8.2-a+sve+fp16")
|
|
877
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
|
|
691
878
|
|
|
692
|
-
SIMSIMD_PUBLIC void
|
|
693
|
-
|
|
879
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sve(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
880
|
+
simsimd_distance_t *result) {
|
|
881
|
+
simsimd_l2sq_f16_sve(a, b, n, result);
|
|
882
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
883
|
+
}
|
|
884
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
|
|
885
|
+
simsimd_distance_t *result) {
|
|
694
886
|
simsimd_size_t i = 0;
|
|
695
887
|
svfloat16_t d2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
696
|
-
simsimd_f16_for_arm_simd_t const*
|
|
697
|
-
simsimd_f16_for_arm_simd_t const*
|
|
888
|
+
simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
|
|
889
|
+
simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
|
|
698
890
|
do {
|
|
699
891
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
700
892
|
svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
|
|
@@ -707,14 +899,14 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f16_sve(simsimd_f16_t const* a_enum, simsimd_f1
|
|
|
707
899
|
*result = d2_f16;
|
|
708
900
|
}
|
|
709
901
|
|
|
710
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const*
|
|
711
|
-
simsimd_distance_t*
|
|
902
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const *a_enum, simsimd_f16_t const *b_enum, simsimd_size_t n,
|
|
903
|
+
simsimd_distance_t *result) {
|
|
712
904
|
simsimd_size_t i = 0;
|
|
713
905
|
svfloat16_t ab_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
714
906
|
svfloat16_t a2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
715
907
|
svfloat16_t b2_vec = svdupq_n_f16(0, 0, 0, 0, 0, 0, 0, 0);
|
|
716
|
-
simsimd_f16_for_arm_simd_t const*
|
|
717
|
-
simsimd_f16_for_arm_simd_t const*
|
|
908
|
+
simsimd_f16_for_arm_simd_t const *a = (simsimd_f16_for_arm_simd_t const *)(a_enum);
|
|
909
|
+
simsimd_f16_for_arm_simd_t const *b = (simsimd_f16_for_arm_simd_t const *)(b_enum);
|
|
718
910
|
do {
|
|
719
911
|
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
720
912
|
svfloat16_t a_vec = svld1_f16(pg_vec, a + i);
|
|
@@ -728,324 +920,467 @@ SIMSIMD_PUBLIC void simsimd_cos_f16_sve(simsimd_f16_t const* a_enum, simsimd_f16
|
|
|
728
920
|
simsimd_f16_for_arm_simd_t ab = svaddv_f16(svptrue_b16(), ab_vec);
|
|
729
921
|
simsimd_f16_for_arm_simd_t a2 = svaddv_f16(svptrue_b16(), a2_vec);
|
|
730
922
|
simsimd_f16_for_arm_simd_t b2 = svaddv_f16(svptrue_b16(), b2_vec);
|
|
923
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
924
|
+
}
|
|
731
925
|
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
926
|
+
#pragma clang attribute pop
|
|
927
|
+
#pragma GCC pop_options
|
|
928
|
+
#endif // SIMSIMD_TARGET_SVE_F16
|
|
929
|
+
|
|
930
|
+
#if SIMSIMD_TARGET_SVE_BF16
|
|
931
|
+
#pragma GCC push_options
|
|
932
|
+
#pragma GCC target("arch=armv8.2-a+sve+bf16")
|
|
933
|
+
#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
|
|
934
|
+
|
|
935
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_sve(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
936
|
+
simsimd_distance_t *result) {
|
|
937
|
+
simsimd_l2sq_bf16_sve(a, b, n, result);
|
|
938
|
+
*result = _simsimd_sqrt_f32_neon(*result);
|
|
939
|
+
}
|
|
940
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
|
|
941
|
+
simsimd_distance_t *result) {
|
|
942
|
+
simsimd_size_t i = 0;
|
|
943
|
+
svfloat32_t d2_low_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
944
|
+
svfloat32_t d2_high_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
945
|
+
simsimd_u16_t const *a = (simsimd_u16_t const *)(a_enum);
|
|
946
|
+
simsimd_u16_t const *b = (simsimd_u16_t const *)(b_enum);
|
|
947
|
+
do {
|
|
948
|
+
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
949
|
+
svuint16_t a_vec = svld1_u16(pg_vec, a + i);
|
|
950
|
+
svuint16_t b_vec = svld1_u16(pg_vec, b + i);
|
|
951
|
+
|
|
952
|
+
// There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
|
|
953
|
+
svbool_t pg_low_vec = svwhilelt_b32((unsigned int)(i), (unsigned int)n);
|
|
954
|
+
svbool_t pg_high_vec = svwhilelt_b32((unsigned int)(i + svcnth() / 2), (unsigned int)n);
|
|
955
|
+
svfloat32_t a_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(a_vec), 16));
|
|
956
|
+
svfloat32_t a_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(a_vec), 16));
|
|
957
|
+
svfloat32_t b_low_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_low_vec, svunpklo_u32(b_vec), 16));
|
|
958
|
+
svfloat32_t b_high_vec = svreinterpret_f32_u32(svlsl_n_u32_x(pg_high_vec, svunpkhi_u32(b_vec), 16));
|
|
959
|
+
|
|
960
|
+
svfloat32_t a_minus_b_low_vec = svsub_f32_x(pg_low_vec, a_low_vec, b_low_vec);
|
|
961
|
+
svfloat32_t a_minus_b_high_vec = svsub_f32_x(pg_high_vec, a_high_vec, b_high_vec);
|
|
962
|
+
d2_low_vec = svmla_f32_x(pg_vec, d2_low_vec, a_minus_b_low_vec, a_minus_b_low_vec);
|
|
963
|
+
d2_high_vec = svmla_f32_x(pg_vec, d2_high_vec, a_minus_b_low_vec, a_minus_b_low_vec);
|
|
964
|
+
i += svcnth();
|
|
965
|
+
} while (i < n);
|
|
966
|
+
simsimd_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_vec) + svaddv_f32(svptrue_b32(), d2_high_vec);
|
|
967
|
+
*result = d2;
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_sve(simsimd_bf16_t const *a_enum, simsimd_bf16_t const *b_enum, simsimd_size_t n,
|
|
971
|
+
simsimd_distance_t *result) {
|
|
972
|
+
simsimd_size_t i = 0;
|
|
973
|
+
svfloat32_t ab_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
974
|
+
svfloat32_t a2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
975
|
+
svfloat32_t b2_vec = svdupq_n_f32(0.f, 0.f, 0.f, 0.f);
|
|
976
|
+
simsimd_bf16_for_arm_simd_t const *a = (simsimd_bf16_for_arm_simd_t const *)(a_enum);
|
|
977
|
+
simsimd_bf16_for_arm_simd_t const *b = (simsimd_bf16_for_arm_simd_t const *)(b_enum);
|
|
978
|
+
do {
|
|
979
|
+
svbool_t pg_vec = svwhilelt_b16((unsigned int)i, (unsigned int)n);
|
|
980
|
+
svbfloat16_t a_vec = svld1_bf16(pg_vec, a + i);
|
|
981
|
+
svbfloat16_t b_vec = svld1_bf16(pg_vec, b + i);
|
|
982
|
+
ab_vec = svbfdot_f32(ab_vec, a_vec, b_vec);
|
|
983
|
+
a2_vec = svbfdot_f32(a2_vec, a_vec, a_vec);
|
|
984
|
+
b2_vec = svbfdot_f32(b2_vec, b_vec, b_vec);
|
|
985
|
+
i += svcnth();
|
|
986
|
+
} while (i < n);
|
|
987
|
+
|
|
988
|
+
simsimd_f32_t ab = svaddv_f32(svptrue_b32(), ab_vec);
|
|
989
|
+
simsimd_f32_t a2 = svaddv_f32(svptrue_b32(), a2_vec);
|
|
990
|
+
simsimd_f32_t b2 = svaddv_f32(svptrue_b32(), b2_vec);
|
|
991
|
+
*result = _simsimd_cos_normalize_f32_neon(ab, a2, b2);
|
|
738
992
|
}
|
|
739
993
|
|
|
740
994
|
#pragma clang attribute pop
|
|
741
995
|
#pragma GCC pop_options
|
|
742
|
-
#endif //
|
|
743
|
-
#endif //
|
|
996
|
+
#endif // SIMSIMD_TARGET_SVE_BF16
|
|
997
|
+
#endif // _SIMSIMD_TARGET_ARM
|
|
998
|
+
|
|
999
|
+
#if _SIMSIMD_TARGET_X86
|
|
1000
|
+
#if SIMSIMD_TARGET_HASWELL
|
|
1001
|
+
#pragma GCC push_options
|
|
1002
|
+
#pragma GCC target("avx2")
|
|
1003
|
+
#pragma clang attribute push(__attribute__((target("avx2"))), apply_to = function)
|
|
1004
|
+
|
|
1005
|
+
SIMSIMD_INTERNAL simsimd_f32_t _simsimd_sqrt_f32_haswell(simsimd_f32_t x) {
|
|
1006
|
+
return _mm_cvtss_f32(_mm_sqrt_ps(_mm_set_ss(x)));
|
|
1007
|
+
}
|
|
1008
|
+
SIMSIMD_INTERNAL simsimd_f64_t _simsimd_sqrt_f64_haswell(simsimd_f64_t x) {
|
|
1009
|
+
return _mm_cvtsd_f64(_mm_sqrt_pd(_mm_set_sd(x)));
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_haswell(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
1013
|
+
simsimd_f64_t b2) {
|
|
1014
|
+
|
|
1015
|
+
// If both vectors have magnitude 0, the distance is 0.
|
|
1016
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
1017
|
+
// If any one of the vectors is 0, the square root of the product is 0,
|
|
1018
|
+
// the division is illformed, and the result is 1.
|
|
1019
|
+
else if (ab == 0)
|
|
1020
|
+
return 1;
|
|
1021
|
+
// We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
|
|
1022
|
+
// https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
|
|
1023
|
+
// The latency of the native instruction is 4 cycles and it's broadly supported.
|
|
1024
|
+
// For single-precision floats it has a maximum relative error of 1.5*2^-12.
|
|
1025
|
+
// Higher precision isn't implemented on older CPUs. See `_simsimd_cos_normalize_f64_skylake` for that.
|
|
1026
|
+
__m128d squares = _mm_set_pd(a2, b2);
|
|
1027
|
+
__m128d rsqrts = _mm_cvtps_pd(_mm_rsqrt_ps(_mm_cvtpd_ps(squares)));
|
|
1028
|
+
// Newton-Raphson iteration for reciprocal square root:
|
|
1029
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
1030
|
+
rsqrts = _mm_add_pd( //
|
|
1031
|
+
_mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
|
|
1032
|
+
_mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
|
|
1033
|
+
|
|
1034
|
+
simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
|
|
1035
|
+
simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
|
|
1036
|
+
simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
|
|
1037
|
+
return result > 0 ? result : 0;
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f32_haswell(simsimd_f32_t ab, simsimd_f32_t a2,
|
|
1041
|
+
simsimd_f32_t b2) {
|
|
744
1042
|
|
|
745
|
-
|
|
1043
|
+
// If both vectors have magnitude 0, the distance is 0.
|
|
1044
|
+
if (a2 == 0.0f && b2 == 0.0f) return 0.0f;
|
|
1045
|
+
// If any one of the vectors is 0, the square root of the product is 0,
|
|
1046
|
+
// the division is illformed, and the result is 1.
|
|
1047
|
+
else if (ab == 0.0f)
|
|
1048
|
+
return 1.0f;
|
|
1049
|
+
|
|
1050
|
+
// Load the squares into an __m128 register for single-precision floating-point operations
|
|
1051
|
+
__m128 squares = _mm_set_ps(a2, b2, a2, b2); // We replicate to make use of full register
|
|
1052
|
+
|
|
1053
|
+
// Compute the reciprocal square root of the squares using _mm_rsqrt_ps (single-precision)
|
|
1054
|
+
__m128 rsqrts = _mm_rsqrt_ps(squares);
|
|
1055
|
+
|
|
1056
|
+
// Perform one iteration of Newton-Raphson refinement to improve the precision of rsqrt:
|
|
1057
|
+
// Formula: y' = y * (1.5 - 0.5 * x * y * y)
|
|
1058
|
+
__m128 half = _mm_set1_ps(0.5f);
|
|
1059
|
+
__m128 three_halves = _mm_set1_ps(1.5f);
|
|
1060
|
+
rsqrts =
|
|
1061
|
+
_mm_mul_ps(rsqrts, _mm_sub_ps(three_halves, _mm_mul_ps(half, _mm_mul_ps(squares, _mm_mul_ps(rsqrts, rsqrts)))));
|
|
1062
|
+
|
|
1063
|
+
// Extract the reciprocal square roots of a2 and b2 from the __m128 register
|
|
1064
|
+
simsimd_f32_t a2_reciprocal = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1065
|
+
simsimd_f32_t b2_reciprocal = _mm_cvtss_f32(rsqrts);
|
|
1066
|
+
|
|
1067
|
+
// Calculate the cosine distance: 1 - ab * a2_reciprocal * b2_reciprocal
|
|
1068
|
+
simsimd_distance_t result = 1.0f - ab * a2_reciprocal * b2_reciprocal;
|
|
1069
|
+
return result > 0 ? result : 0;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
#pragma clang attribute pop
|
|
1073
|
+
#pragma GCC pop_options
|
|
1074
|
+
#endif // SIMSIMD_TARGET_HASWELL
|
|
1075
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
1076
|
+
|
|
1077
|
+
#if _SIMSIMD_TARGET_X86
|
|
746
1078
|
#if SIMSIMD_TARGET_HASWELL
|
|
747
1079
|
#pragma GCC push_options
|
|
748
1080
|
#pragma GCC target("avx2", "f16c", "fma")
|
|
749
1081
|
#pragma clang attribute push(__attribute__((target("avx2,f16c,fma"))), apply_to = function)
|
|
750
1082
|
|
|
751
|
-
SIMSIMD_PUBLIC void
|
|
752
|
-
|
|
1083
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1084
|
+
simsimd_distance_t *result) {
|
|
1085
|
+
simsimd_l2sq_f16_haswell(a, b, n, result);
|
|
1086
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1087
|
+
}
|
|
1088
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1089
|
+
simsimd_distance_t *result) {
|
|
1090
|
+
__m256 a_vec, b_vec;
|
|
753
1091
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
754
|
-
simsimd_size_t i = 0;
|
|
755
|
-
for (; i + 8 <= n; i += 8) {
|
|
756
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
757
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
758
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
759
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
760
|
-
}
|
|
761
1092
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
__m128i f16_vec;
|
|
768
|
-
simsimd_f16_t f16[8];
|
|
769
|
-
} a_padded_tail, b_padded_tail;
|
|
770
|
-
simsimd_size_t j = 0;
|
|
771
|
-
for (; i < n; ++i, ++j)
|
|
772
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
773
|
-
for (; j < 8; ++j)
|
|
774
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
775
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
776
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
777
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
778
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1093
|
+
simsimd_l2sq_f16_haswell_cycle:
|
|
1094
|
+
if (n < 8) {
|
|
1095
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1096
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1097
|
+
n = 0;
|
|
779
1098
|
}
|
|
1099
|
+
else {
|
|
1100
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1101
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1102
|
+
n -= 8, a += 8, b += 8;
|
|
1103
|
+
}
|
|
1104
|
+
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
1105
|
+
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1106
|
+
if (n) goto simsimd_l2sq_f16_haswell_cycle;
|
|
780
1107
|
|
|
781
|
-
|
|
782
|
-
d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
|
|
783
|
-
d2_vec = _mm256_hadd_ps(d2_vec, d2_vec);
|
|
784
|
-
|
|
785
|
-
simsimd_f32_t f32_result;
|
|
786
|
-
_mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
|
|
787
|
-
*result = f32_result;
|
|
1108
|
+
*result = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
788
1109
|
}
|
|
789
1110
|
|
|
790
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const*
|
|
791
|
-
simsimd_distance_t*
|
|
792
|
-
|
|
1111
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_haswell(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1112
|
+
simsimd_distance_t *result) {
|
|
1113
|
+
__m256 a_vec, b_vec;
|
|
793
1114
|
__m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
|
|
794
|
-
simsimd_size_t i = 0;
|
|
795
|
-
for (; i + 8 <= n; i += 8) {
|
|
796
|
-
__m256 a_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(a + i)));
|
|
797
|
-
__m256 b_vec = _mm256_cvtph_ps(_mm_loadu_si128((__m128i const*)(b + i)));
|
|
798
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
799
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
800
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
801
|
-
}
|
|
802
1115
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
a_padded_tail.f16[j] = a[i], b_padded_tail.f16[j] = b[i];
|
|
814
|
-
for (; j < 8; ++j)
|
|
815
|
-
a_padded_tail.f16[j] = 0, b_padded_tail.f16[j] = 0;
|
|
816
|
-
__m256 a_vec = _mm256_cvtph_ps(a_padded_tail.f16_vec);
|
|
817
|
-
__m256 b_vec = _mm256_cvtph_ps(b_padded_tail.f16_vec);
|
|
818
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
819
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
820
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1116
|
+
simsimd_cos_f16_haswell_cycle:
|
|
1117
|
+
if (n < 8) {
|
|
1118
|
+
a_vec = _simsimd_partial_load_f16x8_haswell(a, n);
|
|
1119
|
+
b_vec = _simsimd_partial_load_f16x8_haswell(b, n);
|
|
1120
|
+
n = 0;
|
|
1121
|
+
}
|
|
1122
|
+
else {
|
|
1123
|
+
a_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)a));
|
|
1124
|
+
b_vec = _mm256_cvtph_ps(_mm_lddqu_si128((__m128i const *)b));
|
|
1125
|
+
n -= 8, a += 8, b += 8;
|
|
821
1126
|
}
|
|
1127
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1128
|
+
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1129
|
+
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1130
|
+
if (n) goto simsimd_cos_f16_haswell_cycle;
|
|
1131
|
+
|
|
1132
|
+
simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1133
|
+
simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
1134
|
+
simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
|
|
1135
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1136
|
+
}
|
|
822
1137
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
1138
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1139
|
+
simsimd_distance_t *result) {
|
|
1140
|
+
simsimd_l2sq_bf16_haswell(a, b, n, result);
|
|
1141
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1142
|
+
}
|
|
1143
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1144
|
+
simsimd_distance_t *result) {
|
|
1145
|
+
__m256 a_vec, b_vec;
|
|
1146
|
+
__m256 d2_vec = _mm256_setzero_ps();
|
|
827
1147
|
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
1148
|
+
simsimd_l2sq_bf16_haswell_cycle:
|
|
1149
|
+
if (n < 8) {
|
|
1150
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
|
|
1151
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
|
|
1152
|
+
n = 0;
|
|
1153
|
+
}
|
|
1154
|
+
else {
|
|
1155
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
|
|
1156
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
|
|
1157
|
+
n -= 8, a += 8, b += 8;
|
|
1158
|
+
}
|
|
1159
|
+
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
1160
|
+
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1161
|
+
if (n) goto simsimd_l2sq_bf16_haswell_cycle;
|
|
831
1162
|
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
1163
|
+
*result = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
1164
|
+
}
|
|
835
1165
|
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
1166
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_haswell(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1167
|
+
simsimd_distance_t *result) {
|
|
1168
|
+
__m256 a_vec, b_vec;
|
|
1169
|
+
__m256 ab_vec = _mm256_setzero_ps(), a2_vec = _mm256_setzero_ps(), b2_vec = _mm256_setzero_ps();
|
|
840
1170
|
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1171
|
+
simsimd_cos_bf16_haswell_cycle:
|
|
1172
|
+
if (n < 8) {
|
|
1173
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(a, n));
|
|
1174
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_simsimd_partial_load_bf16x8_haswell(b, n));
|
|
1175
|
+
n = 0;
|
|
1176
|
+
}
|
|
1177
|
+
else {
|
|
1178
|
+
a_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)a));
|
|
1179
|
+
b_vec = _simsimd_bf16x8_to_f32x8_haswell(_mm_lddqu_si128((__m128i const *)b));
|
|
1180
|
+
n -= 8, a += 8, b += 8;
|
|
1181
|
+
}
|
|
1182
|
+
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1183
|
+
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1184
|
+
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1185
|
+
if (n) goto simsimd_cos_bf16_haswell_cycle;
|
|
1186
|
+
|
|
1187
|
+
simsimd_f32_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1188
|
+
simsimd_f32_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
1189
|
+
simsimd_f32_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
|
|
1190
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
847
1191
|
}
|
|
848
1192
|
|
|
849
|
-
SIMSIMD_PUBLIC void
|
|
850
|
-
|
|
851
|
-
|
|
1193
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1194
|
+
simsimd_distance_t *result) {
|
|
1195
|
+
simsimd_l2sq_i8_haswell(a, b, n, result);
|
|
1196
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1197
|
+
}
|
|
1198
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1199
|
+
simsimd_distance_t *result) {
|
|
1200
|
+
|
|
1201
|
+
__m256i d2_i32_low_vec = _mm256_setzero_si256();
|
|
1202
|
+
__m256i d2_i32_high_vec = _mm256_setzero_si256();
|
|
1203
|
+
|
|
852
1204
|
simsimd_size_t i = 0;
|
|
853
|
-
for (; i +
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
1205
|
+
for (; i + 32 <= n; i += 32) {
|
|
1206
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1207
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1208
|
+
|
|
1209
|
+
// Sign extend `i8` to `i16`
|
|
1210
|
+
__m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a_i8_vec));
|
|
1211
|
+
__m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
|
|
1212
|
+
__m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_i8_vec));
|
|
1213
|
+
__m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
|
|
1214
|
+
|
|
1215
|
+
// Subtract
|
|
1216
|
+
// After this we will be squaring the values. The sign will be dropped
|
|
1217
|
+
// and each difference will be in the range [0, 255].
|
|
1218
|
+
__m256i d_i16_low_vec = _mm256_sub_epi16(a_i16_low_vec, b_i16_low_vec);
|
|
1219
|
+
__m256i d_i16_high_vec = _mm256_sub_epi16(a_i16_high_vec, b_i16_high_vec);
|
|
1220
|
+
|
|
1221
|
+
// Accumulate into `i32` vectors
|
|
1222
|
+
d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
|
|
1223
|
+
d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
|
|
862
1224
|
}
|
|
863
1225
|
|
|
864
|
-
//
|
|
865
|
-
|
|
866
|
-
// loads and stores with vectorized math.
|
|
867
|
-
if (i < n) {
|
|
868
|
-
union {
|
|
869
|
-
__m128i bf16_vec;
|
|
870
|
-
simsimd_bf16_t bf16[8];
|
|
871
|
-
} a_padded_tail, b_padded_tail;
|
|
872
|
-
simsimd_size_t j = 0;
|
|
873
|
-
for (; i < n; ++i, ++j)
|
|
874
|
-
a_padded_tail.bf16[j] = a[i], b_padded_tail.bf16[j] = b[i];
|
|
875
|
-
for (; j < 8; ++j)
|
|
876
|
-
a_padded_tail.bf16[j] = 0, b_padded_tail.bf16[j] = 0;
|
|
877
|
-
__m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
|
|
878
|
-
__m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
|
|
879
|
-
__m256 d_vec = _mm256_sub_ps(a_vec, b_vec);
|
|
880
|
-
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
881
|
-
}
|
|
1226
|
+
// Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
|
|
1227
|
+
int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
|
|
882
1228
|
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
1229
|
+
// Take care of the tail:
|
|
1230
|
+
for (; i < n; ++i) {
|
|
1231
|
+
int n = (int)(a[i]) - b[i];
|
|
1232
|
+
d2 += n * n;
|
|
1233
|
+
}
|
|
886
1234
|
|
|
887
|
-
|
|
888
|
-
_mm_store_ss(&f32_result, _mm256_castps256_ps128(d2_vec));
|
|
889
|
-
*result = f32_result;
|
|
1235
|
+
*result = (simsimd_f64_t)d2;
|
|
890
1236
|
}
|
|
891
1237
|
|
|
892
|
-
SIMSIMD_PUBLIC void
|
|
893
|
-
|
|
1238
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1239
|
+
simsimd_distance_t *result) {
|
|
894
1240
|
|
|
895
|
-
|
|
1241
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1242
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1243
|
+
__m256i a2_i32_low_vec = _mm256_setzero_si256();
|
|
1244
|
+
__m256i a2_i32_high_vec = _mm256_setzero_si256();
|
|
1245
|
+
__m256i b2_i32_low_vec = _mm256_setzero_si256();
|
|
1246
|
+
__m256i b2_i32_high_vec = _mm256_setzero_si256();
|
|
1247
|
+
|
|
1248
|
+
// AVX2 has no instructions for 8-bit signed integer dot-products,
|
|
1249
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1250
|
+
// So we need to normalize the first vector to its absolute value,
|
|
1251
|
+
// and shift the product sign into the second vector.
|
|
1252
|
+
//
|
|
1253
|
+
// __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
|
|
1254
|
+
// __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
|
|
1255
|
+
// __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
|
|
1256
|
+
//
|
|
1257
|
+
// The problem with this approach, however, is the `-128` value in the second vector.
|
|
1258
|
+
// Flipping it's sign will do nothing, and the result will be incorrect.
|
|
1259
|
+
// This can easily lead to noticeable numerical errors in the final result.
|
|
896
1260
|
simsimd_size_t i = 0;
|
|
897
|
-
for (; i +
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
__m256 a_vec =
|
|
901
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(a + i))), 16));
|
|
902
|
-
__m256 b_vec =
|
|
903
|
-
_mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i const*)(b + i))), 16));
|
|
904
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
905
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
906
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
907
|
-
}
|
|
1261
|
+
for (; i + 32 <= n; i += 32) {
|
|
1262
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1263
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
908
1264
|
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
__m256 a_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(a_padded_tail.bf16_vec), 16));
|
|
923
|
-
__m256 b_vec = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(b_padded_tail.bf16_vec), 16));
|
|
924
|
-
ab_vec = _mm256_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
925
|
-
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
926
|
-
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1265
|
+
// Unpack `int8` to `int16`
|
|
1266
|
+
__m256i a_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 0));
|
|
1267
|
+
__m256i a_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_i8_vec, 1));
|
|
1268
|
+
__m256i b_i16_low_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 0));
|
|
1269
|
+
__m256i b_i16_high_vec = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_i8_vec, 1));
|
|
1270
|
+
|
|
1271
|
+
// Multiply and accumulate as `int16`, accumulate products as `int32`:
|
|
1272
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1273
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1274
|
+
a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
|
|
1275
|
+
a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
|
|
1276
|
+
b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
|
|
1277
|
+
b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
|
|
927
1278
|
}
|
|
928
1279
|
|
|
929
|
-
//
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
a2_vec = _mm256_add_ps(_mm256_permute2f128_ps(a2_vec, a2_vec, 1), a2_vec);
|
|
935
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
936
|
-
a2_vec = _mm256_hadd_ps(a2_vec, a2_vec);
|
|
937
|
-
|
|
938
|
-
b2_vec = _mm256_add_ps(_mm256_permute2f128_ps(b2_vec, b2_vec, 1), b2_vec);
|
|
939
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
940
|
-
b2_vec = _mm256_hadd_ps(b2_vec, b2_vec);
|
|
1280
|
+
// Further reduce to a single sum for each vector
|
|
1281
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1282
|
+
int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
|
|
1283
|
+
int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
|
|
941
1284
|
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1285
|
+
// Take care of the tail:
|
|
1286
|
+
for (; i < n; ++i) {
|
|
1287
|
+
int ai = a[i], bi = b[i];
|
|
1288
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1289
|
+
}
|
|
946
1290
|
|
|
947
|
-
|
|
948
|
-
__m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
949
|
-
__m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
950
|
-
__m128 result_vec = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Multiply the reciprocal square roots
|
|
951
|
-
result_vec = _mm_mul_ss(result_vec, _mm_set_ss((float)ab)); // Multiply by ab
|
|
952
|
-
*result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
|
|
1291
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
953
1292
|
}
|
|
954
1293
|
|
|
955
|
-
SIMSIMD_PUBLIC void
|
|
956
|
-
|
|
1294
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1295
|
+
simsimd_distance_t *result) {
|
|
1296
|
+
simsimd_l2sq_u8_haswell(a, b, n, result);
|
|
1297
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1298
|
+
}
|
|
1299
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1300
|
+
simsimd_distance_t *result) {
|
|
957
1301
|
|
|
958
|
-
__m256i
|
|
959
|
-
__m256i
|
|
1302
|
+
__m256i d2_i32_low_vec = _mm256_setzero_si256();
|
|
1303
|
+
__m256i d2_i32_high_vec = _mm256_setzero_si256();
|
|
1304
|
+
__m256i const zeros_vec = _mm256_setzero_si256();
|
|
960
1305
|
|
|
961
1306
|
simsimd_size_t i = 0;
|
|
962
1307
|
for (; i + 32 <= n; i += 32) {
|
|
963
|
-
__m256i
|
|
964
|
-
__m256i
|
|
1308
|
+
__m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1309
|
+
__m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
965
1310
|
|
|
966
|
-
//
|
|
967
|
-
__m256i
|
|
968
|
-
__m256i a_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a_vec, 1));
|
|
969
|
-
__m256i b_low = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b_vec));
|
|
970
|
-
__m256i b_high = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b_vec, 1));
|
|
1311
|
+
// Substracting unsigned vectors in AVX2 is done by saturating subtraction:
|
|
1312
|
+
__m256i d_u8_vec = _mm256_or_si256(_mm256_subs_epu8(a_u8_vec, b_u8_vec), _mm256_subs_epu8(b_u8_vec, a_u8_vec));
|
|
971
1313
|
|
|
972
|
-
//
|
|
973
|
-
|
|
974
|
-
__m256i
|
|
975
|
-
__m256i
|
|
976
|
-
__m256i d2_high_part = _mm256_madd_epi16(d_high, d_high);
|
|
1314
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1315
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1316
|
+
__m256i d_i16_low_vec = _mm256_unpacklo_epi8(d_u8_vec, zeros_vec);
|
|
1317
|
+
__m256i d_i16_high_vec = _mm256_unpackhi_epi8(d_u8_vec, zeros_vec);
|
|
977
1318
|
|
|
978
|
-
//
|
|
979
|
-
|
|
980
|
-
|
|
1319
|
+
// Multiply and accumulate at `int16` level, accumulate at `int32` level:
|
|
1320
|
+
d2_i32_low_vec = _mm256_add_epi32(d2_i32_low_vec, _mm256_madd_epi16(d_i16_low_vec, d_i16_low_vec));
|
|
1321
|
+
d2_i32_high_vec = _mm256_add_epi32(d2_i32_high_vec, _mm256_madd_epi16(d_i16_high_vec, d_i16_high_vec));
|
|
981
1322
|
}
|
|
982
1323
|
|
|
983
|
-
// Accumulate the 32-bit integers from `
|
|
984
|
-
|
|
985
|
-
__m128i d2_sum = _mm_add_epi32(_mm256_extracti128_si256(d2_vec, 0), _mm256_extracti128_si256(d2_vec, 1));
|
|
986
|
-
d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
|
|
987
|
-
d2_sum = _mm_hadd_epi32(d2_sum, d2_sum);
|
|
988
|
-
int d2 = _mm_extract_epi32(d2_sum, 0);
|
|
1324
|
+
// Accumulate the 32-bit integers from `d2_i32_high_vec` and `d2_i32_low_vec`
|
|
1325
|
+
int d2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
|
|
989
1326
|
|
|
990
1327
|
// Take care of the tail:
|
|
991
1328
|
for (; i < n; ++i) {
|
|
992
|
-
int n = a[i] - b[i];
|
|
1329
|
+
int n = (int)(a[i]) - b[i];
|
|
993
1330
|
d2 += n * n;
|
|
994
1331
|
}
|
|
995
1332
|
|
|
996
1333
|
*result = (simsimd_f64_t)d2;
|
|
997
1334
|
}
|
|
998
1335
|
|
|
999
|
-
SIMSIMD_PUBLIC void
|
|
1000
|
-
simsimd_distance_t*
|
|
1001
|
-
|
|
1002
|
-
__m256i
|
|
1003
|
-
__m256i
|
|
1004
|
-
__m256i
|
|
1005
|
-
__m256i
|
|
1006
|
-
__m256i
|
|
1007
|
-
__m256i
|
|
1008
|
-
|
|
1336
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_haswell(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1337
|
+
simsimd_distance_t *result) {
|
|
1338
|
+
|
|
1339
|
+
__m256i ab_i32_low_vec = _mm256_setzero_si256();
|
|
1340
|
+
__m256i ab_i32_high_vec = _mm256_setzero_si256();
|
|
1341
|
+
__m256i a2_i32_low_vec = _mm256_setzero_si256();
|
|
1342
|
+
__m256i a2_i32_high_vec = _mm256_setzero_si256();
|
|
1343
|
+
__m256i b2_i32_low_vec = _mm256_setzero_si256();
|
|
1344
|
+
__m256i b2_i32_high_vec = _mm256_setzero_si256();
|
|
1345
|
+
__m256i const zeros_vec = _mm256_setzero_si256();
|
|
1346
|
+
|
|
1347
|
+
// AVX2 has no instructions for 8-bit signed integer dot-products,
|
|
1348
|
+
// but it has a weird instruction for mixed signed-unsigned 8-bit dot-product.
|
|
1349
|
+
// So we need to normalize the first vector to its absolute value,
|
|
1350
|
+
// and shift the product sign into the second vector.
|
|
1351
|
+
//
|
|
1352
|
+
// __m256i a_i8_abs_vec = _mm256_abs_epi8(a_i8_vec);
|
|
1353
|
+
// __m256i b_i8_flipped_vec = _mm256_sign_epi8(b_i8_vec, a_i8_vec);
|
|
1354
|
+
// __m256i ab_i16_vec = _mm256_maddubs_epi16(a_i8_abs_vec, b_i8_flipped_vec);
|
|
1355
|
+
//
|
|
1356
|
+
// The problem with this approach, however, is the `-128` value in the second vector.
|
|
1357
|
+
// Flipping it's sign will do nothing, and the result will be incorrect.
|
|
1358
|
+
// This can easily lead to noticeable numerical errors in the final result.
|
|
1009
1359
|
simsimd_size_t i = 0;
|
|
1010
1360
|
for (; i + 32 <= n; i += 32) {
|
|
1011
|
-
__m256i
|
|
1012
|
-
__m256i
|
|
1361
|
+
__m256i a_u8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
1362
|
+
__m256i b_u8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
1013
1363
|
|
|
1014
|
-
//
|
|
1015
|
-
|
|
1016
|
-
__m256i
|
|
1017
|
-
__m256i
|
|
1018
|
-
__m256i
|
|
1364
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
1365
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
1366
|
+
__m256i a_i16_low_vec = _mm256_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
1367
|
+
__m256i a_i16_high_vec = _mm256_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
1368
|
+
__m256i b_i16_low_vec = _mm256_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
1369
|
+
__m256i b_i16_high_vec = _mm256_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
1019
1370
|
|
|
1020
1371
|
// Multiply and accumulate as `int16`, accumulate products as `int32`
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
}
|
|
1028
|
-
|
|
1029
|
-
// Horizontal sum across the 256-bit register
|
|
1030
|
-
__m256i ab_vec = _mm256_add_epi32(ab_low_vec, ab_high_vec);
|
|
1031
|
-
__m128i ab_sum = _mm_add_epi32(_mm256_extracti128_si256(ab_vec, 0), _mm256_extracti128_si256(ab_vec, 1));
|
|
1032
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1033
|
-
ab_sum = _mm_hadd_epi32(ab_sum, ab_sum);
|
|
1034
|
-
|
|
1035
|
-
__m256i a2_vec = _mm256_add_epi32(a2_low_vec, a2_high_vec);
|
|
1036
|
-
__m128i a2_sum = _mm_add_epi32(_mm256_extracti128_si256(a2_vec, 0), _mm256_extracti128_si256(a2_vec, 1));
|
|
1037
|
-
a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
|
|
1038
|
-
a2_sum = _mm_hadd_epi32(a2_sum, a2_sum);
|
|
1039
|
-
|
|
1040
|
-
__m256i b2_vec = _mm256_add_epi32(b2_low_vec, b2_high_vec);
|
|
1041
|
-
__m128i b2_sum = _mm_add_epi32(_mm256_extracti128_si256(b2_vec, 0), _mm256_extracti128_si256(b2_vec, 1));
|
|
1042
|
-
b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
|
|
1043
|
-
b2_sum = _mm_hadd_epi32(b2_sum, b2_sum);
|
|
1372
|
+
ab_i32_low_vec = _mm256_add_epi32(ab_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, b_i16_low_vec));
|
|
1373
|
+
ab_i32_high_vec = _mm256_add_epi32(ab_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, b_i16_high_vec));
|
|
1374
|
+
a2_i32_low_vec = _mm256_add_epi32(a2_i32_low_vec, _mm256_madd_epi16(a_i16_low_vec, a_i16_low_vec));
|
|
1375
|
+
a2_i32_high_vec = _mm256_add_epi32(a2_i32_high_vec, _mm256_madd_epi16(a_i16_high_vec, a_i16_high_vec));
|
|
1376
|
+
b2_i32_low_vec = _mm256_add_epi32(b2_i32_low_vec, _mm256_madd_epi16(b_i16_low_vec, b_i16_low_vec));
|
|
1377
|
+
b2_i32_high_vec = _mm256_add_epi32(b2_i32_high_vec, _mm256_madd_epi16(b_i16_high_vec, b_i16_high_vec));
|
|
1378
|
+
}
|
|
1044
1379
|
|
|
1045
1380
|
// Further reduce to a single sum for each vector
|
|
1046
|
-
int ab =
|
|
1047
|
-
int a2 =
|
|
1048
|
-
int b2 =
|
|
1381
|
+
int ab = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
1382
|
+
int a2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
|
|
1383
|
+
int b2 = _simsimd_reduce_i32x8_haswell(_mm256_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
|
|
1049
1384
|
|
|
1050
1385
|
// Take care of the tail:
|
|
1051
1386
|
for (; i < n; ++i) {
|
|
@@ -1053,18 +1388,16 @@ SIMSIMD_PUBLIC void simsimd_cos_i8_haswell(simsimd_i8_t const* a, simsimd_i8_t c
|
|
|
1053
1388
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1054
1389
|
}
|
|
1055
1390
|
|
|
1056
|
-
|
|
1057
|
-
__m128 a2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)a2));
|
|
1058
|
-
__m128 b2_sqrt_recip = _mm_rsqrt_ss(_mm_set_ss((float)b2));
|
|
1059
|
-
|
|
1060
|
-
// Compute cosine similarity: ab / sqrt(a2 * b2)
|
|
1061
|
-
__m128 denom = _mm_mul_ss(a2_sqrt_recip, b2_sqrt_recip); // Reciprocal of sqrt(a2 * b2)
|
|
1062
|
-
__m128 result_vec = _mm_mul_ss(_mm_set_ss((float)ab), denom); // ab * reciprocal of sqrt(a2 * b2)
|
|
1063
|
-
*result = ab != 0 ? 1 - _mm_cvtss_f32(result_vec) : 1; // Extract the final result
|
|
1391
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1064
1392
|
}
|
|
1065
1393
|
|
|
1066
|
-
SIMSIMD_PUBLIC void
|
|
1067
|
-
|
|
1394
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1395
|
+
simsimd_distance_t *result) {
|
|
1396
|
+
simsimd_l2sq_f32_haswell(a, b, n, result);
|
|
1397
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1398
|
+
}
|
|
1399
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1400
|
+
simsimd_distance_t *result) {
|
|
1068
1401
|
|
|
1069
1402
|
__m256 d2_vec = _mm256_setzero_ps();
|
|
1070
1403
|
simsimd_size_t i = 0;
|
|
@@ -1075,7 +1408,7 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
1075
1408
|
d2_vec = _mm256_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1076
1409
|
}
|
|
1077
1410
|
|
|
1078
|
-
simsimd_f64_t d2 =
|
|
1411
|
+
simsimd_f64_t d2 = _simsimd_reduce_f32x8_haswell(d2_vec);
|
|
1079
1412
|
for (; i < n; ++i) {
|
|
1080
1413
|
float d = a[i] - b[i];
|
|
1081
1414
|
d2 += d * d;
|
|
@@ -1084,8 +1417,8 @@ SIMSIMD_PUBLIC void simsimd_l2sq_f32_haswell(simsimd_f32_t const* a, simsimd_f32
|
|
|
1084
1417
|
*result = d2;
|
|
1085
1418
|
}
|
|
1086
1419
|
|
|
1087
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const*
|
|
1088
|
-
simsimd_distance_t*
|
|
1420
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1421
|
+
simsimd_distance_t *result) {
|
|
1089
1422
|
|
|
1090
1423
|
__m256 ab_vec = _mm256_setzero_ps();
|
|
1091
1424
|
__m256 a2_vec = _mm256_setzero_ps();
|
|
@@ -1098,22 +1431,66 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
1098
1431
|
a2_vec = _mm256_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1099
1432
|
b2_vec = _mm256_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1100
1433
|
}
|
|
1101
|
-
|
|
1102
|
-
simsimd_f64_t
|
|
1103
|
-
simsimd_f64_t
|
|
1434
|
+
|
|
1435
|
+
simsimd_f64_t ab = _simsimd_reduce_f32x8_haswell(ab_vec);
|
|
1436
|
+
simsimd_f64_t a2 = _simsimd_reduce_f32x8_haswell(a2_vec);
|
|
1437
|
+
simsimd_f64_t b2 = _simsimd_reduce_f32x8_haswell(b2_vec);
|
|
1104
1438
|
for (; i < n; ++i) {
|
|
1105
1439
|
float ai = a[i], bi = b[i];
|
|
1106
1440
|
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1107
1441
|
}
|
|
1442
|
+
*result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
|
|
1443
|
+
}
|
|
1108
1444
|
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1445
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1446
|
+
simsimd_distance_t *result) {
|
|
1447
|
+
simsimd_l2sq_f64_haswell(a, b, n, result);
|
|
1448
|
+
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1449
|
+
}
|
|
1450
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1451
|
+
simsimd_distance_t *result) {
|
|
1452
|
+
|
|
1453
|
+
__m256d d2_vec = _mm256_setzero_pd();
|
|
1454
|
+
simsimd_size_t i = 0;
|
|
1455
|
+
for (; i + 4 <= n; i += 4) {
|
|
1456
|
+
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
1457
|
+
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
1458
|
+
__m256d d_vec = _mm256_sub_pd(a_vec, b_vec);
|
|
1459
|
+
d2_vec = _mm256_fmadd_pd(d_vec, d_vec, d2_vec);
|
|
1460
|
+
}
|
|
1112
1461
|
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1462
|
+
simsimd_f64_t d2 = _simsimd_reduce_f64x4_haswell(d2_vec);
|
|
1463
|
+
for (; i < n; ++i) {
|
|
1464
|
+
simsimd_f64_t d = a[i] - b[i];
|
|
1465
|
+
d2 += d * d;
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
*result = d2;
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_haswell(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1472
|
+
simsimd_distance_t *result) {
|
|
1473
|
+
|
|
1474
|
+
__m256d ab_vec = _mm256_setzero_pd();
|
|
1475
|
+
__m256d a2_vec = _mm256_setzero_pd();
|
|
1476
|
+
__m256d b2_vec = _mm256_setzero_pd();
|
|
1477
|
+
simsimd_size_t i = 0;
|
|
1478
|
+
for (; i + 4 <= n; i += 4) {
|
|
1479
|
+
__m256d a_vec = _mm256_loadu_pd(a + i);
|
|
1480
|
+
__m256d b_vec = _mm256_loadu_pd(b + i);
|
|
1481
|
+
ab_vec = _mm256_fmadd_pd(a_vec, b_vec, ab_vec);
|
|
1482
|
+
a2_vec = _mm256_fmadd_pd(a_vec, a_vec, a2_vec);
|
|
1483
|
+
b2_vec = _mm256_fmadd_pd(b_vec, b_vec, b2_vec);
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
simsimd_f64_t ab = _simsimd_reduce_f64x4_haswell(ab_vec);
|
|
1487
|
+
simsimd_f64_t a2 = _simsimd_reduce_f64x4_haswell(a2_vec);
|
|
1488
|
+
simsimd_f64_t b2 = _simsimd_reduce_f64x4_haswell(b2_vec);
|
|
1489
|
+
for (; i < n; ++i) {
|
|
1490
|
+
simsimd_f64_t ai = a[i], bi = b[i];
|
|
1491
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
1492
|
+
}
|
|
1493
|
+
*result = _simsimd_cos_normalize_f64_haswell(ab, a2, b2);
|
|
1117
1494
|
}
|
|
1118
1495
|
|
|
1119
1496
|
#pragma clang attribute pop
|
|
@@ -1122,11 +1499,16 @@ SIMSIMD_PUBLIC void simsimd_cos_f32_haswell(simsimd_f32_t const* a, simsimd_f32_
|
|
|
1122
1499
|
|
|
1123
1500
|
#if SIMSIMD_TARGET_SKYLAKE
|
|
1124
1501
|
#pragma GCC push_options
|
|
1125
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2")
|
|
1126
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2"))), apply_to = function)
|
|
1502
|
+
#pragma GCC target("avx2", "avx512f", "avx512bw", "avx512vl", "bmi2")
|
|
1503
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512bw,avx512vl,bmi2"))), apply_to = function)
|
|
1127
1504
|
|
|
1128
|
-
SIMSIMD_PUBLIC void
|
|
1129
|
-
|
|
1505
|
+
SIMSIMD_PUBLIC void simsimd_l2_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1506
|
+
simsimd_distance_t *result) {
|
|
1507
|
+
simsimd_l2sq_f32_skylake(a, b, n, result);
|
|
1508
|
+
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1509
|
+
}
|
|
1510
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1511
|
+
simsimd_distance_t *result) {
|
|
1130
1512
|
__m512 d2_vec = _mm512_setzero();
|
|
1131
1513
|
__m512 a_vec, b_vec;
|
|
1132
1514
|
|
|
@@ -1136,21 +1518,64 @@ simsimd_l2sq_f32_skylake_cycle:
|
|
|
1136
1518
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1137
1519
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1138
1520
|
n = 0;
|
|
1139
|
-
}
|
|
1521
|
+
}
|
|
1522
|
+
else {
|
|
1140
1523
|
a_vec = _mm512_loadu_ps(a);
|
|
1141
1524
|
b_vec = _mm512_loadu_ps(b);
|
|
1142
1525
|
a += 16, b += 16, n -= 16;
|
|
1143
1526
|
}
|
|
1144
1527
|
__m512 d_vec = _mm512_sub_ps(a_vec, b_vec);
|
|
1145
1528
|
d2_vec = _mm512_fmadd_ps(d_vec, d_vec, d2_vec);
|
|
1146
|
-
if (n)
|
|
1147
|
-
|
|
1529
|
+
if (n) goto simsimd_l2sq_f32_skylake_cycle;
|
|
1530
|
+
|
|
1531
|
+
*result = _simsimd_reduce_f32x16_skylake(d2_vec);
|
|
1532
|
+
}
|
|
1148
1533
|
|
|
1149
|
-
|
|
1534
|
+
SIMSIMD_INTERNAL simsimd_distance_t _simsimd_cos_normalize_f64_skylake(simsimd_f64_t ab, simsimd_f64_t a2,
|
|
1535
|
+
simsimd_f64_t b2) {
|
|
1536
|
+
|
|
1537
|
+
// If both vectors have magnitude 0, the distance is 0.
|
|
1538
|
+
if (a2 == 0 && b2 == 0) return 0;
|
|
1539
|
+
// If any one of the vectors is 0, the square root of the product is 0,
|
|
1540
|
+
// the division is illformed, and the result is 1.
|
|
1541
|
+
else if (ab == 0)
|
|
1542
|
+
return 1;
|
|
1543
|
+
|
|
1544
|
+
// We want to avoid the `simsimd_approximate_inverse_square_root` due to high latency:
|
|
1545
|
+
// https://web.archive.org/web/20210208132927/http://assemblyrequired.crashworks.org/timing-square-root/
|
|
1546
|
+
// The maximum relative error for this approximation is less than 2^-14, which is 6x lower than
|
|
1547
|
+
// for single-precision floats in the `_simsimd_cos_normalize_f64_haswell` implementation.
|
|
1548
|
+
// Mysteriously, MSVC has no `_mm_rsqrt14_pd` intrinsic, but has it's masked variants,
|
|
1549
|
+
// so let's use `_mm_maskz_rsqrt14_pd(0xFF, ...)` instead.
|
|
1550
|
+
__m128d squares = _mm_set_pd(a2, b2);
|
|
1551
|
+
__m128d rsqrts = _mm_maskz_rsqrt14_pd(0xFF, squares);
|
|
1552
|
+
|
|
1553
|
+
// Let's implement a single Newton-Raphson iteration to refine the result.
|
|
1554
|
+
// This is how it affects downstream applications:
|
|
1555
|
+
//
|
|
1556
|
+
// +--------+------+----------+---------------------+---------------------+---------------------+
|
|
1557
|
+
// | Metric | NDim | DType | Baseline Error | Old SimSIMD Error | New SimSIMD Error |
|
|
1558
|
+
// +--------+------+----------+---------------------+---------------------+---------------------+
|
|
1559
|
+
// | cosine | 1536 | bfloat16 | 1.89e-08 ± 1.59e-08 | 3.07e-07 ± 3.09e-07 | 3.53e-09 ± 2.70e-09 |
|
|
1560
|
+
// | cosine | 1536 | float16 | 1.67e-02 ± 1.44e-02 | 2.68e-05 ± 1.95e-05 | 2.02e-05 ± 1.39e-05 |
|
|
1561
|
+
// | cosine | 1536 | float32 | 2.21e-08 ± 1.65e-08 | 3.47e-07 ± 3.49e-07 | 3.77e-09 ± 2.84e-09 |
|
|
1562
|
+
// | cosine | 1536 | float64 | 0.00e+00 ± 0.00e+00 | 3.80e-07 ± 4.50e-07 | 1.35e-11 ± 1.85e-11 |
|
|
1563
|
+
// | cosine | 1536 | int8 | 0.00e+00 ± 0.00e+00 | 4.60e-04 ± 3.36e-04 | 4.20e-04 ± 4.88e-04 |
|
|
1564
|
+
// +--------+------+----------+---------------------+---------------------+---------------------+
|
|
1565
|
+
//
|
|
1566
|
+
// https://en.wikipedia.org/wiki/Newton%27s_method
|
|
1567
|
+
rsqrts = _mm_add_pd( //
|
|
1568
|
+
_mm_mul_pd(_mm_set1_pd(1.5), rsqrts),
|
|
1569
|
+
_mm_mul_pd(_mm_mul_pd(_mm_mul_pd(squares, _mm_set1_pd(-0.5)), rsqrts), _mm_mul_pd(rsqrts, rsqrts)));
|
|
1570
|
+
|
|
1571
|
+
simsimd_f64_t a2_reciprocal = _mm_cvtsd_f64(_mm_unpackhi_pd(rsqrts, rsqrts));
|
|
1572
|
+
simsimd_f64_t b2_reciprocal = _mm_cvtsd_f64(rsqrts);
|
|
1573
|
+
simsimd_distance_t result = 1 - ab * a2_reciprocal * b2_reciprocal;
|
|
1574
|
+
return result > 0 ? result : 0;
|
|
1150
1575
|
}
|
|
1151
1576
|
|
|
1152
|
-
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const*
|
|
1153
|
-
simsimd_distance_t*
|
|
1577
|
+
SIMSIMD_PUBLIC void simsimd_cos_f32_skylake(simsimd_f32_t const *a, simsimd_f32_t const *b, simsimd_size_t n,
|
|
1578
|
+
simsimd_distance_t *result) {
|
|
1154
1579
|
__m512 ab_vec = _mm512_setzero();
|
|
1155
1580
|
__m512 a2_vec = _mm512_setzero();
|
|
1156
1581
|
__m512 b2_vec = _mm512_setzero();
|
|
@@ -1162,7 +1587,8 @@ simsimd_cos_f32_skylake_cycle:
|
|
|
1162
1587
|
a_vec = _mm512_maskz_loadu_ps(mask, a);
|
|
1163
1588
|
b_vec = _mm512_maskz_loadu_ps(mask, b);
|
|
1164
1589
|
n = 0;
|
|
1165
|
-
}
|
|
1590
|
+
}
|
|
1591
|
+
else {
|
|
1166
1592
|
a_vec = _mm512_loadu_ps(a);
|
|
1167
1593
|
b_vec = _mm512_loadu_ps(b);
|
|
1168
1594
|
a += 16, b += 16, n -= 16;
|
|
@@ -1170,24 +1596,21 @@ simsimd_cos_f32_skylake_cycle:
|
|
|
1170
1596
|
ab_vec = _mm512_fmadd_ps(a_vec, b_vec, ab_vec);
|
|
1171
1597
|
a2_vec = _mm512_fmadd_ps(a_vec, a_vec, a2_vec);
|
|
1172
1598
|
b2_vec = _mm512_fmadd_ps(b_vec, b_vec, b2_vec);
|
|
1173
|
-
if (n)
|
|
1174
|
-
goto simsimd_cos_f32_skylake_cycle;
|
|
1599
|
+
if (n) goto simsimd_cos_f32_skylake_cycle;
|
|
1175
1600
|
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1181
|
-
// Mysteriously, MSVC has no `_mm_rsqrt14_ps` intrinsic, but has it's masked variants,
|
|
1182
|
-
// so let's use `_mm_maskz_rsqrt14_ps(0xFF, ...)` instead.
|
|
1183
|
-
__m128 rsqrts = _mm_maskz_rsqrt14_ps(0xFF, _mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1184
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1185
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1186
|
-
*result = 1 - ab * rsqrt_a2 * rsqrt_b2;
|
|
1601
|
+
simsimd_f64_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1602
|
+
simsimd_f64_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
|
|
1603
|
+
simsimd_f64_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
|
|
1604
|
+
*result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
|
|
1187
1605
|
}
|
|
1188
1606
|
|
|
1189
|
-
SIMSIMD_PUBLIC void
|
|
1190
|
-
|
|
1607
|
+
SIMSIMD_PUBLIC void simsimd_l2_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1608
|
+
simsimd_distance_t *result) {
|
|
1609
|
+
simsimd_l2sq_f64_skylake(a, b, n, result);
|
|
1610
|
+
*result = _simsimd_sqrt_f64_haswell(*result);
|
|
1611
|
+
}
|
|
1612
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1613
|
+
simsimd_distance_t *result) {
|
|
1191
1614
|
__m512d d2_vec = _mm512_setzero_pd();
|
|
1192
1615
|
__m512d a_vec, b_vec;
|
|
1193
1616
|
|
|
@@ -1197,21 +1620,21 @@ simsimd_l2sq_f64_skylake_cycle:
|
|
|
1197
1620
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1198
1621
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1199
1622
|
n = 0;
|
|
1200
|
-
}
|
|
1623
|
+
}
|
|
1624
|
+
else {
|
|
1201
1625
|
a_vec = _mm512_loadu_pd(a);
|
|
1202
1626
|
b_vec = _mm512_loadu_pd(b);
|
|
1203
1627
|
a += 8, b += 8, n -= 8;
|
|
1204
1628
|
}
|
|
1205
1629
|
__m512d d_vec = _mm512_sub_pd(a_vec, b_vec);
|
|
1206
1630
|
d2_vec = _mm512_fmadd_pd(d_vec, d_vec, d2_vec);
|
|
1207
|
-
if (n)
|
|
1208
|
-
goto simsimd_l2sq_f64_skylake_cycle;
|
|
1631
|
+
if (n) goto simsimd_l2sq_f64_skylake_cycle;
|
|
1209
1632
|
|
|
1210
1633
|
*result = _mm512_reduce_add_pd(d2_vec);
|
|
1211
1634
|
}
|
|
1212
1635
|
|
|
1213
|
-
SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const*
|
|
1214
|
-
simsimd_distance_t*
|
|
1636
|
+
SIMSIMD_PUBLIC void simsimd_cos_f64_skylake(simsimd_f64_t const *a, simsimd_f64_t const *b, simsimd_size_t n,
|
|
1637
|
+
simsimd_distance_t *result) {
|
|
1215
1638
|
__m512d ab_vec = _mm512_setzero_pd();
|
|
1216
1639
|
__m512d a2_vec = _mm512_setzero_pd();
|
|
1217
1640
|
__m512d b2_vec = _mm512_setzero_pd();
|
|
@@ -1223,7 +1646,8 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1223
1646
|
a_vec = _mm512_maskz_loadu_pd(mask, a);
|
|
1224
1647
|
b_vec = _mm512_maskz_loadu_pd(mask, b);
|
|
1225
1648
|
n = 0;
|
|
1226
|
-
}
|
|
1649
|
+
}
|
|
1650
|
+
else {
|
|
1227
1651
|
a_vec = _mm512_loadu_pd(a);
|
|
1228
1652
|
b_vec = _mm512_loadu_pd(b);
|
|
1229
1653
|
a += 8, b += 8, n -= 8;
|
|
@@ -1231,20 +1655,12 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1231
1655
|
ab_vec = _mm512_fmadd_pd(a_vec, b_vec, ab_vec);
|
|
1232
1656
|
a2_vec = _mm512_fmadd_pd(a_vec, a_vec, a2_vec);
|
|
1233
1657
|
b2_vec = _mm512_fmadd_pd(b_vec, b_vec, b2_vec);
|
|
1234
|
-
if (n)
|
|
1235
|
-
goto simsimd_cos_f64_skylake_cycle;
|
|
1236
|
-
|
|
1237
|
-
simsimd_f32_t ab = (simsimd_f32_t)_mm512_reduce_add_pd(ab_vec);
|
|
1238
|
-
simsimd_f32_t a2 = (simsimd_f32_t)_mm512_reduce_add_pd(a2_vec);
|
|
1239
|
-
simsimd_f32_t b2 = (simsimd_f32_t)_mm512_reduce_add_pd(b2_vec);
|
|
1658
|
+
if (n) goto simsimd_cos_f64_skylake_cycle;
|
|
1240
1659
|
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1246
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1247
|
-
*result = 1 - ab * rsqrt_a2 * rsqrt_b2;
|
|
1660
|
+
simsimd_f64_t ab = _mm512_reduce_add_pd(ab_vec);
|
|
1661
|
+
simsimd_f64_t a2 = _mm512_reduce_add_pd(a2_vec);
|
|
1662
|
+
simsimd_f64_t b2 = _mm512_reduce_add_pd(b2_vec);
|
|
1663
|
+
*result = _simsimd_cos_normalize_f64_skylake(ab, a2, b2);
|
|
1248
1664
|
}
|
|
1249
1665
|
|
|
1250
1666
|
#pragma clang attribute pop
|
|
@@ -1253,15 +1669,71 @@ simsimd_cos_f64_skylake_cycle:
|
|
|
1253
1669
|
|
|
1254
1670
|
#if SIMSIMD_TARGET_GENOA
|
|
1255
1671
|
#pragma GCC push_options
|
|
1256
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1257
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))),
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1672
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512bf16")
|
|
1673
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512bf16"))), \
|
|
1674
|
+
apply_to = function)
|
|
1675
|
+
|
|
1676
|
+
SIMSIMD_INTERNAL __m512i _simsimd_substract_bf16x32_genoa(__m512i a_i16, __m512i b_i16) {
|
|
1677
|
+
|
|
1678
|
+
union {
|
|
1679
|
+
__m512 fvec;
|
|
1680
|
+
__m512i ivec;
|
|
1681
|
+
simsimd_f32_t f32[16];
|
|
1682
|
+
simsimd_u16_t u16[32];
|
|
1683
|
+
simsimd_bf16_t bf16[32];
|
|
1684
|
+
} d_odd, d_even, d, a_f32_even, b_f32_even, d_f32_even, a_f32_odd, b_f32_odd, d_f32_odd, a, b;
|
|
1685
|
+
a.ivec = a_i16;
|
|
1686
|
+
b.ivec = b_i16;
|
|
1687
|
+
|
|
1688
|
+
// There are several approaches to perform subtraction in `bf16`. The first one is:
|
|
1689
|
+
//
|
|
1690
|
+
// Perform a couple of casts - each is a bitshift. To convert `bf16` to `f32`,
|
|
1691
|
+
// expand it to 32-bit integers, then shift the bits by 16 to the left.
|
|
1692
|
+
// Then subtract as floats, and shift back. During expansion, we will double the space,
|
|
1693
|
+
// and should use separate registers for top and bottom halves.
|
|
1694
|
+
// Some compilers don't have `_mm512_extracti32x8_epi32`, so we use `_mm512_extracti64x4_epi64`:
|
|
1695
|
+
//
|
|
1696
|
+
// a_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
|
|
1697
|
+
// _mm512_cvtepu16_epi32(_mm512_castsi512_si256(a_i16)), 16));
|
|
1698
|
+
// b_f32_bot.fvec = _mm512_castsi512_ps(_mm512_slli_epi32(
|
|
1699
|
+
// _mm512_cvtepu16_epi32(_mm512_castsi512_si256(b_i16)), 16));
|
|
1700
|
+
// a_f32_top.fvec =_mm512_castsi512_ps(
|
|
1701
|
+
// _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16, 1)), 16));
|
|
1702
|
+
// b_f32_top.fvec =_mm512_castsi512_ps(
|
|
1703
|
+
// _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16, 1)), 16));
|
|
1704
|
+
// d_f32_top.fvec = _mm512_sub_ps(a_f32_top.fvec, b_f32_top.fvec);
|
|
1705
|
+
// d_f32_bot.fvec = _mm512_sub_ps(a_f32_bot.fvec, b_f32_bot.fvec);
|
|
1706
|
+
// d.ivec = _mm512_castsi256_si512(_mm512_cvtepi32_epi16(
|
|
1707
|
+
// _mm512_srli_epi32(_mm512_castps_si512(d_f32_bot.fvec), 16)));
|
|
1708
|
+
// d.ivec = _mm512_inserti64x4(d.ivec, _mm512_cvtepi32_epi16(
|
|
1709
|
+
// _mm512_srli_epi32(_mm512_castps_si512(d_f32_top.fvec), 16)), 1);
|
|
1710
|
+
//
|
|
1711
|
+
// Instead of using multple shifts and an insertion, we can achieve similar result with fewer expensive
|
|
1712
|
+
// calls to `_mm512_permutex2var_epi16`, or a cheap `_mm512_mask_shuffle_epi8` and blend:
|
|
1713
|
+
//
|
|
1714
|
+
a_f32_odd.ivec = _mm512_and_si512(a_i16, _mm512_set1_epi32(0xFFFF0000));
|
|
1715
|
+
a_f32_even.ivec = _mm512_slli_epi32(a_i16, 16);
|
|
1716
|
+
b_f32_odd.ivec = _mm512_and_si512(b_i16, _mm512_set1_epi32(0xFFFF0000));
|
|
1717
|
+
b_f32_even.ivec = _mm512_slli_epi32(b_i16, 16);
|
|
1718
|
+
|
|
1719
|
+
d_f32_odd.fvec = _mm512_sub_ps(a_f32_odd.fvec, b_f32_odd.fvec);
|
|
1720
|
+
d_f32_even.fvec = _mm512_sub_ps(a_f32_even.fvec, b_f32_even.fvec);
|
|
1721
|
+
|
|
1722
|
+
d_f32_even.ivec = _mm512_srli_epi32(d_f32_even.ivec, 16);
|
|
1723
|
+
d.ivec = _mm512_mask_blend_epi16(0x55555555, d_f32_odd.ivec, d_f32_even.ivec);
|
|
1724
|
+
|
|
1725
|
+
return d.ivec;
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
SIMSIMD_PUBLIC void simsimd_l2_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1729
|
+
simsimd_distance_t *result) {
|
|
1730
|
+
simsimd_l2sq_bf16_genoa(a, b, n, result);
|
|
1731
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1732
|
+
}
|
|
1733
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1734
|
+
simsimd_distance_t *result) {
|
|
1735
|
+
__m512 d2_vec = _mm512_setzero_ps();
|
|
1736
|
+
__m512i a_i16_vec, b_i16_vec, d_i16_vec;
|
|
1265
1737
|
|
|
1266
1738
|
simsimd_l2sq_bf16_genoa_cycle:
|
|
1267
1739
|
if (n < 32) {
|
|
@@ -1269,44 +1741,21 @@ simsimd_l2sq_bf16_genoa_cycle:
|
|
|
1269
1741
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1270
1742
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1271
1743
|
n = 0;
|
|
1272
|
-
}
|
|
1744
|
+
}
|
|
1745
|
+
else {
|
|
1273
1746
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1274
1747
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1275
1748
|
a += 32, b += 32, n -= 32;
|
|
1276
1749
|
}
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
// Some compilers don't have `_mm512_extracti32x8_epi32`, so we need to use `_mm512_extracti64x4_epi64`
|
|
1288
|
-
a_f32_top_vec =
|
|
1289
|
-
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(a_i16_vec, 1)), 16));
|
|
1290
|
-
b_f32_top_vec =
|
|
1291
|
-
_mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64(b_i16_vec, 1)), 16));
|
|
1292
|
-
|
|
1293
|
-
// Subtract and cast back
|
|
1294
|
-
d_top_vec = _mm512_sub_ps(a_f32_top_vec, b_f32_top_vec);
|
|
1295
|
-
d_bot_vec = _mm512_sub_ps(a_f32_bot_vec, b_f32_bot_vec);
|
|
1296
|
-
d_top_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_top_vec), 16));
|
|
1297
|
-
d_bot_vec = _mm512_castsi512_ps(_mm512_srli_epi32(_mm512_castps_si512(d_bot_vec), 16));
|
|
1298
|
-
|
|
1299
|
-
// Square and accumulate
|
|
1300
|
-
d2_top_vec = _mm512_dpbf16_ps(d2_top_vec, (__m512bh)(d_top_vec), (__m512bh)(d_top_vec));
|
|
1301
|
-
d2_bot_vec = _mm512_dpbf16_ps(d2_bot_vec, (__m512bh)(d_bot_vec), (__m512bh)(d_bot_vec));
|
|
1302
|
-
if (n)
|
|
1303
|
-
goto simsimd_l2sq_bf16_genoa_cycle;
|
|
1304
|
-
|
|
1305
|
-
*result = _mm512_reduce_add_ps(d2_top_vec) + _mm512_reduce_add_ps(d2_bot_vec);
|
|
1306
|
-
}
|
|
1307
|
-
|
|
1308
|
-
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const* a, simsimd_bf16_t const* b, simsimd_size_t n,
|
|
1309
|
-
simsimd_distance_t* result) {
|
|
1750
|
+
d_i16_vec = _simsimd_substract_bf16x32_genoa(a_i16_vec, b_i16_vec);
|
|
1751
|
+
d2_vec = _mm512_dpbf16_ps(d2_vec, (__m512bh)(d_i16_vec), (__m512bh)(d_i16_vec));
|
|
1752
|
+
if (n) goto simsimd_l2sq_bf16_genoa_cycle;
|
|
1753
|
+
|
|
1754
|
+
*result = _simsimd_reduce_f32x16_skylake(d2_vec);
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1757
|
+
SIMSIMD_PUBLIC void simsimd_cos_bf16_genoa(simsimd_bf16_t const *a, simsimd_bf16_t const *b, simsimd_size_t n,
|
|
1758
|
+
simsimd_distance_t *result) {
|
|
1310
1759
|
__m512 ab_vec = _mm512_setzero_ps();
|
|
1311
1760
|
__m512 a2_vec = _mm512_setzero_ps();
|
|
1312
1761
|
__m512 b2_vec = _mm512_setzero_ps();
|
|
@@ -1318,7 +1767,8 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1318
1767
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1319
1768
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1320
1769
|
n = 0;
|
|
1321
|
-
}
|
|
1770
|
+
}
|
|
1771
|
+
else {
|
|
1322
1772
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1323
1773
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1324
1774
|
a += 32, b += 32, n -= 32;
|
|
@@ -1326,18 +1776,12 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1326
1776
|
ab_vec = _mm512_dpbf16_ps(ab_vec, (__m512bh)(a_i16_vec), (__m512bh)(b_i16_vec));
|
|
1327
1777
|
a2_vec = _mm512_dpbf16_ps(a2_vec, (__m512bh)(a_i16_vec), (__m512bh)(a_i16_vec));
|
|
1328
1778
|
b2_vec = _mm512_dpbf16_ps(b2_vec, (__m512bh)(b_i16_vec), (__m512bh)(b_i16_vec));
|
|
1329
|
-
if (n)
|
|
1330
|
-
goto simsimd_cos_bf16_genoa_cycle;
|
|
1331
|
-
|
|
1332
|
-
simsimd_f32_t ab = _mm512_reduce_add_ps(ab_vec);
|
|
1333
|
-
simsimd_f32_t a2 = _mm512_reduce_add_ps(a2_vec);
|
|
1334
|
-
simsimd_f32_t b2 = _mm512_reduce_add_ps(b2_vec);
|
|
1779
|
+
if (n) goto simsimd_cos_bf16_genoa_cycle;
|
|
1335
1780
|
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
simsimd_f32_t
|
|
1339
|
-
|
|
1340
|
-
*result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
|
|
1781
|
+
simsimd_f32_t ab = _simsimd_reduce_f32x16_skylake(ab_vec);
|
|
1782
|
+
simsimd_f32_t a2 = _simsimd_reduce_f32x16_skylake(a2_vec);
|
|
1783
|
+
simsimd_f32_t b2 = _simsimd_reduce_f32x16_skylake(b2_vec);
|
|
1784
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1341
1785
|
}
|
|
1342
1786
|
|
|
1343
1787
|
#pragma clang attribute pop
|
|
@@ -1346,11 +1790,16 @@ simsimd_cos_bf16_genoa_cycle:
|
|
|
1346
1790
|
|
|
1347
1791
|
#if SIMSIMD_TARGET_SAPPHIRE
|
|
1348
1792
|
#pragma GCC push_options
|
|
1349
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
1350
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
1793
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512fp16")
|
|
1794
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512fp16"))), apply_to = function)
|
|
1351
1795
|
|
|
1352
|
-
SIMSIMD_PUBLIC void
|
|
1353
|
-
|
|
1796
|
+
SIMSIMD_PUBLIC void simsimd_l2_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1797
|
+
simsimd_distance_t *result) {
|
|
1798
|
+
simsimd_l2sq_f16_sapphire(a, b, n, result);
|
|
1799
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1800
|
+
}
|
|
1801
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1802
|
+
simsimd_distance_t *result) {
|
|
1354
1803
|
__m512h d2_vec = _mm512_setzero_ph();
|
|
1355
1804
|
__m512i a_i16_vec, b_i16_vec;
|
|
1356
1805
|
|
|
@@ -1360,21 +1809,21 @@ simsimd_l2sq_f16_sapphire_cycle:
|
|
|
1360
1809
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1361
1810
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1362
1811
|
n = 0;
|
|
1363
|
-
}
|
|
1812
|
+
}
|
|
1813
|
+
else {
|
|
1364
1814
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1365
1815
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1366
1816
|
a += 32, b += 32, n -= 32;
|
|
1367
1817
|
}
|
|
1368
1818
|
__m512h d_vec = _mm512_sub_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec));
|
|
1369
1819
|
d2_vec = _mm512_fmadd_ph(d_vec, d_vec, d2_vec);
|
|
1370
|
-
if (n)
|
|
1371
|
-
goto simsimd_l2sq_f16_sapphire_cycle;
|
|
1820
|
+
if (n) goto simsimd_l2sq_f16_sapphire_cycle;
|
|
1372
1821
|
|
|
1373
1822
|
*result = _mm512_reduce_add_ph(d2_vec);
|
|
1374
1823
|
}
|
|
1375
1824
|
|
|
1376
|
-
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const*
|
|
1377
|
-
simsimd_distance_t*
|
|
1825
|
+
SIMSIMD_PUBLIC void simsimd_cos_f16_sapphire(simsimd_f16_t const *a, simsimd_f16_t const *b, simsimd_size_t n,
|
|
1826
|
+
simsimd_distance_t *result) {
|
|
1378
1827
|
__m512h ab_vec = _mm512_setzero_ph();
|
|
1379
1828
|
__m512h a2_vec = _mm512_setzero_ph();
|
|
1380
1829
|
__m512h b2_vec = _mm512_setzero_ph();
|
|
@@ -1386,7 +1835,8 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1386
1835
|
a_i16_vec = _mm512_maskz_loadu_epi16(mask, a);
|
|
1387
1836
|
b_i16_vec = _mm512_maskz_loadu_epi16(mask, b);
|
|
1388
1837
|
n = 0;
|
|
1389
|
-
}
|
|
1838
|
+
}
|
|
1839
|
+
else {
|
|
1390
1840
|
a_i16_vec = _mm512_loadu_epi16(a);
|
|
1391
1841
|
b_i16_vec = _mm512_loadu_epi16(b);
|
|
1392
1842
|
a += 32, b += 32, n -= 32;
|
|
@@ -1394,18 +1844,12 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1394
1844
|
ab_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(b_i16_vec), ab_vec);
|
|
1395
1845
|
a2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(a_i16_vec), _mm512_castsi512_ph(a_i16_vec), a2_vec);
|
|
1396
1846
|
b2_vec = _mm512_fmadd_ph(_mm512_castsi512_ph(b_i16_vec), _mm512_castsi512_ph(b_i16_vec), b2_vec);
|
|
1397
|
-
if (n)
|
|
1398
|
-
goto simsimd_cos_f16_sapphire_cycle;
|
|
1847
|
+
if (n) goto simsimd_cos_f16_sapphire_cycle;
|
|
1399
1848
|
|
|
1400
1849
|
simsimd_f32_t ab = _mm512_reduce_add_ph(ab_vec);
|
|
1401
1850
|
simsimd_f32_t a2 = _mm512_reduce_add_ph(a2_vec);
|
|
1402
1851
|
simsimd_f32_t b2 = _mm512_reduce_add_ph(b2_vec);
|
|
1403
|
-
|
|
1404
|
-
// Compute the reciprocal square roots of a2 and b2
|
|
1405
|
-
__m128 rsqrts = _mm_rsqrt14_ps(_mm_set_ps(0.f, 0.f, a2 + 1.e-9f, b2 + 1.e-9f));
|
|
1406
|
-
simsimd_f32_t rsqrt_a2 = _mm_cvtss_f32(rsqrts);
|
|
1407
|
-
simsimd_f32_t rsqrt_b2 = _mm_cvtss_f32(_mm_shuffle_ps(rsqrts, rsqrts, _MM_SHUFFLE(0, 0, 0, 1)));
|
|
1408
|
-
*result = ab != 0 ? 1 - ab * rsqrt_a2 * rsqrt_b2 : 1;
|
|
1852
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1409
1853
|
}
|
|
1410
1854
|
|
|
1411
1855
|
#pragma clang attribute pop
|
|
@@ -1414,94 +1858,464 @@ simsimd_cos_f16_sapphire_cycle:
|
|
|
1414
1858
|
|
|
1415
1859
|
#if SIMSIMD_TARGET_ICE
|
|
1416
1860
|
#pragma GCC push_options
|
|
1417
|
-
#pragma GCC target("avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1418
|
-
#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))),
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1861
|
+
#pragma GCC target("avx2", "avx512f", "avx512vl", "bmi2", "avx512bw", "avx512vnni")
|
|
1862
|
+
#pragma clang attribute push(__attribute__((target("avx2,avx512f,avx512vl,bmi2,avx512bw,avx512vnni"))), \
|
|
1863
|
+
apply_to = function)
|
|
1864
|
+
|
|
1865
|
+
SIMSIMD_PUBLIC void simsimd_l2_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1866
|
+
simsimd_distance_t *result) {
|
|
1867
|
+
simsimd_l2sq_i8_ice(a, b, n, result);
|
|
1868
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1869
|
+
}
|
|
1870
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1871
|
+
simsimd_distance_t *result) {
|
|
1872
|
+
__m512i d2_i32_vec = _mm512_setzero_si512();
|
|
1873
|
+
__m512i a_i16_vec, b_i16_vec, d_i16s_vec;
|
|
1424
1874
|
|
|
1425
1875
|
simsimd_l2sq_i8_ice_cycle:
|
|
1426
|
-
if (n < 32) {
|
|
1876
|
+
if (n < 32) { // TODO: Avoid eaarly i16 upcast to step through 64 values at a time
|
|
1427
1877
|
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1428
|
-
|
|
1429
|
-
|
|
1878
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1879
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
1430
1880
|
n = 0;
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
|
|
1881
|
+
}
|
|
1882
|
+
else {
|
|
1883
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
|
|
1884
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
|
|
1434
1885
|
a += 32, b += 32, n -= 32;
|
|
1435
1886
|
}
|
|
1436
|
-
d_i16s_vec = _mm512_sub_epi16(
|
|
1437
|
-
|
|
1438
|
-
if (n)
|
|
1439
|
-
goto simsimd_l2sq_i8_ice_cycle;
|
|
1887
|
+
d_i16s_vec = _mm512_sub_epi16(a_i16_vec, b_i16_vec);
|
|
1888
|
+
d2_i32_vec = _mm512_dpwssd_epi32(d2_i32_vec, d_i16s_vec, d_i16s_vec);
|
|
1889
|
+
if (n) goto simsimd_l2sq_i8_ice_cycle;
|
|
1440
1890
|
|
|
1441
|
-
*result = _mm512_reduce_add_epi32(
|
|
1891
|
+
*result = _mm512_reduce_add_epi32(d2_i32_vec);
|
|
1442
1892
|
}
|
|
1443
1893
|
|
|
1444
|
-
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const*
|
|
1445
|
-
simsimd_distance_t*
|
|
1446
|
-
__m512i ab_low_i32s_vec = _mm512_setzero_si512();
|
|
1447
|
-
__m512i ab_high_i32s_vec = _mm512_setzero_si512();
|
|
1448
|
-
__m512i a2_i32s_vec = _mm512_setzero_si512();
|
|
1449
|
-
__m512i b2_i32s_vec = _mm512_setzero_si512();
|
|
1450
|
-
__m512i a_vec, b_vec;
|
|
1451
|
-
__m512i a_abs_vec, b_abs_vec;
|
|
1894
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_ice(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
1895
|
+
simsimd_distance_t *result) {
|
|
1452
1896
|
|
|
1897
|
+
__m512i ab_i32_vec = _mm512_setzero_si512();
|
|
1898
|
+
__m512i a2_i32_vec = _mm512_setzero_si512();
|
|
1899
|
+
__m512i b2_i32_vec = _mm512_setzero_si512();
|
|
1900
|
+
__m512i a_i16_vec, b_i16_vec;
|
|
1453
1901
|
simsimd_cos_i8_ice_cycle:
|
|
1454
|
-
if (n <
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1902
|
+
if (n < 32) {
|
|
1903
|
+
__mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
|
|
1904
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, a));
|
|
1905
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_maskz_loadu_epi8(mask, b));
|
|
1458
1906
|
n = 0;
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1907
|
+
}
|
|
1908
|
+
else {
|
|
1909
|
+
a_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)a));
|
|
1910
|
+
b_i16_vec = _mm512_cvtepi8_epi16(_mm256_lddqu_si256((__m256i const *)b));
|
|
1911
|
+
a += 32, b += 32, n -= 32;
|
|
1463
1912
|
}
|
|
1464
1913
|
|
|
1465
1914
|
// We can't directly use the `_mm512_dpbusd_epi32` intrinsic everywhere,
|
|
1466
1915
|
// as it's asymmetric with respect to the sign of the input arguments:
|
|
1916
|
+
//
|
|
1467
1917
|
// Signed(ZeroExtend16(a.byte[4*j]) * SignExtend16(b.byte[4*j]))
|
|
1468
|
-
//
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1918
|
+
//
|
|
1919
|
+
// To compute the squares, we could just drop the sign bit of the second argument.
|
|
1920
|
+
// But this would lead to big-big problems on values like `-128`!
|
|
1921
|
+
// For dot-products we don't have the luxury of optimizing the sign bit away.
|
|
1922
|
+
// Assuming this is an approximate kernel (with reciprocal square root approximations)
|
|
1923
|
+
// in the end, we can allow clamping the value to [-127, 127] range.
|
|
1924
|
+
//
|
|
1925
|
+
// On Ice Lake:
|
|
1926
|
+
//
|
|
1927
|
+
// 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can only execute on port 0, with 5 cycle latency.
|
|
1928
|
+
// 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also only execute on port 0, with 5 cycle latency.
|
|
1929
|
+
// 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 5, with 5 cycle latency.
|
|
1930
|
+
//
|
|
1931
|
+
// On Zen4 Genoa:
|
|
1932
|
+
//
|
|
1933
|
+
// 1. `VPDPBUSDS (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 4 cycle latency.
|
|
1934
|
+
// 2. `VPDPWSSDS (ZMM, ZMM, ZMM)` can also execute on ports 0 and 1, with 4 cycle latency.
|
|
1935
|
+
// 3. `VPMADDWD (ZMM, ZMM, ZMM)` can execute on ports 0 and 1, with 3 cycle latency.
|
|
1936
|
+
//
|
|
1937
|
+
// The old solution was complex replied on 1. and 2.:
|
|
1938
|
+
//
|
|
1939
|
+
// a_i8_abs_vec = _mm512_abs_epi8(a_i8_vec);
|
|
1940
|
+
// b_i8_abs_vec = _mm512_abs_epi8(b_i8_vec);
|
|
1941
|
+
// a2_i32_vec = _mm512_dpbusds_epi32(a2_i32_vec, a_i8_abs_vec, a_i8_abs_vec);
|
|
1942
|
+
// b2_i32_vec = _mm512_dpbusds_epi32(b2_i32_vec, b_i8_abs_vec, b_i8_abs_vec);
|
|
1943
|
+
// ab_i32_low_vec = _mm512_dpwssds_epi32( //
|
|
1944
|
+
// ab_i32_low_vec, //
|
|
1945
|
+
// _mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_vec)), //
|
|
1946
|
+
// _mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_vec)));
|
|
1947
|
+
// ab_i32_high_vec = _mm512_dpwssds_epi32( //
|
|
1948
|
+
// ab_i32_high_vec, //
|
|
1949
|
+
// _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_vec, 1)), //
|
|
1950
|
+
// _mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_vec, 1)));
|
|
1951
|
+
//
|
|
1952
|
+
// The new solution is simpler and relies on 3.:
|
|
1953
|
+
ab_i32_vec = _mm512_add_epi32(ab_i32_vec, _mm512_madd_epi16(a_i16_vec, b_i16_vec));
|
|
1954
|
+
a2_i32_vec = _mm512_add_epi32(a2_i32_vec, _mm512_madd_epi16(a_i16_vec, a_i16_vec));
|
|
1955
|
+
b2_i32_vec = _mm512_add_epi32(b2_i32_vec, _mm512_madd_epi16(b_i16_vec, b_i16_vec));
|
|
1956
|
+
if (n) goto simsimd_cos_i8_ice_cycle;
|
|
1957
|
+
|
|
1958
|
+
int ab = _mm512_reduce_add_epi32(ab_i32_vec);
|
|
1959
|
+
int a2 = _mm512_reduce_add_epi32(a2_i32_vec);
|
|
1960
|
+
int b2 = _mm512_reduce_add_epi32(b2_i32_vec);
|
|
1961
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1962
|
+
}
|
|
1963
|
+
SIMSIMD_PUBLIC void simsimd_l2_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1964
|
+
simsimd_distance_t *result) {
|
|
1965
|
+
simsimd_l2sq_u8_ice(a, b, n, result);
|
|
1966
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
1967
|
+
}
|
|
1968
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
1969
|
+
simsimd_distance_t *result) {
|
|
1970
|
+
__m512i d2_i32_low_vec = _mm512_setzero_si512();
|
|
1971
|
+
__m512i d2_i32_high_vec = _mm512_setzero_si512();
|
|
1972
|
+
__m512i const zeros_vec = _mm512_setzero_si512();
|
|
1973
|
+
__m512i d_i16_low_vec, d_i16_high_vec;
|
|
1974
|
+
__m512i a_u8_vec, b_u8_vec, d_u8_vec;
|
|
1975
|
+
|
|
1976
|
+
simsimd_l2sq_u8_ice_cycle:
|
|
1977
|
+
if (n < 64) {
|
|
1978
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
1979
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
1980
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
1981
|
+
n = 0;
|
|
1982
|
+
}
|
|
1983
|
+
else {
|
|
1984
|
+
a_u8_vec = _mm512_loadu_si512(a);
|
|
1985
|
+
b_u8_vec = _mm512_loadu_si512(b);
|
|
1986
|
+
a += 64, b += 64, n -= 64;
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
// Substracting unsigned vectors in AVX-512 is done by saturating subtraction:
|
|
1990
|
+
d_u8_vec = _mm512_or_si512(_mm512_subs_epu8(a_u8_vec, b_u8_vec), _mm512_subs_epu8(b_u8_vec, a_u8_vec));
|
|
1991
|
+
d_i16_low_vec = _mm512_unpacklo_epi8(d_u8_vec, zeros_vec);
|
|
1992
|
+
d_i16_high_vec = _mm512_unpackhi_epi8(d_u8_vec, zeros_vec);
|
|
1993
|
+
|
|
1994
|
+
// Multiply and accumulate at `int16` level, accumulate at `int32` level:
|
|
1995
|
+
d2_i32_low_vec = _mm512_dpwssd_epi32(d2_i32_low_vec, d_i16_low_vec, d_i16_low_vec);
|
|
1996
|
+
d2_i32_high_vec = _mm512_dpwssd_epi32(d2_i32_high_vec, d_i16_high_vec, d_i16_high_vec);
|
|
1997
|
+
if (n) goto simsimd_l2sq_u8_ice_cycle;
|
|
1998
|
+
|
|
1999
|
+
*result = _mm512_reduce_add_epi32(_mm512_add_epi32(d2_i32_low_vec, d2_i32_high_vec));
|
|
2000
|
+
}
|
|
2001
|
+
|
|
2002
|
+
SIMSIMD_PUBLIC void simsimd_cos_u8_ice(simsimd_u8_t const *a, simsimd_u8_t const *b, simsimd_size_t n,
|
|
2003
|
+
simsimd_distance_t *result) {
|
|
2004
|
+
|
|
2005
|
+
__m512i ab_i32_low_vec = _mm512_setzero_si512();
|
|
2006
|
+
__m512i ab_i32_high_vec = _mm512_setzero_si512();
|
|
2007
|
+
__m512i a2_i32_low_vec = _mm512_setzero_si512();
|
|
2008
|
+
__m512i a2_i32_high_vec = _mm512_setzero_si512();
|
|
2009
|
+
__m512i b2_i32_low_vec = _mm512_setzero_si512();
|
|
2010
|
+
__m512i b2_i32_high_vec = _mm512_setzero_si512();
|
|
2011
|
+
__m512i const zeros_vec = _mm512_setzero_si512();
|
|
2012
|
+
__m512i a_i16_low_vec, a_i16_high_vec, b_i16_low_vec, b_i16_high_vec;
|
|
2013
|
+
__m512i a_u8_vec, b_u8_vec;
|
|
2014
|
+
|
|
2015
|
+
simsimd_cos_u8_ice_cycle:
|
|
2016
|
+
if (n < 64) {
|
|
2017
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n);
|
|
2018
|
+
a_u8_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
2019
|
+
b_u8_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
2020
|
+
n = 0;
|
|
2021
|
+
}
|
|
2022
|
+
else {
|
|
2023
|
+
a_u8_vec = _mm512_loadu_si512(a);
|
|
2024
|
+
b_u8_vec = _mm512_loadu_si512(b);
|
|
2025
|
+
a += 64, b += 64, n -= 64;
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
// Upcast `uint8` to `int16`. Unlike the signed version, we can use the unpacking
|
|
2029
|
+
// instructions instead of extracts, as they are much faster and more efficient.
|
|
2030
|
+
a_i16_low_vec = _mm512_unpacklo_epi8(a_u8_vec, zeros_vec);
|
|
2031
|
+
a_i16_high_vec = _mm512_unpackhi_epi8(a_u8_vec, zeros_vec);
|
|
2032
|
+
b_i16_low_vec = _mm512_unpacklo_epi8(b_u8_vec, zeros_vec);
|
|
2033
|
+
b_i16_high_vec = _mm512_unpackhi_epi8(b_u8_vec, zeros_vec);
|
|
2034
|
+
|
|
2035
|
+
// Multiply and accumulate as `int16`, accumulate products as `int32`:
|
|
2036
|
+
ab_i32_low_vec = _mm512_dpwssds_epi32(ab_i32_low_vec, a_i16_low_vec, b_i16_low_vec);
|
|
2037
|
+
ab_i32_high_vec = _mm512_dpwssds_epi32(ab_i32_high_vec, a_i16_high_vec, b_i16_high_vec);
|
|
2038
|
+
a2_i32_low_vec = _mm512_dpwssds_epi32(a2_i32_low_vec, a_i16_low_vec, a_i16_low_vec);
|
|
2039
|
+
a2_i32_high_vec = _mm512_dpwssds_epi32(a2_i32_high_vec, a_i16_high_vec, a_i16_high_vec);
|
|
2040
|
+
b2_i32_low_vec = _mm512_dpwssds_epi32(b2_i32_low_vec, b_i16_low_vec, b_i16_low_vec);
|
|
2041
|
+
b2_i32_high_vec = _mm512_dpwssds_epi32(b2_i32_high_vec, b_i16_high_vec, b_i16_high_vec);
|
|
2042
|
+
if (n) goto simsimd_cos_u8_ice_cycle;
|
|
2043
|
+
|
|
2044
|
+
int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
2045
|
+
int a2 = _mm512_reduce_add_epi32(_mm512_add_epi32(a2_i32_low_vec, a2_i32_high_vec));
|
|
2046
|
+
int b2 = _mm512_reduce_add_epi32(_mm512_add_epi32(b2_i32_low_vec, b2_i32_high_vec));
|
|
2047
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
SIMSIMD_PUBLIC void simsimd_l2_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
|
|
2051
|
+
simsimd_distance_t *result) {
|
|
2052
|
+
simsimd_l2sq_i4x2_ice(a, b, n_words, result);
|
|
2053
|
+
*result = _simsimd_sqrt_f32_haswell(*result);
|
|
2054
|
+
}
|
|
2055
|
+
SIMSIMD_PUBLIC void simsimd_l2sq_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
|
|
2056
|
+
simsimd_distance_t *result) {
|
|
2057
|
+
|
|
2058
|
+
// While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
|
|
2059
|
+
// The absolute difference between two 4-bit integers is at most 15 and it is always a `uint4_t` value!
|
|
2060
|
+
// Moreover, it's square is at most 225, which fits into `uint8_t` and can be computed with a single
|
|
2061
|
+
// lookup table. Accumulating those values is similar to checksumming, a piece of cake for SIMD!
|
|
2062
|
+
__m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
|
|
2063
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2064
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2065
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2066
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
|
|
2067
|
+
__m512i const u4_squares_lookup_vec = _mm512_set_epi8( //
|
|
2068
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2069
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2070
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2071
|
+
(char)225, (char)196, (char)169, (char)144, 121, 100, 81, 64, 49, 36, 25, 16, 9, 4, 1, 0);
|
|
2072
|
+
|
|
2073
|
+
/// The mask used to take the low nibble of each byte.
|
|
2074
|
+
__m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
|
|
2075
|
+
|
|
2076
|
+
// Temporaries:
|
|
2077
|
+
__m512i a_i4x2_vec, b_i4x2_vec;
|
|
2078
|
+
__m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
|
|
2079
|
+
__m512i d_u8_low_vec, d_u8_high_vec; //! Only the low 4 bits are actually used
|
|
2080
|
+
__m512i d2_u8_low_vec, d2_u8_high_vec;
|
|
2081
|
+
__m512i d2_u16_low_vec, d2_u16_high_vec;
|
|
2082
|
+
|
|
2083
|
+
// Accumulators:
|
|
2084
|
+
__m512i d2_u32_vec = _mm512_setzero_si512();
|
|
2085
|
+
|
|
2086
|
+
simsimd_l2sq_i4x2_ice_cycle:
|
|
2087
|
+
if (n_words < 64) {
|
|
2088
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
|
|
2089
|
+
a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
2090
|
+
b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
2091
|
+
n_words = 0;
|
|
2092
|
+
}
|
|
2093
|
+
else {
|
|
2094
|
+
a_i4x2_vec = _mm512_loadu_epi8(a);
|
|
2095
|
+
b_i4x2_vec = _mm512_loadu_epi8(b);
|
|
2096
|
+
a += 64, b += 64, n_words -= 64;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
// Unpack the 4-bit values into 8-bit values with an empty top nibble.
|
|
2100
|
+
a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
|
|
2101
|
+
a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
|
|
2102
|
+
b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
|
|
2103
|
+
b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
|
|
2104
|
+
a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
|
|
2105
|
+
a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
|
|
2106
|
+
b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
|
|
2107
|
+
b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
|
|
2108
|
+
|
|
2109
|
+
// We can implement subtraction with a lookup table, or using `_mm512_sub_epi8`.
|
|
2110
|
+
d_u8_low_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_low_vec, b_i8_low_vec));
|
|
2111
|
+
d_u8_high_vec = _mm512_abs_epi8(_mm512_sub_epi8(a_i8_high_vec, b_i8_high_vec));
|
|
2112
|
+
|
|
2113
|
+
// Now we can use the lookup table to compute the squares of the 4-bit unsigned integers
|
|
2114
|
+
// in the low nibbles of the `d_u8_low_vec` and `d_u8_high_vec` vectors.
|
|
2115
|
+
d2_u8_low_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_low_vec);
|
|
2116
|
+
d2_u8_high_vec = _mm512_shuffle_epi8(u4_squares_lookup_vec, d_u8_high_vec);
|
|
2117
|
+
|
|
2118
|
+
// Aggregating into 16-bit integers, we need to first upcast our 8-bit values to 16 bits.
|
|
2119
|
+
// After that, we will perform one more operation, upcasting further into 32-bit integers.
|
|
2120
|
+
d2_u16_low_vec = //
|
|
2121
|
+
_mm512_add_epi16( //
|
|
2122
|
+
_mm512_unpacklo_epi8(d2_u8_low_vec, _mm512_setzero_si512()),
|
|
2123
|
+
_mm512_unpackhi_epi8(d2_u8_low_vec, _mm512_setzero_si512()));
|
|
2124
|
+
d2_u16_high_vec = //
|
|
2125
|
+
_mm512_add_epi16( //
|
|
2126
|
+
_mm512_unpacklo_epi8(d2_u8_high_vec, _mm512_setzero_si512()),
|
|
2127
|
+
_mm512_unpackhi_epi8(d2_u8_high_vec, _mm512_setzero_si512()));
|
|
2128
|
+
d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_low_vec, _mm512_setzero_si512()));
|
|
2129
|
+
d2_u32_vec = _mm512_add_epi32(d2_u32_vec, _mm512_unpacklo_epi16(d2_u16_high_vec, _mm512_setzero_si512()));
|
|
2130
|
+
if (n_words) goto simsimd_l2sq_i4x2_ice_cycle;
|
|
2131
|
+
|
|
2132
|
+
// Finally, we can reduce the 16-bit integers to 32-bit integers and sum them up.
|
|
2133
|
+
int d2 = _mm512_reduce_add_epi32(d2_u32_vec);
|
|
2134
|
+
*result = d2;
|
|
2135
|
+
}
|
|
2136
|
+
SIMSIMD_PUBLIC void simsimd_cos_i4x2_ice(simsimd_i4x2_t const *a, simsimd_i4x2_t const *b, simsimd_size_t n_words,
|
|
2137
|
+
simsimd_distance_t *result) {
|
|
2138
|
+
|
|
2139
|
+
// We need to compose a lookup table for all the scalar products of 4-bit integers.
|
|
2140
|
+
// While `int8_t` covers the range [-128, 127], `int4_t` covers only [-8, 7].
|
|
2141
|
+
// Practically speaking, the product of two 4-bit signed integers is a 7-bit integer,
|
|
2142
|
+
// as the maximum absolute value of the product is `abs(-8 * -8) == 64`.
|
|
2143
|
+
//
|
|
2144
|
+
// To store 128 possible values of 2^7 bits we only need 128 single-byte scalars,
|
|
2145
|
+
// or just 2x ZMM registers. In that case our lookup will only take `vpermi2b` instruction,
|
|
2146
|
+
// easily inokable with `_mm512_permutex2var_epi8` intrinsic with latency of 6 on Sapphire Rapids.
|
|
2147
|
+
// The problem is converting 2d indices of our symmetric matrix into 1d offsets in the dense array.
|
|
2148
|
+
//
|
|
2149
|
+
// Alternatively, we can take the entire symmetric (16 x 16) matrix of products,
|
|
2150
|
+
// put into 4x ZMM registers, and use it with `_mm512_shuffle_epi8`, remembering
|
|
2151
|
+
// that it can only lookup with 128-bit lanes (16x 8-bit values).
|
|
2152
|
+
// That intrinsic has latency 1, but will need to be repeated and combined with
|
|
2153
|
+
// multiple iterations of `_mm512_shuffle_i64x2` that has latency 3.
|
|
2154
|
+
//
|
|
2155
|
+
// Altenatively, we can get down to 3 cycles per lookup with `vpermb` and `_mm512_permutexvar_epi8` intrinsics.
|
|
2156
|
+
// For that we can split our (16 x 16) matrix into 4x (8 x 8) submatrices, and use 4x ZMM registers.
|
|
2157
|
+
//
|
|
2158
|
+
// Still, all of those solutions are quite heavy compared to two parallel calls to `_mm512_dpbusds_epi32`
|
|
2159
|
+
// for the dot product. But we can still use the `_mm512_permutexvar_epi8` to compute the squares of the
|
|
2160
|
+
// 16 possible `int4_t` values faster.
|
|
2161
|
+
//
|
|
2162
|
+
// Here is how our `int4_t` range looks:
|
|
2163
|
+
//
|
|
2164
|
+
// dec: 0 1 2 3 4 5 6 7 -8 -7 -6 -5 -4 -3 -2 -1
|
|
2165
|
+
// hex: 0 1 2 3 4 5 6 7 8 9 A B C D E F
|
|
2166
|
+
//
|
|
2167
|
+
// Squared:
|
|
2168
|
+
//
|
|
2169
|
+
// dec2: 0 1 4 9 16 25 36 49 64 49 36 25 16 9 4 1
|
|
2170
|
+
// hex2: 0 1 4 9 10 19 24 31 40 31 24 19 10 9 4 1
|
|
2171
|
+
//
|
|
2172
|
+
// Broadcast it to every lane, so that: `square(x) == _mm512_shuffle_epi8(i4_squares_lookup_vec, x)`.
|
|
2173
|
+
__m512i const i4_to_i8_lookup_vec = _mm512_set_epi8( //
|
|
2174
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2175
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2176
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0, //
|
|
2177
|
+
-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0);
|
|
2178
|
+
__m512i const i4_squares_lookup_vec = _mm512_set_epi8( //
|
|
2179
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2180
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2181
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0, //
|
|
2182
|
+
1, 4, 9, 16, 25, 36, 49, 64, 49, 36, 25, 16, 9, 4, 1, 0);
|
|
2183
|
+
|
|
2184
|
+
/// The mask used to take the low nibble of each byte.
|
|
2185
|
+
__m512i const i4_nibble_vec = _mm512_set1_epi8(0x0F);
|
|
2186
|
+
|
|
2187
|
+
// Temporaries:
|
|
2188
|
+
__m512i a_i4x2_vec, b_i4x2_vec;
|
|
2189
|
+
__m512i a_i8_low_vec, a_i8_high_vec, b_i8_low_vec, b_i8_high_vec;
|
|
2190
|
+
__m512i a2_u8_vec, b2_u8_vec;
|
|
2191
|
+
|
|
2192
|
+
// Accumulators:
|
|
2193
|
+
__m512i a2_u16_low_vec = _mm512_setzero_si512();
|
|
2194
|
+
__m512i a2_u16_high_vec = _mm512_setzero_si512();
|
|
2195
|
+
__m512i b2_u16_low_vec = _mm512_setzero_si512();
|
|
2196
|
+
__m512i b2_u16_high_vec = _mm512_setzero_si512();
|
|
2197
|
+
__m512i ab_i32_low_vec = _mm512_setzero_si512();
|
|
2198
|
+
__m512i ab_i32_high_vec = _mm512_setzero_si512();
|
|
2199
|
+
|
|
2200
|
+
simsimd_cos_i4x2_ice_cycle:
|
|
2201
|
+
if (n_words < 64) {
|
|
2202
|
+
__mmask64 mask = (__mmask64)_bzhi_u64(0xFFFFFFFFFFFFFFFF, n_words);
|
|
2203
|
+
a_i4x2_vec = _mm512_maskz_loadu_epi8(mask, a);
|
|
2204
|
+
b_i4x2_vec = _mm512_maskz_loadu_epi8(mask, b);
|
|
2205
|
+
n_words = 0;
|
|
2206
|
+
}
|
|
2207
|
+
else {
|
|
2208
|
+
a_i4x2_vec = _mm512_loadu_epi8(a);
|
|
2209
|
+
b_i4x2_vec = _mm512_loadu_epi8(b);
|
|
2210
|
+
a += 64, b += 64, n_words -= 64;
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
// Unpack the 4-bit values into 8-bit values with an empty top nibble.
|
|
2214
|
+
// For now, they are not really 8-bit integers, as they are not sign-extended.
|
|
2215
|
+
// That part will come later, using the `i4_to_i8_lookup_vec` lookup.
|
|
2216
|
+
a_i8_low_vec = _mm512_and_si512(a_i4x2_vec, i4_nibble_vec);
|
|
2217
|
+
a_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(a_i4x2_vec, 4), i4_nibble_vec);
|
|
2218
|
+
b_i8_low_vec = _mm512_and_si512(b_i4x2_vec, i4_nibble_vec);
|
|
2219
|
+
b_i8_high_vec = _mm512_and_si512(_mm512_srli_epi64(b_i4x2_vec, 4), i4_nibble_vec);
|
|
2220
|
+
|
|
2221
|
+
// Compute the squares of the 4-bit integers.
|
|
2222
|
+
// For symmetry we could have used 4 registers, aka "a2_i8_low_vec", "a2_i8_high_vec", "b2_i8_low_vec",
|
|
2223
|
+
// "b2_i8_high_vec". But the largest square value is just 64, so we can safely aggregate into 8-bit unsigned values.
|
|
2224
|
+
a2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_low_vec),
|
|
2225
|
+
_mm512_shuffle_epi8(i4_squares_lookup_vec, a_i8_high_vec));
|
|
2226
|
+
b2_u8_vec = _mm512_add_epi8(_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_low_vec),
|
|
2227
|
+
_mm512_shuffle_epi8(i4_squares_lookup_vec, b_i8_high_vec));
|
|
2228
|
+
|
|
2229
|
+
// We can safely aggregate into just 16-bit sums without overflow, if the vectors have less than:
|
|
2230
|
+
// (2 scalars / byte) * (64 bytes / register) * (256 non-overflowing 8-bit additions in 16-bit intesgers)
|
|
2231
|
+
// = 32'768 dimensions.
|
|
2232
|
+
//
|
|
2233
|
+
// We use saturated addition here to clearly inform in case of overflow.
|
|
2234
|
+
a2_u16_low_vec = _mm512_adds_epu16(a2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
|
|
2235
|
+
a2_u16_high_vec = _mm512_adds_epu16(a2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
|
|
2236
|
+
b2_u16_low_vec = _mm512_adds_epu16(b2_u16_low_vec, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(a2_u8_vec)));
|
|
2237
|
+
b2_u16_high_vec = _mm512_adds_epu16(b2_u16_high_vec, _mm512_cvtepu8_epi16(_mm512_extracti64x4_epi64(a2_u8_vec, 1)));
|
|
2238
|
+
|
|
2239
|
+
// Time to perform the proper sign extension of the 4-bit integers to 8-bit integers.
|
|
2240
|
+
a_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_low_vec);
|
|
2241
|
+
a_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, a_i8_high_vec);
|
|
2242
|
+
b_i8_low_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_low_vec);
|
|
2243
|
+
b_i8_high_vec = _mm512_shuffle_epi8(i4_to_i8_lookup_vec, b_i8_high_vec);
|
|
1473
2244
|
|
|
1474
2245
|
// The same trick won't work for the primary dot-product, as the signs vector
|
|
1475
|
-
// components may differ significantly. So we have to use two `
|
|
2246
|
+
// components may differ significantly. So we have to use two `_mm512_dpwssds_epi32`
|
|
1476
2247
|
// intrinsics instead, upcasting four chunks to 16-bit integers beforehand!
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
2248
|
+
// Alternatively, we can flip the signs of the second argument and use `_mm512_dpbusds_epi32`,
|
|
2249
|
+
// but it ends up taking more instructions.
|
|
2250
|
+
ab_i32_low_vec = _mm512_dpwssds_epi32( //
|
|
2251
|
+
ab_i32_low_vec, //
|
|
2252
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_low_vec)), //
|
|
2253
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_low_vec)));
|
|
2254
|
+
ab_i32_low_vec = _mm512_dpwssds_epi32( //
|
|
2255
|
+
ab_i32_low_vec, //
|
|
2256
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_low_vec, 1)), //
|
|
2257
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_low_vec, 1)));
|
|
2258
|
+
ab_i32_high_vec = _mm512_dpwssds_epi32( //
|
|
2259
|
+
ab_i32_high_vec, //
|
|
2260
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(a_i8_high_vec)), //
|
|
2261
|
+
_mm512_cvtepi8_epi16(_mm512_castsi512_si256(b_i8_high_vec)));
|
|
2262
|
+
ab_i32_high_vec = _mm512_dpwssds_epi32( //
|
|
2263
|
+
ab_i32_high_vec, //
|
|
2264
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(a_i8_high_vec, 1)), //
|
|
2265
|
+
_mm512_cvtepi8_epi16(_mm512_extracti64x4_epi64(b_i8_high_vec, 1)));
|
|
2266
|
+
if (n_words) goto simsimd_cos_i4x2_ice_cycle;
|
|
2267
|
+
|
|
2268
|
+
int ab = _mm512_reduce_add_epi32(_mm512_add_epi32(ab_i32_low_vec, ab_i32_high_vec));
|
|
2269
|
+
unsigned short a2_u16[32], b2_u16[32];
|
|
2270
|
+
_mm512_storeu_si512(a2_u16, _mm512_add_epi16(a2_u16_low_vec, a2_u16_high_vec));
|
|
2271
|
+
unsigned int a2 = 0, b2 = 0;
|
|
2272
|
+
for (int i = 0; i < 32; ++i) a2 += a2_u16[i], b2 += b2_u16[i];
|
|
2273
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
1499
2274
|
}
|
|
1500
2275
|
|
|
1501
2276
|
#pragma clang attribute pop
|
|
1502
2277
|
#pragma GCC pop_options
|
|
1503
2278
|
#endif // SIMSIMD_TARGET_ICE
|
|
1504
|
-
|
|
2279
|
+
|
|
2280
|
+
#if SIMSIMD_TARGET_SIERRA
|
|
2281
|
+
#pragma GCC push_options
|
|
2282
|
+
#pragma GCC target("avx2", "bmi2", "avx2vnni")
|
|
2283
|
+
#pragma clang attribute push(__attribute__((target("avx2,bmi2,avx2vnni"))), apply_to = function)
|
|
2284
|
+
|
|
2285
|
+
SIMSIMD_PUBLIC void simsimd_cos_i8_sierra(simsimd_i8_t const *a, simsimd_i8_t const *b, simsimd_size_t n,
|
|
2286
|
+
simsimd_distance_t *result) {
|
|
2287
|
+
|
|
2288
|
+
__m256i ab_i32_vec = _mm256_setzero_si256();
|
|
2289
|
+
__m256i a2_i32_vec = _mm256_setzero_si256();
|
|
2290
|
+
__m256i b2_i32_vec = _mm256_setzero_si256();
|
|
2291
|
+
|
|
2292
|
+
simsimd_size_t i = 0;
|
|
2293
|
+
for (; i + 32 <= n; i += 32) {
|
|
2294
|
+
__m256i a_i8_vec = _mm256_lddqu_si256((__m256i const *)(a + i));
|
|
2295
|
+
__m256i b_i8_vec = _mm256_lddqu_si256((__m256i const *)(b + i));
|
|
2296
|
+
ab_i32_vec = _mm256_dpbssds_epi32(ab_i32_vec, a_i8_vec, b_i8_vec);
|
|
2297
|
+
a2_i32_vec = _mm256_dpbssds_epi32(a2_i32_vec, a_i8_vec, a_i8_vec);
|
|
2298
|
+
b2_i32_vec = _mm256_dpbssds_epi32(b2_i32_vec, b_i8_vec, b_i8_vec);
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
// Further reduce to a single sum for each vector
|
|
2302
|
+
int ab = _simsimd_reduce_i32x8_haswell(ab_i32_vec);
|
|
2303
|
+
int a2 = _simsimd_reduce_i32x8_haswell(a2_i32_vec);
|
|
2304
|
+
int b2 = _simsimd_reduce_i32x8_haswell(b2_i32_vec);
|
|
2305
|
+
|
|
2306
|
+
// Take care of the tail:
|
|
2307
|
+
for (; i < n; ++i) {
|
|
2308
|
+
int ai = a[i], bi = b[i];
|
|
2309
|
+
ab += ai * bi, a2 += ai * ai, b2 += bi * bi;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
*result = _simsimd_cos_normalize_f32_haswell(ab, a2, b2);
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
#pragma clang attribute pop
|
|
2316
|
+
#pragma GCC pop_options
|
|
2317
|
+
#endif // SIMSIMD_TARGET_SIERRA
|
|
2318
|
+
#endif // _SIMSIMD_TARGET_X86
|
|
1505
2319
|
|
|
1506
2320
|
#ifdef __cplusplus
|
|
1507
2321
|
}
|